from lxml import etree from src.LoggingClasses import SimpleLogger import src.settings as settings import datetime class DataContainer: def __init__(self): self.timestampData = [] self.jobId = None self.loggerId = None self.auth = None self.responseRootNode = etree.Element('response') self.responseTypeNode = None self.missingValue = None self.invalidXMLFlag = False self.bulk_insert = False self.overwrite = False # attribute values are checked for empty strings and None values # if an incorrect value is found the reading process stops and returns # an error. NOTE: for a None value an INVALID_XML message is returned and no # DB queries are executed. For an empty attribute value only the data for the # given device and time stamp is discarded and a status code MISSING_[ATTR_NAME] is returned, the rest of the data is saved. def readSetXMLRequest(self, xmlMsg): root = etree.fromstring(xmlMsg) setData = root.find('.//setData') if setData == None or setData==[]: setData = root.find('.//setdata') self.overwrite = True if setData.get('overwrite', 'false').lower() == "true" else False self.bulk_insert = True if setData.get('bulk', "false").lower() == "true" else False self.ignore_version = True if setData.get('ignoreVersion', "false").lower() == "true" else False self.loggerId = root.get('slid') if self.loggerId == '': self.missingValue = 'slid' return if self.loggerId == None: self.invalidXMLFlag = True return self.auth = root.get('auth') if self.auth == '': self.missingValue = 'auth' return if self.auth == None: self.invalidXMLFlag = True return for d in setData: tsdata = TimestampData() tsdata.readFromXML(d) if tsdata.invalidXMLFlag: settings.logger.error('INVALID_XML') self.invalidXMLFlag = True return if tsdata.missingValue!=None: settings.logger.error('MISSSING_' + tsdata.missingValue.upper()) self.responseAppendDataNode(tsdata.timestamp, 'MISSING_' + tsdata.missingValue.upper()) continue if tsdata.missingDeviceData!={}: for k,v in tsdata.missingDeviceData.items(): settings.logger.error('MISSING_' + v.upper()) self.responseAppendDeviceNode(tsdata.timestamp, k, 'MISSING_' + v.upper()) self.timestampData.append(tsdata) return def readGetXMLRequest(self,xmlMsg): root = etree.fromstring(xmlMsg) def writeXMLSetResponse(self, listDBErrors): for tsData in self.timestampData: sstr = "//data[@ts='"+tsData.timestamp+"']" dataNodeList = self.responseTypeNode.xpath(sstr) if dataNodeList==[]: dataNode=etree.Element('data', ts=tsData.timestamp) self.responseTypeNode.append(dataNode ) else: dataNode = dataNodeList[0] for dev in tsData.deviceData: d = etree.Element('device', sdid=dev.deviceId) if dev.deviceId not in listDBErrors.get(tsData.timestamp, []): sc = etree.Element('status', code = 'SUCCESS') else: sc = etree.Element('status', code = listDBErrors[tsData.timestamp].get(dev.deviceId)) d.append(sc) dataNode.append(d) def writeXMLGetResponse(self): pass # the message response type is set e.g. or def setResponseType(self,respType): self.responseTypeNode = etree.Element(respType) # append data node to XML response tree def responseAppendDataNode(self,ts): data = etree.Element(ts) self.responseTypeNode.append(data) # append a device node to the response XML tree def responseAppendDeviceNode(self, ts, sdid, code): deviceNode = etree.Element('device', sdid=sdid) statusNode = etree.Element('status', code=code) deviceNode.append(statusNode) dataNode = self.responseTypeNode.xpath("//data[@ts=ts]") if dataNode==[]: dataNode=etree.Element('data', ts=ts) self.responseTypeNode.append(dataNode) dataNode.append(deviceNode) def responseAppendStatusCodeNode(self, code): statusCode = etree.Element('status', code=code) self.responseTypeNode.append(statusCode) # returns the response tree as a string def getResponseMsg(self): self.responseRootNode.append(self.responseTypeNode) return etree.tostring(self.responseRootNode) class TimestampData: def __init__(self, timestamp=None): self.timestamp = timestamp self.deviceData = [] self.missingValue = None self.missingDeviceData = {} self.invalidXMLFlag = False self.invalidXMLDeviceData = [] def readFromXML(self, tree): tstamp = tree.get('ts') if tstamp == '': self.missingValue = 'ts' return if tstamp == None: self.invalidXMLFlag = True return # truncate timestamp seconds dt = datetime.datetime.strptime(tstamp, "%Y-%m-%d %H:%M:%S") truncatedTimestamp = dt.replace(second=0) self.timestamp = truncatedTimestamp.strftime('%Y-%m-%d %H:%M:%S') for devd in tree: devData = DeviceData() devData.readFromTree(devd) if devData.missingValue != None: self.missingDeviceData[devData.deviceId] = devData.missingValue continue if devData.invalidXMLFlag: self.invalidXMLFlag = True return self.deviceData.append(devData) class DeviceData: def __init__(self, deviceId = None, version= None): self.deviceId = deviceId self.version = version self.valueData = [] self.missingValue = None self.invalidXMLFlag = False def readFromTree(self, treeNode): self.deviceId = treeNode.get('sdid') if self.deviceId == '': self.missingValue = 'sdid' return if self.deviceId == None: self.invalidXMLFlag = True return self.version = treeNode.get('version') if self.version == '': self.missingValue = 'version' return if self.version == None: self.invalidXMLFlag = True return for n in treeNode: dataValue = Value() dataValue.readFromTreeNode(n) if (dataValue.value == '' or dataValue.value is None or dataValue.value.lower() == 'none' or 'nan' in dataValue.value.lower()) : dataValue.value = 'NULL' if dataValue.invalidXMLFlag: self.invalidXMLFlag = True return self.valueData.append(dataValue) class Value: def __init__(self, id = None, value = None, upid=None, udid=None, uvid=None, dtype=None, aggregate=None): self.id = id self.value = value self.missingValue = None self.invalidXMLFlag = False def readFromTreeNode(self, treeNode): self.id = treeNode.get('svid') if self.id == '': self.missingValue = 'svid' return if self.id == None: self.invalidXMLFlag = True return try: self.value = treeNode.text except Exception as e: pass class Time: def __init__(self, fromTS = None, toTS = None, interval = None, tz = None): self.fromTS = fromTS self.toTS = toTS self.interval = interval self.tz = tz def readFromTreeNode(self, treeNode): self.fromTS = treeNode.get('fromTS') self.toTS = treeNode.get('toTS') self.interval = treeNode('interval') self.tz = treeNode('tz')