From 58415dfdd7a48dca3d395cfc5246011b54d8a880 Mon Sep 17 00:00:00 2001 From: Nico Melone Date: Sat, 3 Jun 2023 11:42:26 -0500 Subject: [PATCH] added migration --- .DS_Store | Bin 10244 -> 12292 bytes RPi Mistaway Originals/device_base.py | 229 +++ RPi Mistaway Originals/main.py | 1464 +++++++++++++++++ RPi Mistaway Originals/mainMeshify.py | 154 ++ RPi Mistaway Originals/meshifyData.py | 109 ++ meshifyDrivers/.DS_Store | Bin 6148 -> 6148 bytes meshifyDrivers/mainHPRPI/mainMeshify.py | 25 +- migration/__pycache__/lattice.cpython-311.pyc | Bin 0 -> 20652 bytes migration/convertDynamoDB.ipynb | 202 +++ migration/convertVanity.ipynb | 67 + migration/lattice.py | 419 +++++ migration/migration.ipynb | 248 +++ 12 files changed, 2915 insertions(+), 2 deletions(-) create mode 100644 RPi Mistaway Originals/device_base.py create mode 100644 RPi Mistaway Originals/main.py create mode 100644 RPi Mistaway Originals/mainMeshify.py create mode 100644 RPi Mistaway Originals/meshifyData.py create mode 100644 migration/__pycache__/lattice.cpython-311.pyc create mode 100644 migration/convertDynamoDB.ipynb create mode 100644 migration/convertVanity.ipynb create mode 100644 migration/lattice.py create mode 100644 migration/migration.ipynb diff --git a/.DS_Store b/.DS_Store index d52e02295a8fcedeafa09d372a763090a39de961..dbbf972183059056f734374175e632cab21107a8 100644 GIT binary patch delta 720 zcmZn(Xi1P~U|?W$DortDV9)?EIe-{M3-B;7uy8RjC`^<$U^Lzsu$!MzX|jWW(quh> zrNW#HxeS>M=?p~-i3}w`Rz5@CWCy_pQByM$9R(vJvsxX6YC}U4104l(LzBsIlE(E| zbfgp~=OpFl=Kw8Zz#?51T$GoSpO+4lWdvd|h9HIjpj8SCzCf}VXl)`xIS^L@Mf`yd z0XZj=ArB~$!%&=5UR;orlb;0CwPa#a z*|{WX%t<#4PR`FQ06GQ+Ao}Zx^i6KQiwo2@9QhY_`I%fk;s`W}BuC`tr(l>-PpTmW z8901Gq({)Abn|5a3--+l9PBI;8*(;tORQj@948nk19TNDB%n07fwU`9TIAj=$nl zrJEB(EZ8@5E39CjSQsM73RDdQ3P24Eu1FfWHw$vSXP(Tj /etc/resolv.conf") + + + def sendtodbLoc(self, ch, channel, value, timestamp, deviceName, mac): + + + #this will add your derived nodes the master nodes list, allowing them to receive sets!! + localNodesName = deviceName + "_" + str(ch) + "99" + + if not self.localNodes.has_key(localNodesName): + self.localNodes[localNodesName] = True + self.nodes[localNodesName] = self + + #make the techname + lst = textwrap.wrap(str(mac), width=2) + tech = "" + for i in range(len(lst)): + tech += lst[i].lower() + ":" + + + chName2 = '_[' + tech + + if int(ch) < 10: + ch = "0" + str(int(ch)) + + if len(ch) > 2: + ch = ch[:-2] + + dname = deviceName + chName2 + str(ch) + ":98]!" + + csplit = re.split(r"(.*?)_\[(.*?)\]", dname) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + if int(timestamp) == 0: + timestamp = self.getTime() + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodbDevJSON(self, ch, channel, value, timestamp, deviceName): + + if int(ch) < 10: + ch = "0" + str(int(ch)) + dname = deviceName + self.chName2 + str(ch) + ":99]!" + if int(timestamp) == 0: + timestamp = self.getTime() + + csplit = re.split(r"(.*?)_\[(.*?)\]", dname) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":%s}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodbLora(self, ch, channel, value, timestamp, deviceName): + + if ":" not in ch: + ch = ch[0:2] + ":" + ch[2:4] + + #this will add your derived nodes the master nodes list, allowing them to receive sets!! + localNodesName = deviceName + "_" + str(ch).replace(':', "") + + if not self.localNodes.has_key(localNodesName): + self.localNodes[localNodesName] = True + self.nodes[localNodesName] = self + + + + dname = deviceName + self.chName2 + str(ch) + "]!" + + csplit = re.split(r"(.*?)_\[(.*?)\]", dname) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + if int(timestamp) == 0: + timestamp = self.getTime() + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower, channel) + print topic + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodbDev(self, ch, channel, value, timestamp, deviceName): + + + #this will add your derived nodes the master nodes list, allowing them to receive sets!! + localNodesName = deviceName + "_" + str(ch) + "99" + + if not self.localNodes.has_key(localNodesName): + self.localNodes[localNodesName] = True + self.nodes[localNodesName] = self + + if int(ch) < 10: + ch = "0" + str(int(ch)) + + dname = deviceName + self.chName2 + str(ch) + ":99]!" + + + + if int(timestamp) == 0: + timestamp = self.getTime() + + csplit = re.split(r"(.*?)_\[(.*?)\]", dname) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodbCH(self, ch, channel, value, timestamp): + + + if int(ch) < 10: + ch = "0" + str(ch) + + dname = self.chName + str(ch) + ":99]!" + + + + if int(timestamp) == 0: + timestamp = self.getTime() + + csplit = re.split(r"(.*?)_\[(.*?)\]", dname) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodb(self, channel, value, timestamp): + + if int(timestamp) == 0: + timestamp = self.getTime() + if timestamp < 1400499858: + return + else: + timestamp = str(int(timestamp) + int(self.offset)) + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + def sendtodbJSON(self, channel, value, timestamp): + + if int(timestamp) == 0: + timestamp = self.getTime() + if timestamp < 1400499858: + return + else: + timestamp = str(int(timestamp) + int(self.offset)) + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + msg = """[{"value":%s}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + def getTime(self): + return str(int(time.time() + int(self.offset))) + + + + diff --git a/RPi Mistaway Originals/main.py b/RPi Mistaway Originals/main.py new file mode 100644 index 0000000..818f75a --- /dev/null +++ b/RPi Mistaway Originals/main.py @@ -0,0 +1,1464 @@ +import sys +import string +import socket, fcntl, struct +import client as paho +import os +import re +import time +import ssl +try: + import json +except: + import simplejson as json +import thread +import threading +import random +import base64 + +from meshifyData import meshifyData +#from devices import mainMistaway, m1, apg #, gds #cam, +#from devices import gdsMT as gds +import Queue +import pickle +import urllib2 +import urllib +try: + from sqlQueue import myqueue as SQLQ +except: + pass +try: + import SkyWaveDataAPI + SAT = True +except: + SAT = False + +try: + import schedule + sced = True +except: + sced = False + +unitName = "mainMeshify" + +broker = "mq194.imistaway.net" + +root = os.getcwd() + "/" +#non secure +port = 1883 + +LORA = True +try: + import lora_main +except: + LORA = False + + + +#secure port +#port = 1883 + +#driver for a virtual device called "main", it can never send data up to the website, but it hold methods that +#can be called easier than others because it has no unquie name assosicated with it +class main(): + + def __init__(self, name, number, mac, q, mcu, company, offset, mqtt, nodes, topics): + self.topics = topics + self.nodes = nodes + self.offset = offset + self.company = company + self.name = name + self.number = number + self.q = q + self.mqtt = mqtt + self.deviceName = name #+ '_[' + mac + ':' + number[0:2] + ':' + number[2:] + ']!' + print 'device name is:' + print self.deviceName + mac2 = mac.replace(":", "") + self.mac = mac2.upper() + self.mcu = mcu + self.count = 0 + self.dst = "" + #queue for sets to the mesh network will handeled through a queue in this main driver + self.meshQ = Queue.Queue() + version = "12" # 6 - mistification # 5 - updated for SAT data and generic sets. 4 - devices changed to drivers for dia + + # self.sendtodb("version", version, 0) + thread.start_new_thread(self.registerThread, ()) + + #pickle.dump( version, open( "coreVersion.p", "wb" ) ) + + def sendtodb(self, channel, value, timestamp): + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" # + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + + + def getTime(self): + return str(int(time.time() + int(self.offset))) + + + def main_mcuupdate(self, name, value): + success = self.mcu.firmUpdate(value) + + if success == True: + val = "update to " + value + " was a success" + elif success == False: + print "you need to reboot and pray you didn't brick the MCU" + val = "update to " + value + " failed" + else: + val = "update to " + value + " failed because " + str(success) + print "you need to reboot and pray you didn't brick the MCU" + + + #reseting the MCU also resets the Cellular modem, so we need to redo our mqtt connection once the modem comes back to life + time.sleep(180) + try: + self.mqtt.reconnect() + except Exception,e: + print str(e) + os.system("/root/reboot") + self.sendtodb(name, val, 0) + + def normalThread(self): + time.sleep(10) + os.system("/root/normalStart.sh") + + def debugThread(self): + time.sleep(10) + os.system("/root/debugStart.sh") + + def rebootThread(self): + time.sleep(10) + os.system("/root/reboot") + + def main_normal(self, name, value): + + if int(value) == 1: + + thread.start_new_thread(self.normalThread, ()) + return True + else: + return False + + def main_debug(self, name, value): + + if int(value) == 1: + + thread.start_new_thread(self.debugThread, ()) + return True + else: + return False + + def main_reboot(self, name, value): + if bool(value) == True: + #reset the modem + #self.mcu.resetModem() + #time.sleep(2) + + #resest the MCU + #try: + # os.system("/root/mcu2reset") + #except: + # pass + thread.start_new_thread(self.rebootThread, ()) + return True + + #os.system("/root/reboot") + + def main_SAT(self, name, value): + print "SAT is SET in MAIN" + st = "date -s @" + str(int(float(value))) + os.system(st) + + return True + + def registerThread(self): + while True: + time.sleep(3600 * 24) + + try: + os.system("/usr/bin/ntpdate pool.ntp.org") + os.system("/usr/sbin/ntpdate pool.ntp.org") + except: + pass + + try: + for name, driver in self.nodes.iteritems(): + + + try: + driver.offset = self.offset + driver.company = self.company + + + except Exception,e: + print e + except: + pass + + def main_register(self, name, value): + #try and sync the time + + try: + #setData = """$$$%s""" % ("main.register", "On", 1) + setData = {} + setData['name'] = 'main.register' + setData['value'] = 'On' + setData['id'] = 1 + print setData + self.meshQ.put(setData) + except: + pass + + try: + meshData = meshifyData.meshifyData(self.mac) + self.offset, self.dst, self.company = meshData.getdata() + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + + print ("meshify/sets/" + str(self.company) + "/" + self.mac + "/#") + + self.mqtt.subscribe(("meshify/sets/" + company + "/" + uniqueID.upper() + "/#"), 0) + self.mqtt.subscribe(("meshify/files/" + company + "/" + uniqueID.upper() + "/#"), 0) + + self.topics.append("meshify/sets/" + company + "/" + uniqueID.upper() + "/#") + self.topics.append("meshify/files/" + company + "/" + uniqueID.upper() + "/#") + + plcfreshID = uniqueID[:18] + "01:99" + + self.mqtt.subscribe(("meshify/sets/" + company + "/" + plcfreshID.upper() + "/#"), 0) + self.mqtt.subscribe(("meshify/files/" + company + "/" + plcfreshID.upper() + "/#"), 0) + + self.topics.append("meshify/sets/" + company + "/" + plcfreshID.upper() + "/#") + self.topics.append("meshify/files/" + company + "/" + plcfreshID.upper() + "/#") + + except Exception,e: + print e + for name, driver in self.nodes.iteritems(): + + + try: + driver.offset = self.offset + driver.company = self.company + driver.register() + + except Exception,e: + print e + + return True + + #this is where I need to put the function to have all of my devices check in + +class meshifyMain(): + + + + + def __init__(self): + + + #add google nameserver + os.system("/bin/echo nameserver 8.8.8.8 > /etc/resolv.conf") + #make dictionary for xbee sets + + #check and see if the drivers folder is there: + #if not os.path.exists("drivers"): + # #make the drivers dir + # os.makedirs("drivers") + self.MCU_ON = False + #marker or wether or not the device is a fly + self.FLY = False + self.reconnecting = False + self.mqttQ = Queue.Queue() + #get MAC address, if this breaks, you are [no longer] screwed + try: + mac = self.getHwAddr('eth0') + print "success getting mac" + except: + print "error getting mac" + try: + from uuid import getnode as get_mac + mac = get_mac() + mac = hex(mac).replace('0x', '') + n = 2 + mac = [mac[i:i+n] for i in range(0, len(mac), n)] + newMac = "" + for i in mac: + newMac = newMac + i + ":" + + mac = newMac[:-1] + except: + mac = "12:34:56:78:91:23" + mac = str(mac) + self.OK2Send = True + self.zigMac = mac + self.deviceUrlList = [] + self.deviceVersions = {} + try: + # todo: change this to sd card if present on some devices + self.SQLQ = SQLQ.SqliteQueue(root + "sqlQueue/q2") + except: + pass + + #varible for if the device is connected to the satellites. + self.SAT_COM = False + + + #if schedule module is there, then set up the global + try: + if sced == True: + self.schedule = schedule.schedule(self.sets, self.mqttQ, self.getTime) + else: + self.schedule = False + except Exception,e: + print "####################" + print e + + + #try and sync the time + try: + os.system("/usr/bin/ntpdate pool.ntp.org") + os.system("/usr/sbin/ntpdate pool.ntp.org") + except: + pass + + + #dictionary of all the nodes attched + self.nodes = {} + + mac = mac.replace(":", "") + mac = mac.upper() + print "here is the mac address" + print mac + + #get your time offset, DST value and company id from meshify + try: + meshData = meshifyData.meshifyData(mac) + self.offset, self.dst, self.companyId = meshData.getdata() + except: + print "didn't work on api to get meshify data" + + + self.mac = mac + + #start the debug thread: + thread.start_new_thread(self.debugThread, ()) + + + #set up placeholder for self.mqtt + + self.mqtt = paho.Client(client_id=self.mac, clean_session=True) + + + + #change to false for mqtt.meshify.com + #self.mqtt.tls_insecure_set(True) + #self.mqtt.tls_set(root + "ca.crt", certfile=root + "client.crt", keyfile=root + "client.key", cert_reqs=ssl.CERT_NONE) + self.mqtt.username_pw_set("admin", "columbus") + + print "now I'm here" + #Set up the last will and testiment to tell the system that I'm disconneted + lc = int(self.getTime()) + 30 + + + + self.deviceName = unitName + '_[' + self.zigMac + ':' + "00" + ':' + "00" + ']!' + self.deviceNameMain = "mainMeshify" + '_[' + self.zigMac + ':' + "00" + ':' + "00" + ']!' + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "connected") + msg = """[{"value":"%s"}]""" % ("false") + self.mqtt.will_set(topic, msg, 2) + + print "will set up on topic: " + topic + + #tell mqtt what to do on connect + self.mqtt.on_connect = self.on_connect + + #tell mqtt which function to call when a message is received + self.mqtt.on_message = self.on_message + self.mqtt.on_disconnect = self.on_disconnect + + #make conneciton to the MCU (only on M1 V5 right now): + + + try: + from mcu import mcu_main + self.MCU_ON = False + except: + self.MCU_ON = False + + try: + if self.MCU_ON == True: + print "mcu is on" + try: + print "mcu loading" + self.mcu = mcu_main.mcu_main() + + #added these for ifttt actions + self.mcu.mac = self.mac + self.mcu.on_message = self.on_message + self.mcu.FLY = self.FLY + + + #adding this global for turning all data coming out of the device off + self.mcu.cutAllDataOff = False + + print "mcu loaded" + #now see if we are a fly + flyTries = 0 + self.mcu.xbeeCom(0) + self.mcu.xbee.write("+++") + time.sleep(1) + self.mcu.xbee.write("ATFR" +chr(13)) + time.sleep(1) + while flyTries < 2: + #self.mcu.send_mcu("""{\"SET\": [{\"XBEECOM\": \"0\"}]}\n""") + flyTries += 1 + self.mcu.xbee.read() + self.mcu.xbee.write("+++") + time.sleep(2) + self.mcu.xbee.write("ATDD" +chr(13)) + time.sleep(3) + response = self.mcu.xbee.read() + if "777" in response or LORA: + print "I'm a fly" + #self.mcu.spiOn(0) + self.FLY = True + self.mcu.FLY = self.FLY + if LORA: + #self.mcu.xbee = lora_main.lora() + #self.mcu.xbee.fileTransfer = False + pass + #set the xbee to talk to the coordinator + #this needs to be working for network switching + #self.mcu.xbee.atCommandSet("DH", "0") + #self.mcu.xbee.atCommandSet("DL", "FFFF") + thread.start_new_thread(self.xbeeGetThread, ()) + break + else: + self.FLY = False + except Exception, e: + print e + self.mcu.FLY = self.FLY + self.FLY = False + else: + print "MCU NOT ON" + self.mcu = None + + + except Exception, e: + print e + + + #while True: + # time.sleep(3) + # print self.mcu.getDict() + + #turn off connected LED's + if self.MCU_ON: + self.mcu.FLY = self.FLY + self.mcu.ledControl(2,0,0) + self.mcu.ledControl(3,0,0) + + + + + self.topics = [] + + #we need an api to get some of the unique data for the topic structure + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + self.topics.append("meshify/sets/194/" + uniqueID.upper() + "/#") + self.topics.append("meshify/files/194/" + uniqueID.upper() + "/#") + self.topics.append("meshify/sets/" + uniqueID.upper() + "/" + self.mac + "/#") + self.topics.append("meshify/files/" + uniqueID.upper() + "/" + self.mac + "/#") + + plcfreshID = uniqueID[:18] + "01:99" + self.topics.append("meshify/sets/194/" + plcfreshID.upper() + "/#") + self.topics.append("meshify/files/194/" + plcfreshID.upper() + "/#") + self.topics.append("meshify/sets/" + plcfreshID.upper() + "/" + self.mac + "/#") + self.topics.append("meshify/files/" + plcfreshID.upper() + "/" + self.mac + "/#") + + #wait a few seconds to connect + time.sleep(5) + + ##################################################### + # this is the loading of the devices, if it fails, we still need main to work + #order of operations: + # 1. From API + # 2. From Text file + # 3. From previous loads pickle file + # 4. Nothing, it will only run main (not mainMeshify, just main wich can do reboot and register) + + + #the json data should come back from an API, for now it will come from in the root/python_firmware deviceList.txt + + try: + try: + json_data = open('deviceList.txt') + data = json.load(json_data) + data = meshData.checkConfig() + if len(data) < 1: + raise ValueError('too short') + except: + json_data = open('deviceList.txt') + data = json.load(json_data) + #build a list of urls for your device driver buckets + for i in data: + print i + #check and see if we are running the dia, if so then kill the mcu thread running the xbee so the dia can have full access to the xbee + if i == "dia" or i == "spider": + self.mcu.xbee.runXbee = False + self.deviceUrlList.append(data[i]) + print i + print data[i] + + pickle.dump( self.deviceUrlList, open( "deviceUrls.p", "wb" ) ) + except Exception,e: + print "####################" + print e + #if we can't get it from the web, look for the pickled version + try: + self.deviceUrlList = pickle.load( open( "deviceUrls.p", "rb" ) ) + except: + print "couldn't load devices from pickle" + + + + + + + + #try and load your versions dictionary, if this can't load, then load all drivers for the first time + try: + self.deviceVersions = pickle.load( open( root + "deviceVersions.p", "rb" ) ) + except: + print "couldn't load devices Versions from pickle" + + self.deviceList = {} + + + if self.FLY == False: + for i in self.deviceUrlList: + try: + print i + data = json.load(urllib.urlopen(i + "config.txt")) # json.load(urllib2.urlopen("http://" + i + "config.txt")) + #data = urllib2.urlopen(("http://" + i + "config.txt")) + #data = data.read() + print data + #data = json.load(data) + + #download the files + print "trying to download the files" + try: + if int(data["releaseVersion"]) > int(self.deviceVersions[(data["deviceName"] + "_" + data["driverId"])]): + print "new version found in repo", data["releaseVersion"] + for x in data["files"]: + print (i + data["files"][x]) + urllib.urlretrieve((i + data["files"][x]), ("./drivers/" + (data["files"][x]))) + else: + print "we have the latest version for: " + (data["deviceName"] + "_" + data["driverId"]) + " of " + str(self.deviceVersions[(data["deviceName"] + "_" + data["driverId"])]) + except: + print "probably didn't have any files to start with, loading all files" + for x in data["files"]: + print (i + data["files"][x]) + urllib.urlretrieve((i + data["files"][x]), ("./drivers/" + (data["files"][x]))) + + dList = [ data["driverFileName"].replace(".py", ""), data["deviceName"], data["driverId"], data["releaseVersion"]] + self.deviceList[(data["deviceName"] + "_" + data["driverId"])] = dList + print self.deviceList + except Exception,e: + print e + continue + + + #if our device list is still empty, try and grab the saved one + if len(self.deviceList) < 1: + #get the old device list + try: + dl = pickle.load( open( "deviceList.p", "rb" ) ) + self.deviceList = dl + print self.deviceList + except Exception,e: + print e + self.deviceList = {} + print "couldn't load deviceList from pickle" + else: + pickle.dump( self.deviceList, open( "deviceList.p", "wb" ) ) + + + + + + + + + try: + self.main = main('main', '', self.zigMac, self.mqttQ, self.mcu, self.companyId, self.offset, self.mqtt, self.nodes, self.topics) + except Exception,e: + print e + + + self.nodes["main"] = self.main + + + + #filename, node name, 4 digit identifier, version number (must be an integer) + + + #gds deviceList + #deviceList = [ ["mainMistaway", "mainMistaway", "0000", "1"], ["gdsMT", "gdsc", "0005", "1"] ] + + #gate Device List + #deviceList = [ ["mainMistaway", "mainMistaway", "0000", "1"], ["gate", "gate", "0006", "1"] ] + + for device in self.deviceList: + print "trying to load: " + device + + try: + device = self.deviceList[device] + + #subscribe to the device first, if it breaks, this makes it possible to fix all of this type at once + self.topics.append("meshify/files/" + str(device[1]).upper() + "/#") + self.topics.append("meshify/sets/" + str(device[1]).upper() + "/#") + topic = str(("meshify/sets/" + str(device[1]).upper() + "/#")) + print "######", topic + self.mqtt.subscribe(topic, 0) + topic = str(("meshify/files/" + str(device[1]).upper() + "/#")) + print "######", topic + self.mqtt.subscribe(topic, 0) + + + #import the file from the devices folder + imported_module = __import__("drivers." + str(device[0])) + #import the code from the device driver + fileImport = getattr(imported_module, str(device[0])) + #import the driver class from the file + funct = getattr(fileImport, "start") + #start this instance and add it to the devices dictionary + self.nodes[(str(device[1]) + "_" + str(device[2]))] = funct(name=str(device[1]), number=str(device[2]), mac=self.zigMac, Q=self.mqttQ, mcu=self.mcu, companyId=self.companyId, offset=self.offset, mqtt=self.mqtt, Nodes=self.nodes) + + + + #add name and version to a dictionary for pickling + self.deviceVersions[(str(device[1]) + "_" + str(device[2]))] = self.nodes[(str(device[1]) + "_" + str(device[2]))].version + pickle.dump( self.deviceVersions, open(root + "deviceVersions.p", "wb" ) ) + + + + + + + except Exception,e: + print e + lc = self.getTime() + value = "Failed Loading: " + str(device[2]) + " on startup with error: " + str(e) + msg = """[ { "value":"%s", "msgId":"%s" } ]""" % ((value), "1") + topic = "meshify/errors/" + self.mqttQ.put([topic, str(msg), 2]) + print e + + + + if self.FLY == False: + print "made it to here, not a fly" + #start logging data!!! + self.reconnecting = True + thread.start_new_thread(self.sqlPoolThread, ()) + + #connect to broker + #thread.start_new_thread(self.connect_thread, ()) + while True: + try: + self.connect_to_broker() + self.reconnecting = False + try: + os.system("/usr/bin/ntpdate pool.ntp.org") + os.system("/usr/sbin/ntpdate pool.ntp.org") + os.system("ntpdate pool.ntp.org") + except: + pass + break + except: + print "didn't work this time" + time.sleep(30) + + + #tell the MQTT client to run forever!! + print "made it here" + while True: + print "##### here is the connection status #### ", str(self.mqtt._state) + try: + if not self.FLY: + print self.mqtt.loop() + if self.MCU_ON: + if self.mcu.cutAllDataOff: + print "All Data is Being cut off, no outbound alloud" + time.sleep(5) + continue + + if not self.mqttQ.empty(): #or self.SQLQ.getLen() > 0: + if str(self.mqtt._state) == "1" or self.FLY == True: + + try: + print self.SQLQ.getLen() + if self.SQLQ.getLen() > 0: + loopCount = 0 + while self.SQLQ.getLen() > 0: + loopCount += 1 + if loopCount > 20: + loopCount = 0 + if not self.FLY: + print self.mqtt.loop() + val = self.SQLQ.popleft() + if not self.FLY: + resp = self.mqtt.publish(val[0], val[1], val[2]) + + else: + if val[0].split("/")[1] == "db": + xeebVal = json.dumps(json.loads(val[1])[0]["value"]) + if xeebVal.startswith('"') and xeebVal.endswith('"'): + xeebVal = xeebVal[1:-1] + upld = val[0].split("/")[3] + "/" + val[0].split("/")[4] + "/" + val[0].split("/")[5] + "/" + xeebVal + self.xbeeSend(upld) + time.sleep(.5) + else: + respId = (''.join(random.choice('0123456789ABCDEF') for i in range(4))) + self.xbeeSend("%%" + val[0] + "%%" + json.dumps(json.loads(val[1])[0]["value"]) + "%%" ) + + except Exception, e: + print e + print "no SQL Queue on this device" + + try: + val = self.mqttQ.get(block=False, timeout=1) + #print "Outputting: ", val + if not self.FLY: + resp = self.mqtt.publish(val[0], val[1], val[2]) + elif self.mcu.xbee.fileTransfer == True: + #if we are sending a file, put this data point back in the Q + self.mqttQ.put(val) + time.sleep(3) + continue + else: + if val[0].split("/")[1] == "db": + xeebVal = json.dumps(json.loads(val[1])[0]["value"]) + if xeebVal.startswith('"') and xeebVal.endswith('"'): + xeebVal = xeebVal[1:-1] + upld = val[0].split("/")[3] + "/" + val[0].split("/")[4] + "/" + val[0].split("/")[5] + "/" + xeebVal + self.xbeeSend( upld ) + else: + self.xbeeSend("%%" + val[0] + "%%" + json.dumps(json.loads(val[1])[0]["value"]) + "%%" ) + time.sleep(2) + #print "####### here is the response" + #make a dictionary of response codes xbee + #print resp + except: + print "Q had an error" + time.sleep(1) + loopCount = 0 + while self.mqttQ.qsize() > 20: + loopCount += 1 + if loopCount > 20: + loopCount = 0 + if not self.FLY: + print self.mqtt.loop() + try: + val = self.mqttQ.get(block=False, timeout=1) + #print "Outputting: ", val + + if not self.FLY: + resp = self.mqtt.publish(val[0], val[1], val[2]) + elif self.mcu.xbee.fileTransfer == True: + #if we are sending a file, put this data point back in the Q + self.mqttQ.put(val) + time.sleep(3) + continue + else: + if val[0].split("/")[1] == "db": + xeebVal = json.dumps(json.loads(val[1])[0]["value"]) + if xeebVal.startswith('"') and xeebVal.endswith('"'): + xeebVal = xeebVal[1:-1] + upld = val[0].split("/")[3] + "/" + val[0].split("/")[4] + "/" + val[0].split("/")[5] + "/" + xeebVal + self.xbeeSend(upld) + + else: + self.xbeeSend("%%" + val[0] + "%%" + json.dumps(json.loads(val[1])[0]["value"]) + "%%" ) + time.sleep(2) + + except: + print "Q had an error" + time.sleep(1) + + + + except Exception,e: + print e + time.sleep(10) + if self.reconnecting == False: + self.recon() + time.sleep(.5) + + + + def connect_thread(self): + while True: + try: + print "####### connecting to the broker ########" + self.connect_to_broker() + + + except Exception,e: + print(str(e)) + print "didn't work to connect, restarting...." + os.system('/root/reboot') + + def connect_to_broker(self): + + if self.companyId == "1": + meshData = meshifyData.meshifyData(self.mac) + self.offset, self.dst, self.companyId = meshData.getdata() + + + self.mqtt.connect(broker, port, keepalive=120) + self.topic_sub() + + #tell the MQTT client to run forever!! + #self.mqtt.loop_forever() + + + def on_disconnect(self, mosq, userdata, rc): + print "################ DISCONECCTED #################" + + #turn off connected LED + if self.MCU_ON: + self.mcu.ledControl(2,0,0) + + self.recon() + + + def connect_check(self): + #this funtion checks to see how many times its tried to reconnect in the 3 minutes that it goes to sleep + #if it wakes up and you are on the same attempt, then its froze and needs to reboot + local_count = self.count + time.sleep(180) + if local_count == self.count: + #Dont reboot if connected to SAT + if self.SAT_COM == True: + pass + else: + os.system('/root/reboot') + + + + def sqlPoolThread(self): + + #while spooling the data I am going to attempt a handshake with the SkyWave satellites + #handshake data will include my mac address + #the web will return my current UTC time, which by the time I get it, it will be off by a few seconds, but whos counting ;) + #if it connects, it will set a variable called SAT_COM to True, by way of the main device + print "starting up sat connection" + + #start a Queue for the sending up of data to the SATS + satQ = Queue.Queue() + if SAT == True and self.FLY == False: + thread.start_new_thread(SkyWaveDataAPI.skywave, (self.sets, self.mac, self.stateData, satQ, self.mcu)) + + #wait 1 minute to see if we connect to the SATs + time.sleep(60) + try: + while self.reconnecting == True: + val = self.mqttQ.get() + try: + if self.SQLQ.getLen() > 5000: + #delete oldest value + trash = self.SQLQ.popleft() + + except: + pass + if self.SAT_COM == True: + print "sending up data to sats" + satQ.put(val) + else: + print "storing up data for later 3g connection" + try: + self.SQLQ.append(val) + except: + pass + except: + pass + + def recon(self): + self.reconnecting = True + thread.start_new_thread(self.sqlPoolThread, ()) + count = 0 + while True: + + if count > 2000: + if self.SAT_COM == True: + count = 0 + else: + break + + count += 1 + print count + try: + self.count = count + thread.start_new_thread(self.connect_check, ()) + self.mqtt.reconnect() + self.count = 0 + self.reconnecting = False + break + except: + print "couldn't reconnect, retrying in 30 seconds" + os.system("/bin/echo nameserver 8.8.8.8 > /etc/resolv.conf") + os.system("/sbin/ifup 3g") + time.sleep(30) + + + if count > 2000 and not self.FLY: + #don't reboot if connected to SAT + print "rebooting now" + os.system('/root/reboot') + + + def on_connect(self, mosq, userdata, rc): + + #turn connected LED + if self.MCU_ON: + self.mcu.ledControl(2,1,0) + self.mcu.ledControl(3,0,0) + else: + print "on_connect() MCU_OFF = False!!!!!" + + + + #stop using sat data + self.SAT_COM = False + + # let the watchdog know we are not on SAT anymore + os.system('/bin/echo False > /root/SAT') + + #wait a few seconds for the connection to be made solid + time.sleep(4) + + #set the channel connected to true + lc = self.getTime() + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "connected") + msg = """[{"value":"%s"}]""" % ("true") + self.mqttQ.put([topic, msg, 2]) + + #set the network to 3g + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "net") + msg = """[{"value":"%s"}]""" % ("3g") + self.mqttQ.put([topic, msg, 2]) + + print(" ############### Connection returned " + str(rc) + " ###############") + self.topic_sub() + + + def topic_sub(self): + for topic in self.topics: + print topic + self.mqtt.subscribe(topic, 0) + + def sets(self, msg, sch=False): + entireMsg = msg + #[{"user":"demo@meshify.com","mac":"000CE373293D","company":"188","payload":{"name":"wipom_[00:0c:e3:73:29:3d:00:21]!.do2","value":"1","expires":"1389369695"},"msgId":4478}] + print "I got the set" + print msg + try: + data = json.loads(msg) + data = data[0] + msgId = str(data['msgId']) + data = data['payload'] + name = data['name'] + value = data['value'] + expires = data['expires'] + print name, value, expires + + #name = 'oulet_[00:13:a2:00:40:a0:48:cb]!.switch' + # grab the 4 digit unique device code from the end of the MAC address + + + n = name.split('.') + channel = n[1] + n = n[0] + + try: + #if the first part of the zigbee mac address is legit, meaning it comes from digi + #then I need to treat this like a mesh node and send it to the dia + #This is only for the SPIDER's Dia Nodes + #the best check is to see if the addresses are derived from the same address + if name.split('.')[0].split("_")[1].replace("[", "").replace(":", "").replace("]!", "")[:-10].upper() == "0013A2": + print "found an xbee" + #setData = """$$$%s""" % (name, value, msgId) + setData = {} + setData['name'] = name + setData['value'] = value + setData['id'] = msgId + print setData + self.main.meshQ.put(setData) + return + except: + print "couldn't determine if there was an xbee" + + + if n == "main": + if channel == "SAT": + print "got ping from sat cloud app, this mean we have a connection" + try: + if float(value) > 1422910000: + print "SAT CONNECTED!!!" + self.SAT_COM = True + #turn connected LED to amber + if self.MCU_ON: + self.mcu.ledControl(3,1,0) + self.mcu.ledControl(2,0,0) + #send connected = True + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s' % (self.companyId, self.mac, self.deviceName, "connected") + msg = """[ { "value":"%s", "timestamp":"%s" } ]""" % ("True", str(lc)) + self.mqttQ.put([topic, msg, 2]) + + #set the network to sat + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s' % (self.companyId, self.mac, self.deviceName, "net") + msg = """[ { "value":"%s", "timestamp":"%s" } ]""" % ("sat", str(lc)) + self.mqttQ.put([topic, msg, 2]) + + # let the watchdog know if we are on SAT + os.system('/bin/echo True > /root/SAT') + except: + print "didn't send the right date" + nodeName = "main" + nodeNumber = "" + + else: + m = n + m = m.split('_') + nodeName = m[0] + n = n.replace(']!', '') + n = n[-5:] + nodeNumber = n.replace(':', '') + nodeNumber = "_" + nodeNumber + except: + print "not valid JSON" + return + + + + #check and see if you are setting the scheduler + if channel.startswith("sch-"): + print "data now being sent to the scheduler" + try: + if self.schedule != False: + self.schedule.message(channel, json.loads(entireMsg)) + return + except: + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + print "BAD JSON" + return + + + + #this grabs the class for the driver for the node: #this should be called className + funcName = nodeName + nodeNumber + #this grabs the method for that channel inside of the driver for that node: + funcChan = nodeName + '_' + channel + print funcName + #try: + # # if nodes[funcName] != undefined + # #channelCallback = getattr(nodes[funcName], funcChan) + # #success = channelCallback(channel, value) + # func = getattr(self, funcName) + #except: + # print 'no Set callback found for channel: ' + funcName + #else: + try: + #classFunc = getattr(func, funcChan) #func(value) + #success = classFunc(channel, value) + + #first try a specific callback set, the fundtion will look like: deviceName_0000 + try: + print "trying to find callback for:" + print funcChan + channelCallback = getattr(self.nodes[funcName], funcChan) + print channelCallback + success = channelCallback(channel, value) + except: + print "looking for genericSet" + #now try a generic one, that looks like self.genericSet(self, channel, value, UnitNumber) Unit number is the second to last 2 digits of the tech name + channelCallback = getattr(self.nodes[funcName], "genericSet") + try: + success = channelCallback(channel, value, nodeNumber[1:3], nodeNumber[1:]) + except: + success = channelCallback(channel, value, nodeNumber[1:3]) + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceNameMain) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + if success == True: + if int(msgId) == 0: + return 0 + lc = self.getTime() + if sch == False: + val = value + value = str(self.mac) + " Success Setting: " + channel + " To: " + value + msg = """[{"value":"%s","msgId":"%s"}]""" % (value, msgId) + """ """ + topic = "meshify/responses/" + msgId + self.mqttQ.put([topic, str(msg), 2]) + + confirm = {"channel": str(channel), "value": str(val), "status": "success"} + confirm = str(confirm).replace("'", '"') + msg = """[ { "value":%s, "msgId":"%s" } ]""" % (confirm, msgId) + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID, "commands") + self.mqttQ.put([topic, str(msg), 2]) + else: + return 0 + + else: + if sch == False: + lc = self.getTime() + if success == False: + reason = "(Internal Gateway/Device Error)" + else: + reason = success + val = value + value = str(self.mac) + " Failed Setting: " + channel + " To: " + value + " " + reason + msg = """[{"value":"%s","msgId":"%s"}]""" % (value, msgId) + topic = "meshify/responses/" + msgId + self.mqttQ.put([topic, str(msg), 2]) + + confirm = {"channel": str(channel), "value": str(val), "status": "failed"} + confirm = str(confirm).replace("'", '"') + msg = """[ { "value":%s, "msgId":"%s" } ]""" % (confirm, msgId) + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID, "commands") + self.mqttQ.put([topic, str(msg), 2]) + else: + return 1 + except: + if int(msgId) == 0: + return 2 + if sch == False: + lc = self.getTime() + val = value + value = str(self.mac) + " Failed Setting: " + channel + " To: " + value + " (No Callback Found)" + msg = """[{"value":"%s","msgId":"%s"}]""" % (value, msgId) + topic = "meshify/responses/" + msgId + self.mqttQ.put([topic, str(msg), 2]) + print 'no Set callback found for channel: ' + funcName + + confirm = {"channel": str(channel), "value": str(val), "status": "no callback"} + confirm = str(confirm).replace("'", '"') + msg = """[ { "value":%s, "msgId":"%s" } ]""" % (confirm, msgId) + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID, "commands") + self.mqttQ.put([topic, str(msg), 2]) + else: + return 2 + + + #function to be called when a message is received + def handle_message(self, topic, payload, qos): + try: + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceNameMain) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + + print("Message received on topic "+topic+" with QoS "+str(qos)) + topics = topic.split("/") + if topics[1] == "files" and topics[4] == "write": + self.OK2Send = False + path = topics[5] + path = path.replace("$", "/") + print path + with open(path, 'wb') as fd: + fd.write(payload) + fd.close() + print "file written" + #update the channel mainMistaway_files, a dummy channel for keeping track of file transactions + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "files") + msg = """[{"value":"%s"}]""" % ((str(self.mac) + " File Written: " + path)) + self.mqttQ.put([topic, msg, 2]) + elif topics[1] == "files" and topics[4] == "get": + + self.OK2Send = False + path = topics[5] + mqttpath = path + path = path.replace("$", "/") + print path + f = open(path, 'rb') + byteArray = f.read() + #byteArray = bytes(byteArray) + topic = 'meshify/get/%s/%s/%s/%s' % (company, "_", uniqueID.lower(), mqttpath) + msg = byteArray + self.mqtt.publish(topic, msg, 0) + f.close() + print "message sent on topic: ", topic + + #update the channel mainMistaway_files, a dummy channel for keeping track of file transactions + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "files") + msg = """[{"value":"%s"}]""" % ((str(self.mac) + " File Sent: " + path)) + self.mqttQ.put([topic, msg, 2]) + elif topics[1] == "files" and topics[4] == "delete" and payload == "delete": + path = topics[5] + path = path.replace("$", "/") + val = "Success Deleting " + try: + os.remove(path) + except OSError, e: ## if failed, report it back to the user ## + val = "Error Deleting " + print ("Error: %s - %s." % (e.filename,e.strerror)) + + #update the channel mainMistaway_files, a dummy channel for keeping track of file transactions + lc = self.getTime() + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), "files") + msg = """[{"value":"%s"}]""" % ((val + path)) + self.mqttQ.put([topic, msg, 2]) + elif topics[1] == "sets": + self.sets(payload) + self.OK2Send = True + except Exception,e: + print e + self.OK2Send = True + print "error understanding the mqtt message" + + + def on_message(self, mosq, obj, msg): + + try: + print msg + + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceNameMain) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + plcfreshID = uniqueID[:18] + "01:99" + + #if the message has either a device name or the mac address of this unit then send to handle_message if not send to dia + #if the name isn't hex, then its an ascii group set + try: + num = int(msg.topic.split("/")[2], 16) + is_hex = True + except: + is_hex = False + + try: + #if the device is in the xbeeQ lookup dictionary, then pass it there + devName = json.loads(msg.payload)[0]["payload"]["name"].split(".")[0] + if devName in self.mcu.xbees: + #this is an xbee node running the dia + xbeeSet = self.mcu.xbees[devName] + xbeeSet.put([msg.topic, msg.payload]) + return + except Exception, e: + print e + print "error parsing set for xbee" + #case 1, the message is meant for me or I'm a fly + #case 2, the company ID is a number, not a word, try the set to the xbee again i guess?? + #case 3, the compnay id was actually a group, send it to both the + if msg.topic.split("/")[3] == uniqueID.upper() or msg.topic.split("/")[3] == plcfreshID.upper() or self.FLY == True: + thread.start_new_thread(self.handle_message, (msg.topic, msg.payload, msg.qos)) + elif is_hex: + xbeeSet = self.mcu.xbees[msg.topic.split("/")[3]] + xbeeSet.put([msg.topic, msg.payload]) + else: + #this is a group, so set both + thread.start_new_thread(self.handle_message, (msg.topic, msg.payload, msg.qos)) + xbeeSet = self.mcu.xbees[msg.topic.split("/")[3]] + xbeeSet.put([msg.topic, msg.payload]) + except: + pass + + + + # this retrieves the MAC address from the ethernet card + def getHwAddr(self, ifname): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', ifname[:15])) + return ''.join(['%02x:' % ord(char) for char in info[18:24]])[:-1] + + + + def stateData(self): + return self.SAT_COM, self.reconnecting + + def getTime(self): + return str(int(time.time() + int(self.offset))) + # Here is the spot for all of your channel set callbacks + # Callbacks are to be writen with the following nameing scheme: + # deviceName_DeviceId_ChannelName + # the function will take in the new value it is getting set to + # for any action to take place on that channel set you must define a callback + # with the name defined by the nameing scheme above. + + def debugThread(self): + + #this thread is reading the system output of the core as its runs and publishes it to + #the topic: meshify/debug/macaddress + + #TODO Use iMist2 rset tracing module + + try: + print 'Number of arguments:', len(sys.argv), 'arguments.' + if str(sys.argv[1]).lower().strip() == "debug=true" or str(sys.argv[1]).lower().strip() == "debug = true" or str(sys.argv[1]).lower().strip() == "true": + try: + if len(str(sys.argv[2]).lower().strip()) > 2: + fileLocation = str(sys.argv[2]).lower().strip() + else: + fileLocation = "/tmp/main.log" + except: + fileLocation = "/tmp/main.log" + + file = open(fileLocation, 'r+') + while 1: + where = file.tell() + line = file.readline() + if not line: + file.seek(where) + file.truncate(0) + time.sleep(1) + file.seek(0) + else: + topic = "meshify/debug/" + self.mac + msg = filter(lambda x: x in string.printable,str(line)) + self.mqttQ.put([topic, msg, 0]) + except Exception, e: + print "debug error" + print e + + + def xbeeSend(self, data): + + respId = (''.join(random.choice('0123456789ABCDEF') for i in range(4))) + data = data + respId + "$" + count = 0 + + while True and self.mcu.xbee.fileTransfer == False: + if count > 5: + print "failed getting response after 3 tries" + if self.xbeeConnected == True: + self.xbeeConnected = False + #broadcast to coordinator + #self.mcu.xbee.atCommandSet("DH", "0") + #self.mcu.xbee.atCommandSet("DL", "FFFF") + #turn off connected LED + if self.MCU_ON: + self.mcu.ledControl(2,0,0) + self.mcu.ledControl(1,0,0) + # let the watchdog know we are not connected to the SPIDER + os.system('/bin/echo False > /root/XBEE') + return False + count += 1 + try: + self.mcu.xbee.write(base64.b64encode(data.encode('utf-8'))) + except Exception,e: + print "error writing xbee to gateway" + print e + inner_count = 0 + while True and self.mcu.xbee.fileTransfer == False: + inner_count += 1 + time.sleep(.5) + if respId in self.xbeeResponseList: + print "id found!!!" + if self.xbeeConnected == False: + self.xbeeConnected = True + #turn connected LED + if self.MCU_ON: + self.mcu.ledControl(2,1,0) + self.mcu.ledControl(1,1,0) + # let the watchdog know we are connected to the SPIDER + os.system('/bin/echo True > /root/XBEE') + return True + elif inner_count > 12: + print "no response found" + break + + def xbeeGetThread(self): + #build a list of last 20 responses ie: if id in listofIds then OK + #when the ID's start populating, turn connected on!! LED + self.xbeeResponseList = [] + self.xbeeConnected = False + data = "" + while True: + if self.mcu.xbee.fileTransfer == True: + time.sleep(5) + continue + else: + try: + newData = self.mcu.xbee.read() + if newData != "": + data += newData + print data + if "$$" in data: + list_of_sets = data.split("$$") + if len( list_of_sets[len(list_of_sets) - 1]) < 1: + data = "" + del list_of_sets[-1] + else: + data = list_of_sets[len(list_of_sets) - 1] + del list_of_sets[-1] + for item in list_of_sets: + if len(item) == 4: + print "new response id", item + self.xbeeResponseList.append(item) + if len(self.xbeeResponseList) > 20: + self.xbeeResponseList.pop(0) + continue + print "we have a complete message" + topic = item.split("%%")[1] + payload = item.split("%%")[2] + self.handle_message(topic, payload, 1) + time.sleep(2) + else: + time.sleep(.5) + continue + + except Exception,e: + print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + print "xbee read error" + print e + time.sleep(1) + +def startMain(): + try: + test = meshifyMain() + except: + pass +startMain() \ No newline at end of file diff --git a/RPi Mistaway Originals/mainMeshify.py b/RPi Mistaway Originals/mainMeshify.py new file mode 100644 index 0000000..3ff7bc1 --- /dev/null +++ b/RPi Mistaway Originals/mainMeshify.py @@ -0,0 +1,154 @@ +import time +import os +try: + import json +except: + import simplejson as json +import thread +import threading +import re + + + + + +class start(threading.Thread): + + def __init__(self, name=None, number=None, mac=None, Q=None, mcu=None, companyId=None, offset=None, mqtt=None, Nodes=None): + threading.Thread.__init__(self) + self.daemon = True + self.offset = offset + self.company = companyId + self.name = name + self.number = number + self.q = Q + self.deviceName = name + '_[' + mac + ':' + number[0:2] + ':' + number[2:] + ']!' + print 'device name is:' + print self.deviceName + mac2 = mac.replace(":", "") + self.mac = mac2.upper() + self.version = "16" #mistification #added Nodes in v5 + self.finished = threading.Event() + + threading.Thread.start(self) + + + + #this is a required function for all drivers, its goal is to upload some piece of data + #about your device so it can be seen on the web + def register(self): + #self.mainMistaway_hb('hb', 'On') + self.sendtodb("connected", "true", 0) + + def stop (self): + self.finished.set() + self.join() + + def sendtodb(self, channel, value, timestamp): + if int(timestamp) == 0: + timestamp = self.getTime() + if timestamp < 1400499858: + return + csplit = re.split(r"(.*?)_\[(.*?)\]", self.deviceName) + nodeTypeName = csplit[1] + uniqueID = csplit[2] + company = "194" + + + try: + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) + print topic + if channel == "files": + #for the file structure I had to take off the " " around the value + msg = """[{"value":%s}]""" % (str(value)) + else: + msg = """[{"value":"%s"}]""" % (str(value)) + print msg + self.q.put([topic, msg, 0]) + except: + print "didn't work to send up MQTT data" + + def run(self): + #on startup send the version number + self.sendtodb("version", str(self.version), 0) + + if os.path.isfile('/root/python_firmware/drivers/device_base.py'): + print "found new device_base file" + os.system("/bin/mv -f /root/python_firmware/drivers/device_base.py /root/python_firmware/device_base.py") + os.system("/bin/rm -f /root/python_firmware/drivers/device_base.py") + os.system("/bin/rm -f /root/python_firmware/drivers/device_base.pyc") + + if os.path.isfile('/root/python_firmware/drivers/meshifyData.py'): + print "found new meshifyData file" + os.system("/bin/mv -f /root/python_firmware/drivers/meshifyData.py /root/python_firmware/meshifyData/meshifyData.py") + os.system("/bin/rm -f /root/python_firmware/drivers/meshifyData.py") + os.system("/bin/rm -f /root/python_firmware/drivers/meshifyData.pyc") + + if os.path.isfile('/root/python_firmware/drivers/main.py'): + print "found new main.py file" + os.system("/bin/mv -f /root/python_firmware/drivers/main.py /root/python_firmware/main.py") + os.system("/bin/rm -f /root/python_firmware/drivers/main.py") + os.system("/bin/rm -f /root/python_firmware/drivers/main.pyc") + time.sleep(0.5) + os.system('/root/reboot') + + while True: + try: + self.mainMeshify_hb('hb', 'On') + self.sendtodb("connected", "true", 0) + time.sleep(3600 * 4) + except Exception, e: + print e + + def mainMeshify_files(self, name, value): + name = 'files' + + + + + + dict = {} + for dirname, dirnames, filenames in os.walk(str(value)): + # print path to all subdirectories first. + + + print "##########################################" + print "new directory: " + dirname + print "##########################################" + # print path to all filenames. + tempDictParent = {} + for filename in filenames: + tempDict = {} + filepath = os.path.join(dirname, filename) + try: + fileMem = os.stat(filepath).st_size + fileDate = os.stat(filepath).st_mtime + except: + fileMem = "" + fileDate = "" + print filepath, fileMem, fileDate + tempDict["mem"] = fileMem + tempDict["date"] = fileDate + tempDictParent[filename] = tempDict + + dict[dirname] = tempDictParent + + + # Advanced usage: + # editing the 'dirnames' list will stop os.walk() from recursing into there. + if '.git' in dirnames: + # don't go into any .git directories. + dirnames.remove('.git') + + value = json.dumps(dict) + self.sendtodb(name, value, 0) + return True + + + def mainMeshify_hb(self, name, value): + self.sendtodb(name, value, 0) + + + def getTime(self): + return str(int(time.time() + int(self.offset))) + diff --git a/RPi Mistaway Originals/meshifyData.py b/RPi Mistaway Originals/meshifyData.py new file mode 100644 index 0000000..2317ea9 --- /dev/null +++ b/RPi Mistaway Originals/meshifyData.py @@ -0,0 +1,109 @@ +import urllib +try: + import json +except: + import simplejson as json +import pickle + +MAC = "00409D53168A" + +class meshifyData(): + + def __init__(self, MAC): + self.mac = MAC[0:6] + "FF-FF" + MAC[6:] + print "here is the mac: " + self.mac + #set the defaults + self.param_dict = {} + + def checkConfig(self): + + url = "https://f5rrbd3r45.execute-api.us-east-1.amazonaws.com/device_config?mac=" + self.mac + + try: + f = urllib.urlopen(url) + except: + print "Error opening url for remote config" + #return the defaults + return None + + try: + s = f.read() + #print s + if len(s) < 5: + return None + s = s[1:-1].replace("'", '"') + print s + data = json.loads(s) + #if we get there then replace the deviceList.txt + with open('/root/python_firmware/deviceList.txt', 'w') as myfile: + json.dump(data, myfile, indent=4) + return data + except Exception as e: + print e + #return the defaults + return None + + def checkAPI(self): + + + offset = -21600 + dst = False + companyId = "1" + + + url = "https://machines.meshify.com/api/gateway?macaddressForTimezone=" + self.mac + + try: + f = urllib.urlopen(url) + except: + print "Error opening url" + #return the defaults + return offset, dst, companyId + + try: + s = f.read() + print s + data = json.loads(s) + offset = int(data["offset"]) + dst = bool(int(data["dst"])) + print bool(int("0")) + companyId = data["companyId"] + return offset, dst, companyId + except Exception,e: + print e + #return the defaults + return -21600, False, "1" + + def getdata(self): + #if the API fails and the company ID of 1 is returned then you need to + #check and see if you have pickled anything. + #if it doesn't fail, and it gives you something other than 1 + #then you need to repickle the object + self.offset, self.dst, self.companyId = self.checkAPI() + if self.companyId == "1": + try: + self.param_dict = pickle.load( open( "params.p", "rb" ) ) + except: + print self.offset, self.dst, self.companyId + return self.offset, self.dst, self.companyId + try: + self.offset = self.param_dict["offset"] + self.dst = self.param_dict["dst"] + self.companyId = self.param_dict["companyId"] + except: + return -21600, False, "1" + + return self.offset, self.dst, self.companyId + + + else: + self.param_dict["offset"] = self.offset + self.param_dict["dst"] = self.dst + self.param_dict["companyId"] = self.companyId + pickle.dump( self.param_dict, open( "params.p", "wb" ) ) + print self.param_dict + print self.offset, self.dst, self.companyId + return self.offset, self.dst, self.companyId + + + diff --git a/meshifyDrivers/.DS_Store b/meshifyDrivers/.DS_Store index 47ee5f4adfbcdedbe9bcd1c706bf0d238434c439..8953481df5e96f4ce7bf8354d2b98ac935b2c4f1 100644 GIT binary patch delta 27 jcmZoMXffEZl$DWTvI3jf<~^);7$?tWlia+KJy-w$f5!;_ delta 29 kcmZoMXffEZl$DWjvI3jf<~^);7=g?=Y!aI{vIh$Q0E@c`0ssI2 diff --git a/meshifyDrivers/mainHPRPI/mainMeshify.py b/meshifyDrivers/mainHPRPI/mainMeshify.py index 6844278..3ff7bc1 100644 --- a/meshifyDrivers/mainHPRPI/mainMeshify.py +++ b/meshifyDrivers/mainHPRPI/mainMeshify.py @@ -27,7 +27,7 @@ class start(threading.Thread): print self.deviceName mac2 = mac.replace(":", "") self.mac = mac2.upper() - self.version = "17" #hp device management + self.version = "16" #mistification #added Nodes in v5 self.finished = threading.Event() threading.Thread.start(self) @@ -56,7 +56,7 @@ class start(threading.Thread): try: - topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID, channel) + topic = 'meshify/db/%s/%s/%s/%s/%s' % (company, "_", nodeTypeName, uniqueID.lower(), channel) print topic if channel == "files": #for the file structure I had to take off the " " around the value @@ -71,6 +71,27 @@ class start(threading.Thread): def run(self): #on startup send the version number self.sendtodb("version", str(self.version), 0) + + if os.path.isfile('/root/python_firmware/drivers/device_base.py'): + print "found new device_base file" + os.system("/bin/mv -f /root/python_firmware/drivers/device_base.py /root/python_firmware/device_base.py") + os.system("/bin/rm -f /root/python_firmware/drivers/device_base.py") + os.system("/bin/rm -f /root/python_firmware/drivers/device_base.pyc") + + if os.path.isfile('/root/python_firmware/drivers/meshifyData.py'): + print "found new meshifyData file" + os.system("/bin/mv -f /root/python_firmware/drivers/meshifyData.py /root/python_firmware/meshifyData/meshifyData.py") + os.system("/bin/rm -f /root/python_firmware/drivers/meshifyData.py") + os.system("/bin/rm -f /root/python_firmware/drivers/meshifyData.pyc") + + if os.path.isfile('/root/python_firmware/drivers/main.py'): + print "found new main.py file" + os.system("/bin/mv -f /root/python_firmware/drivers/main.py /root/python_firmware/main.py") + os.system("/bin/rm -f /root/python_firmware/drivers/main.py") + os.system("/bin/rm -f /root/python_firmware/drivers/main.pyc") + time.sleep(0.5) + os.system('/root/reboot') + while True: try: self.mainMeshify_hb('hb', 'On') diff --git a/migration/__pycache__/lattice.cpython-311.pyc b/migration/__pycache__/lattice.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..add6f3948fb265bc871f1cf07988b7ff5ca18b09 GIT binary patch literal 20652 zcmeHvd2kz7nqN22xB&trcpqYmmqbYfbzjy+k<@8XvL)MN+8zvu21!sPKzD;WU~)ol zlp4;IoZ;+x%G`~TV$FISdhKMjm8z7gvQy=;J;|i9NzklzgzAJTXE#o*|4E?OP8{W^ z{Jz(?Nowq5{@ZPIfBgEr@4ff+>-WCvz2@Kdc$^%Lb05khefv4?FX$$BII_qW&s#X| zEl%QwIf<7{ll(Bx{+fnO_-mdtPYJ^UZ{%5qEiBJEY{g$;(l%uuwrBGk!w#0`9Ck{U zkV~?L+>$NikxZcq6L*%A>~}cH5%Iob=D7E88ON|sa^B&FD#sRFP@ z@&eXMKEOJu60lyX0&I}{fQ?c$V3Sk>*eulown%k=TTEQUInkPH;~gOH_+d3r6UXS z&~%ku-FuUD&&)*R+v25&G9DehEsn-zQ3@qOfzPS*P&cnz$0CVH>;^F5N+>a&Y#dJ{ zrj?_;y?c)A4@~dfvv*G*7D@DmrlY;ddO0#3mlHEeL_RehiOIL8XQrkD;rNtZaq;@$ z>zAI}8x9}5@#^!p!&H0jV2tAC3z{oGfT)jf5b+#|_hIt_HwvMD$3849aJTqj%T=!1 zth8<*oZW8zITDTc%t^2Dej#^MP)4hY03B~M0`gcu(cmBh^G zXds-IeP|Q}?WS4tn0Z$C8UHNzw>i>Jcq~)0>wGMlh=wMk$%vR3kBG%jDGuHWN2U|e zcq}l~ZPjg45k(1&MRb84SNG7<1>>VZ_H?>MnVF8rGI6HcgTW}O4F*w4P$H9~GAUM> z^sek80wE-iDY_#V3{Qp>B^Xq!NEPXre|)_6IlN1yHx>=Ydk5m-nW;!Dq4b`OCeF=_ z^j;f}#>SM>@sKR_PDRJ$5V6-g8A>Fg;YeWmw(LPo$(rKF2;{h_L6&k3;Olwr)4DAS z!d-W|x*mTV>6RS}f@W__+jqQSd1G>EK(%kx=-jb#=>EWio|OIauR8$B&M#0GhiAm| zQkMjSx-%G@ic2$-gu8>mS7$<#*&JIiD8<9UAY)MXWFI{onVdv7@v@J2qZfWkt^{QL z$UqfkoCN^WxnEj1o)V5{1dHVgpW(h8l&L~ib2vY~kU}G68?G;|0Q|so5hDa+k{jS& zY5kEYQAD|5!sOuZ*k}2HGMk}PB0SH%#q-=-83IoWt7I+=)IqDRB@#zL6?azvS# zOeld8K4^~YBR+Nk{FKk}@k+%UKg<8nG|M+2-8e=~**3fqhc@`3cWIuQ0=sqoy3R|A z8DtnxtUn(+j*UfPkz3R9iR2dIAvntT2^^b@heMOfi9kMIL@{M2z}J7t{Uy3QRkai6 za^+&*;;T!QZzdLx7%9b&bt*|Nf$AnMq6uBN9=WZ^EmWPsit%c7S5f;i$sb1YbfLvh zJ>Cq*TXr$9z^_$Q6}Po4;ZCjEsR35()GBtS?1eL9&Xr>^2pB>9Z zqAi!kmGD~3X{opi*%LIlvxg#FNaJxO|u#ODkNNO9!0s6-%0GcXjaDp}tFltXMAbO5fG1FI;|h zAX$}Dqa-z{_M>7U?Mk*0q@DfEd~U=>o{Q6cR|kX7J^OT_^3x$DA|mCeIMnUbeL2o@ z>=sgTyy})|IT}momS}8xCZXG)<*65RyBv9SCZZ%1=!KcYxb8W3?b;R89u4==tvgq9 z-*eZ_6>7~fBsb!P8&ZuVr|t^V>k8rp%uLGxq_pEwmI3CuudJNAIm4MPef%fw`wV2w z?#XbR_p<2`$2qQ;zGM(7z}K3@)Yb!P(?PB2V9H*<*3@?IWNPP0wds`BbP5Td`J3K6 z{?H#-^#?xbJ^4xBGphet&HwED)1SGTGMveB2z93_UQq4d(d^$z*}qd(A8Npvv4D_K z8&kGuA~LDWrtA>dCmR9sxN6Qc3p1r42WPpg-6hPL!X_TF@UFjDVoR_kX8BSd^LKd2 zJ>m7$2F4gcBRsEHsj%3D`9>YV)H$z%+T%J4C5@lV>Zm7){=8bCIIwZ)Ihl z$i%Sj9z~PEk=smP%6mY*7njlx0G;eO0x zkaCWi&z!ZXy6r0+_dTlfgyuYv5>6N$Sfs4Dyv+L)EkNcK<&_(#vygO!PL)jAPIV&P zED3+hRLGm-ie=C&mqguje6h5dpqIvuS-#jnd*4zjvFKG6j5NubZL=XkR!sJo$V)`f z=8e5lc@^WwNCn!G;-%1a$QKdw2^uyLFOffeSI!4y8eWh2-p{Egps{3fSU&$XK(~X< z7P$ww+zp_cF+pY0!@5gJgkWL^!*L1b--Hs6>6XcONWzRwL~fD7vqxgmbezrZSA#S1 zq;7dNh-qE=_RVJpugND+;Za=5b%1&9Gr{}%r8}4AFQskXMgGofi{VvU18F`n?QdG@ zTROdT`psht_BD6&(k<1!ZGLdgU48fIrJA3=oO16~-Fr3n-uc0F=MJs&;Dh$n&f_W9 z;2Znz9$k#QIib3?YObv**VYgAri8)uvx}4tn_cyWLVUvD*x-emHhJNu5(Su7q?m!I zzYrx!xY2s{c)x6R&Y5S;h1ZUu$cB?#oge|1J&GwxYv#G)y!=3JDB&xwq(Pfk#HICQ z2bDS5awQHwhFr9bMpU;6RuA&nNRkhLy>3_eND`(|gM1iyxpJ)g4516E`-0}afFWdaYqt8O{#9EmjlQiA-3t+|qE)qvnq5rU#cw{wz(W3@ zH^;`92&7a1l^a%=z@{PH5yKoy+@6jo@A7P}lLX$D%k&!>XH9Hn!N0HBE zSg?)8CneZg8#3Vp6KAsd)DPEiDFrre|M0|JXoiOMnt>)JdolEAJ6;iP`w8=??KgdNOK)Z35R|QJ#8Tib>n`f zm(Bcg`8lvMj7z>qfDG;u2aW^_>(@U3f)F|)DWpFo%ROO^m#mUavfne!nIwnglw6Wq z@<N7cfd^6r?L_lBWhD zq|d~V7&baRYX@yp3Z{+%qX=4bd}b;X>yaZNiA-~fSm7lGbgLA(fv|&aCG{8$KXy|8 zp#)jCR^_&mh)hAAVUdCRoQB)yq;NhbbxtO4NQ%cMZ|e@iZpzU_1Y?AhGo#lnqtLHZ zO9CeIm~I0bk+F!Z+eYH?$w(-s+cD50i6{awc7n<@(gZlQ;0Do<$=F!8-DsKW{G8s% z=K|IK5`mWqyh3SYP>{*YMjeS1xUO+ld5G@VobMKNTb4riWRV~fQMxl455`%DNYQ;X z^eH))QYN(z8teR}jE{5I zrd_oei@6GE0P`0zZq5x-oG62{hMkpEveHoxDu9$wovG!v>_j-{xVE!TYdJlCcHxpL zG^VS!EZtmA+?&&?dyq8n`lWOu1$~ZcJ5OkhCy{hP6~IN+{LmA|x)q)Lgt5+NN4qf- zFy?8jJ4&okjLnL|NG+8tlxK`v$y=b=SVp~r4tx)naY#PHqC^c|Rs+qi%5PUkftt#) zT4)Xy&C+keNl7inOs08SG?Tw|EyireTyEzQG_&KqPodio1a zYKk}%O0D$Lx50BSa#HP_@O?q5MT~%9H~E{~E%S@qO}@J>xo zj)Wu;QLm%oYjeyAq_DA>IR%sPn~+}ii;>%dvK*InOL#IGzODRp@CZM*P+3Du24R4zNs&)HJ4R8p^~7 zNO7VJ&gvREeU((QM2u8BEFpfat#f&7C8l*9SKCf#Z6_9NixsN9HQn64d`@fbdDy&f zwRxY~d_ZeH@St|J`6Tk~s=Xn-HL!B?{+!l3_^|ilYVSq0_mYNl>k#ris(nkkzHRx? z%Ko>HKWsm^+I~=NKdj-bKZ0!Gho1BnDxK7NPe1HEx7vG7?Y*Gk+;S1wPSxJ@ske5a z;_EfYJs7YdM1~6x*6dwp%0KYXe{t1+@z-Zl*M|S>a_W`us{Wwn532U>Y4-1>Hafq? zK|#e34@9X)=l=fb_S46?505(sIP*ugK>t(bkDjss50j3`9pv25{TC~E6K`=*W(WKp zmd+&s&?1=tt+TvjE6h5i*d+%b>;gb{v9AtFPl2*@K7zJ_)#?S6uRw+LN~sFaKTEAu zGnyKy7I>Xh57+?u#uECbvNoFwx$D|&DWl!;1lp=H+SbizX$gE|IaqFcO0`m()V>KV zFLe}Jz*BY>?n)(_)Fo}ZXa6}*4kv7)?a7_l_(4`j=ORP2L>CgV;zcncigWf~eLkR@ zLgPB8TSsog`MW)Y+ezJY1D1(0Gjc7OfF)uI!-9#;OpRdbo9KSeK?D`#X}xxA1|d|K zG!f}pwtRObB1hvAh(fwKJRW1hLpRODbbgO+l47WLN;gl+37y}Kdl2p0jWB;m_l`#< zrw4CMPljS_B}nJPx|Nv6GL>~Is)U0xx@87yQHt)FipGq_u94Lw#Nx5YV_|R4Yje8k zax7_iZH`bgr~pE@Wb!1t?K(e-TWWYp7pC@v$92=x4V_nXUe@^pd|xvxG(&(1Iv>>q zY8<8zGYpxL#6n%Voq{Go1lr1sqS`40Y-r7BWJZ|^P3NSV{3mFA8J99@h%%wkXHDDP z3nLjbuutv2#g2>x7)&LH2dOr~?SxB_NA>R2yn8cV%JCUgm4sJ;Y9XhX6ops-YC>Scq0{3 zGEIaxQ+g!RLiiTW;a<3ws%qCf9hp{25s~u7)y3+?@i(`q9#Qj%nXQ!G#yKi~+P)|) z9eQ(0t?JOKIx_8)-oaH=u6e5#_b+)h|8~{et$DlC9)H?b``fcYpNVrGe#|dly%#S0e8>sh$Ix=Rn$1mDxwNaXB!5E^`p(=KN*Dz9VxvcaY{` zQVhFcAjL3~N+N`XStXeZIK}i3ru}c2Q&@5>q)S2phrMAb+-0MF|E`SQTA%>EjVvz* z`9H;V+6y$xIj&3oN{&*=1O~v+F*)tZ|yz4mht>dt?e~N6~b!C|qTJm(IWs7Xnb!D@DWH#LX-=!Kz z&)VeiK1pKYvEbSmMjwexV~&fM}i@CJdO6Zm@oXW+|O6s%Vzn0(V6`6Yh& zd&us#Tg*Jzj<=`Db3$|-=g^s&Oe>@AP`((K|BtC)&uI6_`4^U4j`zz{L=ke zt@H4z>u`qiI9k)~z4!UIPu_>QbZp_`l1+7s8I#qsBQ5S)dFE~JeX}O+e<&VZ6^}mH zr-~;v@#Last#kWI%l%ii?jvgFQLXdnqHAeT^|fu1obt7Oz2?U{gvqm`0Meb^*f zg#(Ho-T6y%U*+i<(}y+Hr*~OC+{pumk(@u!*@jG`UgUQfWyr@8u0kFb4y~mY z-~n#VzR7Bny|lGC8|-%cO3U+H%+=2AL4^LM$?$;5Kj!h9<&_%Awn;CPMN+YPR?H*30i+e=ozAji=Mxmrl+V;#U}tm*6_1}W?z(;U{Vkz3eIht<^h zbR?!*rx6-~yCl#OEBqm}?Hs_qq8BJiEbA^7EgnbM82jx=i|Tfk0{4ssQsp_S_xk|d z9+@Pu?lNRB?Rkpm6?sW&+*f5e#9jW${wbC6CdQ*mu+UMuJ@+i~1IqUhv%#qm#i+*y zK8^UZ!415>aTGCZ07>zn2#ez6fT2pfSXaoI1VPjwuYvC0;!@5+enPFd{f#izb08(u zezl$R*Dhu~gZYbVwQZ^PefM8dYx}g?zWK}W5K{hK5B>XA{rm1;{$N1ce_r)p(EJxv z??ugfasFJovS$9mXExWukZNnvY)ubsZL796)z+ceI%wy?{xwhKVqm#L^=#8T+Ym%? z>|Z&)*3f#dY55hkVV~BpZ{Y%MK-kZ+7Z4YY$ ztF?ht?@6`xlvaCc!SxxOsOwAnR7a}@RnPe>Pc0u>tE_%=%d#!C|4gd#tXg?it30~^ zudw2wy>ZpvsM?z~dvnU(oUX!-3b*4a1{O}I{Vf^J>1j)EXCW!;h(tEz55nTDgRE@w=3PX18FB!Uwg{e{#n!E2UQP-)uwY=)44@E7U{uxWm~#o z>*DFOyXwc6e{}hi<^%Uf)aD~v^AXj3RC6Cqu`|;^oH)Qgdk6*|(c^qcppdo^#(Hi& z)mI&y#d+sK%G>$SyKB|EYvm=?dqnddQH7(La5R-Wl}-$(U!JTvEt)?Rh0`6j54Z7v zA61>U4R)D-)g=t>u>Gos2gHC?Zm{tjE>_dTtN!^vdg}UE_ zimjqw>GQymt)c~ojSL>j>GmfBY`4S>wt#DRe})=_{LgLG>(8z?vJE*TYney)O*NtX z5taCuz^@4WGXkWyk}X$f!eRK1qcf9}w?#OT5hS)#+aMAphi~O?P5RrDZvRkX{ zUa+qDw=CUQzL9S2Nw;oGx3;HSyVI?^)2+K0g2XQ1k7}INfiF3L3|Hy6h!8r#h5oOa zIiHvkk31Alt%|2UXiYuyoYwceD!!nJFMRT%oKkM6FW%H%ys3I`Y2I5{m-JlZ*L*c^ zI#W$YA2#)^HuZhb{=w+SJJqHkt!YU0UDkY;Q|$cApgQod>DX%1u?JT_sQuHI)TZ-V z(|Of*LGxWmu`|=oxUtZa8P8|HN9$v6|HmilW2`q{F*#t;>Co78M8uXz)?GGB+iUSoTHY45O6q*WC}O%PLp$m&>_RfT3vXif>uX%DjrYpI@Ed<8%&c#C^}=(+1z zz%N_8Qx&>2p(`bHWm(v&3T>LumJ-_19>&5Z&8>6Kbdtyb!U$Wv|t?NQ)bRGi&e!}PQh9! zvJ~(olod;1sG;z^ECn?RLFwXc6veiL2quz z_9zwzv;bLJ4$p5Zw%Q6GJ|O6All<>jmp&WpjsN%7vDmq;b$cT3{zJrC^Sy7HwM%vH z*MGCtCEunt5OTU3lHL7A0H7#HXP81+i6Q1>OCaeQymk)XHw<3uJ%1(X-g9JcV9&wB zfjxmeyIK5OH(wqdmH#IAWa05{s}XtTN92D*X>SAQ{IvWo{wA%L_VnyOd_WhD9K`CN zFoXp@2o4ri=l2>Ra*CWAC6|E>$sYjfrkf)yl>RQS+o$2Q$HVc-$5}U-;)k?!C;xL$ zvON}{vSJfEw)bI&Ro+&TKc)OXA;9dTe+dlxVzJkiZF|M$YVyRHfuuX~-9noqB>6v3 z&Yu(b3o7MCHbvYG%gMnO4#}}*f!3V&oZB*JL|U>7h+tO!AE@FQiM4i2Er_)WuBtU% z+4`xkW>L;F|4XfSNhYH)!5ij;e=QNsO72})0K7UhNg67Eg{JD*eI7koxhTt z@Jj3a`OJ3C<(}_fD>b~PzP^ngEtq> zFNW_9Ej^{Vx2;$-H?}nX@!TKHJ-Dj&_G!I+svG{K7ZUH>LjOWP;_0Z#f_5@o4m=|4 z%lvI2qYPvAPOeZP4@RZ4Vqt3On(AoN9Bm|Lk1S8F*(>h47SAuu-VdehhgJJw&3+gN z$;%^*gaLT`mLu@^i_;3Z1?`0$mP}_T{?V_|oZW)$kU~^LCAO_4zU;Sy1`9qIzy3+PxHBWGR zeZSlMkz45Z**@~~1h>@>xXmBCg#n-KV?Pgw(0nxM?&*P*p9}DJQ6%0N-fX4Ih-5J{ zf&yUKokM}7J-03(KSJ}#?y_Pu=8Zx|{9-qs7&-LrP*9fZcI%*wuy61 zpOM-hclKxZo`uA+p%oJIgC>A47FYz;7(%o?EspQ$=u0IQXC0D%jdxy<=<cjTo$v zkLiH#+!6P+frg2A+Ht@{yx|yE7-fQqc#tU}eBH$*aVf6@Ft^z4!`3gh-~XYjZPnGL zx;iu$i(`7Td%pHQ^dDLEA9=7t^$%+PLDhanv!6-X&!p`Y3s&-+Czd*Lu|6CJ z2zCyu4ZF35-3t|;As*Vb)UJBAXr3)3%t5nl&F#B;elfN@1~KVAthx_t?!!RV+?97P zr0RMe*6m-d+kb!fgIcxjoK|;Eb)VPV=Tq+UB*!nnhb_QI4F9qyr8svz&Lmo)cnN`x z9)P}x&cR0v|FS5hICnFgHBqQ!GqNuZ;cfl0sk6Vs{2|xJ^}8(}acb=>HpLa0Ty}Xo0p{4)rtxoCCZi*Ua_pvbrW|_+>mWcYF2+ZG|Clgx;o1K>Fun|A zab^~A?x&2i1TGL5BJeB$`pn2!&YUNVc~)$((=sN<@n!cfkc!VzjBP8r9sf{-{i6`w zYQ%TgIwA`rvp^4Xi)2!tZ1m|a1iWe>_ge6)1$0Q=$UL z!4&8DG}n^iTGE_rp8e9CbDsUuTurLQkj>n)A-HUz&5zvmZRP z>fI^Mm-hEC*iO*vpLb`hcK!1rt-Mkz zBRN=9X~xvdBa#P@YnUaRC?Z+^;AQQKJRC%TO${P3q@nn|~q=xdnH-X-Z=9srKqQ(;L^G&689ZU5`Q~ZR=PiXu^QClmX z)UiTF##GwWb+j-+E#wh;K}J)}954L9`#au6ugZxUC#EGlO{}&MT-~s>u literal 0 HcmV?d00001 diff --git a/migration/convertDynamoDB.ipynb b/migration/convertDynamoDB.ipynb new file mode 100644 index 0000000..4b01320 --- /dev/null +++ b/migration/convertDynamoDB.ipynb @@ -0,0 +1,202 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3, json" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "def convertToNewConfig(config):\n", + " #old form\n", + " #\"{'mainHP': 'https://hp-drivers.s3-us-west-2.amazonaws.com/mainMeshify/','m1': 'https://hp-drivers.s3-us-west-2.amazonaws.com/M1/','flowmonitor': 'https://s3.amazonaws.com/pocloud-drivers/flow-monitor/'}\"\n", + "\n", + " #new form\n", + " #\"{'mainHP': 'https://hp-thingsboard.s3.amazonaws.com/mainHP/','m1': 'https://hp-thingsboard.s3.amazonaws.com/M1/','flowmonitor': 'https://hp-thingsboard.s3.amazonaws.com/flowmonitor/'}\"\n", + " mainHPMapping = {\n", + " \"mainMeshify\": \"https://hp-thingsboard.s3.amazonaws.com/mainHP/\",\n", + " \"piflow\": \"https://hp-thingsboard.s3.amazonaws.com/mainHPRPI/\",\n", + " \"plcfresh\": \"https://hp-thingsboard.s3.amazonaws.com/mainHPPLCFRESH/\"\n", + " }\n", + "\n", + " configMapping = {\n", + " \"abbflow\": \"https://hp-thingsboard.s3.amazonaws.com/abbflow/\",\n", + " \"advvfdipp\": \"https://hp-thingsboard.s3.amazonaws.com/advvfdipp/\",\n", + " \"dhsensor\": \"https://hp-thingsboard.s3.amazonaws.com/dhsensor/\",\n", + " \"dual_flowmeter\": \"https://hp-thingsboard.s3.amazonaws.com/dual_flowmeter/\",\n", + " \"flowmeterskid\": \"https://hp-thingsboard.s3.amazonaws.com/flowmeterskid/\",\n", + " \"flowmonitor\": \"https://hp-thingsboard.s3.amazonaws.com/flowmonitor/\",\n", + " \"PiFlow\": \"https://hp-thingsboard.s3.amazonaws.com/PiFlow/\",\n", + " \"ipp\": \"https://hp-thingsboard.s3.amazonaws.com/ipp/\",\n", + " \"plcpond\": \"https://hp-thingsboard.s3.amazonaws.com/plcpond/\",\n", + " \"multisensor\": \"https://hp-thingsboard.s3.amazonaws.com/multisensor/\",\n", + " \"dualactuator\": \"https://hp-thingsboard.s3.amazonaws.com/dualactuator/\",\n", + " \"dualactuatorpri\": \"https://hp-thingsboard.s3.amazonaws.com/dualactuatorpri/\",\n", + " \"plcfreshwater\": \"https://hp-thingsboard.s3.amazonaws.com/plcfreshwater/\",\n", + " \"pondlevel\": \"https://hp-thingsboard.s3.amazonaws.com/pondlevel/\",\n", + " \"promagmbs\": \"https://hp-thingsboard.s3.amazonaws.com/promagmbs/\",\n", + " \"poc\": \"https://hp-thingsboard.s3.amazonaws.com/poc/\",\n", + " \"recycle_train\": \"https://hp-thingsboard.s3.amazonaws.com/recycle_train/\",\n", + " \"rigpump\": \"https://hp-thingsboard.s3.amazonaws.com/rigpump/\",\n", + " \"submonitor\": \"https://hp-thingsboard.s3.amazonaws.com/submonitor/\",\n", + " \"swdcontroller\": \"https://hp-thingsboard.s3.amazonaws.com/swdcontroller/\",\n", + " \"tankalarms\": \"https://hp-thingsboard.s3.amazonaws.com/tankalarms/\",\n", + " \"tanktransfer\": \"https://hp-thingsboard.s3.amazonaws.com/tanktransfer/\",\n", + " \"tenflowmeterskid\": \"https://hp-thingsboard.s3.amazonaws.com/tenflowmeterskid/\",\n", + " \"transferlite\": \"https://hp-thingsboard.s3.amazonaws.com/transferlite/\",\n", + " \"m1\": \"https://hp-thingsboard.s3.amazonaws.com/m1/\"\n", + " }\n", + " config = json.loads(config.replace(\"'\", '\"'))\n", + " for x in dict.keys(config):\n", + " if x == \"mainHP\":\n", + " config[x] = mainHPMapping.get(config[x].split(\"/\")[-2], \"bad_request\")\n", + " else:\n", + " config[x] = configMapping.get(x, \"bad_request\")\n", + " config = json.dumps(config).replace('\"', \"'\")\n", + " print(config)\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "client = boto3.client('dynamodb')\n", + "table = \"HPDeviceList\"\n", + "macs = ['C4:93:00:0C:68:F9']\n", + "for mac in macs:\n", + " resp = client.get_item(Key={'mac': {'S': mac}}, TableName=table)\n", + " oldconfig = resp[\"Item\"]['config']\n", + " print(oldconfig)\n", + " urls = convertToNewConfig(oldconfig)\n", + " newconfig = oldconfig.copy()\n", + " newconfig['S'] = urls\n", + " print(newconfig)\n", + " client.update_item(\n", + " TableName=table,\n", + " Key={'mac': {'S': mac}},\n", + " ExpressionAttributeNames={\"#C\": 'config'},\n", + " ExpressionAttributeValues={':c': newconfig},\n", + " ReturnValues='ALL_NEW',\n", + " UpdateExpression='SET #C = :c'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.list_tables()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Table': {'AttributeDefinitions': [{'AttributeName': 'mac',\n", + " 'AttributeType': 'S'}],\n", + " 'TableName': 'HPDeviceList',\n", + " 'KeySchema': [{'AttributeName': 'mac', 'KeyType': 'HASH'}],\n", + " 'TableStatus': 'ACTIVE',\n", + " 'CreationDateTime': datetime.datetime(2020, 6, 4, 13, 0, 51, 690000, tzinfo=tzlocal()),\n", + " 'ProvisionedThroughput': {'NumberOfDecreasesToday': 0,\n", + " 'ReadCapacityUnits': 5,\n", + " 'WriteCapacityUnits': 5},\n", + " 'TableSizeBytes': 100095,\n", + " 'ItemCount': 495,\n", + " 'TableArn': 'arn:aws:dynamodb:us-east-1:860246592755:table/HPDeviceList',\n", + " 'TableId': 'fdb15ece-4feb-4dca-ae90-909c9d31ca7d',\n", + " 'DeletionProtectionEnabled': False},\n", + " 'ResponseMetadata': {'RequestId': 'KJIR35CPC8J544NID1MP1R45NJVV4KQNSO5AEMVJF66Q9ASUAAJG',\n", + " 'HTTPStatusCode': 200,\n", + " 'HTTPHeaders': {'server': 'Server',\n", + " 'date': 'Thu, 09 Mar 2023 19:49:01 GMT',\n", + " 'content-type': 'application/x-amz-json-1.0',\n", + " 'content-length': '513',\n", + " 'connection': 'keep-alive',\n", + " 'x-amzn-requestid': 'KJIR35CPC8J544NID1MP1R45NJVV4KQNSO5AEMVJF66Q9ASUAAJG',\n", + " 'x-amz-crc32': '2354334289'},\n", + " 'RetryAttempts': 0}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.describe_table(TableName=table)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "m1 = client.get_item(Key={'mac': {'S': \"C4:93:00:0C:68:F9\"}}, TableName=table)[\"Item\"]['config']['S']\n", + "piflow = client.get_item(Key={'mac': {'S': \"B2:78:EB:E7:83:45\"}}, TableName=table)[\"Item\"]['config']['S']\n", + "plcfreshwater = client.get_item(Key={'mac': {'S': \"00:00:05:00:00:08\"}}, TableName=table)[\"Item\"]['config']['S']" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'mainHP': 'https://hp-thingsboard.s3.amazonaws.com/mainHP/', 'm1': 'https://hp-thingsboard.s3.amazonaws.com/m1/', 'flowmonitor': 'https://hp-thingsboard.s3.amazonaws.com/flowmonitor/'}\n", + "{'mainHP': 'https://hp-thingsboard.s3.amazonaws.com/mainHPRPI/', 'PiFlow': 'https://hp-thingsboard.s3.amazonaws.com/PiFlow/'}\n", + "{'mainHP': 'https://hp-thingsboard.s3.amazonaws.com/mainHPPLCFRESH/', 'plcfreshwater': 'https://hp-thingsboard.s3.amazonaws.com/plcfreshwater/'}\n" + ] + } + ], + "source": [ + "convertToNewConfig(m1)\n", + "convertToNewConfig(piflow)\n", + "convertToNewConfig(plcfreshwater)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aws", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/migration/convertVanity.ipynb b/migration/convertVanity.ipynb new file mode 100644 index 0000000..235c5bd --- /dev/null +++ b/migration/convertVanity.ipynb @@ -0,0 +1,67 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Device Name form: Jones Holton FW #2\n", + "#MQTT clientID form: jones-holton-fw-2" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Banay WW 18 #4 banay-ww-18-4\n" + ] + } + ], + "source": [ + "deviceName = \"Banay WW 18 #4\"\n", + "def convertVanityToClientId(deviceName):\n", + " mqttClientId = []\n", + " for c in deviceName:\n", + " if c == \" \":\n", + " mqttClientId.append(\"-\")\n", + " elif c.isalnum():\n", + " mqttClientId.append(c.lower())\n", + " elif c == '\"':\n", + " mqttClientId.append(\"in\")\n", + " elif c == '-':\n", + " mqttClientId.append(c)\n", + " mqttClientId = \"\".join(mqttClientId)\n", + " return mqttClientId\n", + "print(deviceName, convertVanityToClientId(deviceName))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "thingsboard", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/migration/lattice.py b/migration/lattice.py new file mode 100644 index 0000000..bc901bd --- /dev/null +++ b/migration/lattice.py @@ -0,0 +1,419 @@ +"""Query Meshify for data.""" +import json +import csv +from os import getenv +import getpass +import pickle +from pathlib import Path +import requests +import click + + +MESHIFY_BASE_URL = "https://194.p2121.net/api/" #getenv("MESHIFY_BASE_URL") +MESHIFY_USERNAME = "reportuser@henrypump.com" #getenv("MESHIFY_USERNAME") +MESHIFY_PASSWORD = "Kk8kMU2cc6vqVy" #getenv("MESHIFY_PASSWORD") +MESHIFY_AUTH = None + + +class NameNotFound(Exception): + """Thrown when a name is not found in a list of stuff.""" + + def __init__(self, message, name, list_of_stuff, *args): + """Initialize the NameNotFound Exception.""" + self.message = message + self.name = name + self.list_of_stuff = list_of_stuff + super(NameNotFound, self).__init__(message, name, list_of_stuff, *args) + + +def dict_filter(it, *keys): + """Filter dictionary results.""" + for d in it: + yield dict((k, d[k]) for k in keys) + + +def check_setup(): + """Check the global parameters.""" + global MESHIFY_USERNAME, MESHIFY_PASSWORD, MESHIFY_AUTH, MESHIFY_BASE_URL + if not MESHIFY_USERNAME or not MESHIFY_PASSWORD: + print("Simplify the usage by setting the meshify username and password as environment variables MESHIFY_USERNAME and MESHIFY_PASSWORD") + MESHIFY_USERNAME = input("Meshify Username: ") + MESHIFY_PASSWORD = getpass.getpass("Meshify Password: ") + + MESHIFY_AUTH = requests.auth.HTTPBasicAuth(MESHIFY_USERNAME, MESHIFY_PASSWORD) + + if not MESHIFY_BASE_URL: + print("Simplify the usage by setting the environment variable MESHIFY_BASE_URL") + MESHIFY_BASE_URL = input("Meshify Base URL: ") + + +def find_by_name(name, list_of_stuff): + """Find an object in a list of stuff by its name parameter.""" + for x in list_of_stuff: + if x['name'] == name: + return x + raise NameNotFound("Name not found!", name, list_of_stuff) + + +def GET(endpoint): + """Make a query to the meshify API.""" + check_setup() + if endpoint[0] == "/": + endpoint = endpoint[1:] + q_url = MESHIFY_BASE_URL + endpoint + q_req = requests.get(q_url, auth=MESHIFY_AUTH) + return json.loads(q_req.text) if q_req.status_code == 200 else [] + + +def post_meshify_api(endpoint, data): + """Post data to the meshify API.""" + check_setup() + q_url = MESHIFY_BASE_URL + endpoint + q_req = requests.post(q_url, data=json.dumps(data), auth=MESHIFY_AUTH) + if q_req.status_code != 200: + print(q_req.status_code) + return json.loads(q_req.text) if q_req.status_code == 200 else [] + + +def getNodeTypes(): + return GET("nodetypes") + +def getNodes(): + return GET("nodes") + +def getFolders(): + return GET("folders") + +def getChannelValues(nodeId): + return GET("data/current?nodeId={}".format(nodeId)) + +def getUsers(): + return GET("users") + +def decode_channel_parameters(channel): + """Decode a channel object's parameters into human-readable format.""" + channel_types = { + 1: 'device', + 5: 'static', + 6: 'user input', + 7: 'system' + } + + io_options = { + 0: 'readonly', + 1: 'readwrite' + } + + datatype_options = { + 1: "float", + 2: 'string', + 3: 'integer', + 4: 'boolean', + 5: 'datetime', + 6: 'timespan', + 7: 'file', + 8: 'latlng' + } + + channel['channelType'] = channel_types[channel['channelType']] + channel['io'] = io_options[channel['io']] + channel['dataType'] = datatype_options[channel['dataType']] + return channel + + +def encode_channel_parameters(channel): + """Encode a channel object from human-readable format.""" + channel_types = { + 'device': 1, + 'static': 5, + 'user input': 6, + 'system': 7 + } + + io_options = { + 'readonly': False, + 'readwrite': True + } + + datatype_options = { + "float": 1, + 'string': 2, + 'integer': 3, + 'boolean': 4, + 'datetime': 5, + 'timespan': 6, + 'file': 7, + 'latlng': 8 + } + try: + channel['deviceTypeId'] = int(channel['deviceTypeId']) + channel['fromMe'] = channel['fromMe'].lower() == 'true' + channel['channelType'] = channel_types[channel['channelType'].lower()] + channel['io'] = io_options[channel['io'].lower()] + channel['dataType'] = datatype_options[channel['dataType'].lower()] + # channel['id'] = 1 + return channel + except KeyError as e: + click.echo("Unable to convert channel {} due to bad key: {}".format(channel['name'], e)) + + +def make_modbusmap_channel(i, chan, device_type_name): + """Make a channel object for a row in the CSV.""" + json_obj = { + "ah": "", + "bytary": None, + "al": "", + "vn": chan['subTitle'], # Name + "ct": "number", # ChangeType + "le": "16", # Length(16 or 32) + "grp": str(chan['guaranteedReportPeriod']), # GuaranteedReportPeriod + "la": None, + "chn": chan['name'], # ChannelName + "un": "1", # DeviceNumber + "dn": device_type_name, # deviceName + "vm": None, + "lrt": "0", + "da": "300", # DeviceAddress + "a": chan['helpExplanation'], # TagName + "c": str(chan['change']), # Change + "misc_u": str(chan['units']), # Units + "f": "1", # FunctionCode + "mrt": str(chan['minReportTime']), # MinimumReportTime + "m": "none", # multiplier + "m1ch": "2-{}".format(i), + "mv": "0", # MultiplierValue + "s": "On", + "r": "{}-{}".format(chan['min'], chan['max']), # range + "t": "int" # type + } + return json_obj + + +def combine_modbusmap_and_channel(channel_obj, modbus_map): + """Add the parameters from the modbus map to the channel object.""" + channel_part = modbus_map["1"]["addresses"]["300"] + for c in channel_part: + if channel_part[c]["chn"] == channel_obj['name']: + channel_obj['units'] = channel_part[c]["misc_u"] + try: + min_max_range = channel_part[c]["r"].split("-") + channel_obj['min'] = int(min_max_range[0]) + channel_obj['max'] = int(min_max_range[1]) + except Exception: + channel_obj['min'] = None + channel_obj['max'] = None + + channel_obj['change'] = float(channel_part[c]["c"]) + channel_obj['guaranteedReportPeriod'] = int(channel_part[c]["grp"]) + channel_obj['minReportTime'] = int(channel_part[c]["mrt"]) + return channel_obj + return False + + +@click.group() +def cli(): + """Command Line Interface.""" + pass + + +@click.command() +@click.argument("device_type_name") +@click.option("-o", '--output-file', default=None, help="Where to put the CSV of channels.") +@click.option("-m", '--modbusmap-file', default="modbusMap.p", help="The location of the modbusMap.p file") +def get_channel_csv(device_type_name, output_file, modbusmap_file): + """Query the meshify API and create a CSV of the current channels.""" + channel_fieldnames = [ + 'id', + 'name', + 'deviceTypeId', + 'fromMe', + 'io', + 'subTitle', + 'helpExplanation', + 'channelType', + 'dataType', + 'defaultValue', + 'regex', + 'regexErrMsg', + 'units', + 'min', + 'max', + 'change', + 'guaranteedReportPeriod', + 'minReportTime' + ] + devicetypes = GET('devicetypes') + this_devicetype = find_by_name(device_type_name, devicetypes) + channels = GET('devicetypes/{}/channels'.format(this_devicetype['id'])) + modbus_map = None + + if Path(modbusmap_file).exists(): + with open(modbusmap_file, 'rb') as open_mbs_file: + modbus_map = pickle.load(open_mbs_file) + + if not output_file: + output_file = 'channels_{}.csv'.format(device_type_name) + + with open(output_file, 'w') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=channel_fieldnames) + + writer.writeheader() + for ch in channels: + if not modbus_map: + ch['units'] = None + ch['min'] = None + ch['max'] = None + ch['change'] = None + ch['guaranteedReportPeriod'] = None + ch['minReportTime'] = None + else: + combined = combine_modbusmap_and_channel(ch, modbus_map) + if combined: + ch = combined + writer.writerow(decode_channel_parameters(ch)) + + click.echo("Wrote channels to {}".format(output_file)) + + +@click.command() +@click.argument("device_type_name") +@click.argument("csv_file") +def post_channel_csv(device_type_name, csv_file): + """Post values from a CSV to Meshify Channel API.""" + devicetypes = GET('devicetypes') + this_devicetype = find_by_name(device_type_name, devicetypes) + + with open(csv_file, 'r') as inp_file: + reader = csv.DictReader(inp_file) + for row in dict_filter(reader, 'name', + 'deviceTypeId', + 'fromMe', + 'io', + 'subTitle', + 'helpExplanation', + 'channelType', + 'dataType', + 'defaultValue', + 'regex', + 'regexErrMsg'): + # print(row) + # print(encode_channel_parameters(row)) + # click.echo(json.dumps(encode_channel_parameters(row), indent=4)) + if post_meshify_api('devicetypes/{}/channels'.format(this_devicetype['id']), encode_channel_parameters(row)): + click.echo("Successfully added channel {}".format(row['name'])) + else: + click.echo("Unable to add channel {}".format(row['name'])) + + +@click.command() +def print_channel_options(): + """Print channel options for use with the csv files.""" + channel_types = ['device', 'static', 'user input', 'system'] + io_options = ['readonly', 'readwrite'] + datatype_options = [ + "float", + 'string', + 'integer', + 'boolean', + 'datetime', + 'timespan', + 'file', + 'latlng' + ] + + click.echo("\n\nchannelType options") + click.echo("===================") + for chan in channel_types: + click.echo(chan) + + click.echo("\n\nio options") + click.echo("==========") + for i in io_options: + click.echo(i) + + click.echo("\n\ndataType options") + click.echo("================") + for d in datatype_options: + click.echo(d) + + +@click.command() +@click.argument("device_type_name") +@click.argument("csv_file") +def create_modbusMap(device_type_name, csv_file): + """Create modbusMap.p from channel csv file.""" + modbusMap = { + "1": { + "c": "ETHERNET/IP", + "b": "192.168.1.10", + "addresses": { + "300": {} + }, + "f": "Off", + "p": "", + "s": "1" + }, + "2": { + "c": "M1-485", + "b": "9600", + "addresses": {}, + "f": "Off", + "p": "None", + "s": "1" + } + } + ind = 1 + with open(csv_file, 'r') as inp_file: + reader = csv.DictReader(inp_file) + for row in reader: + modbusMap["1"]["addresses"]["300"]["2-{}".format(ind)] = make_modbusmap_channel(ind, row, device_type_name) + ind += 1 + with open("modbusMap.p", 'wb') as mod_map_file: + pickle.dump(modbusMap, mod_map_file, protocol=0) + + with open("modbusMap.json", 'w') as json_file: + json.dump(modbusMap, json_file, indent=4) + + +@click.command() +@click.option("-i", "--input-file", default="modbusMap.p", help="The modbus map pickle file to convert.") +@click.option("-o", "--output", default="modbusMap.json", help="The modbus map json file output filename.") +def pickle_to_json(input_file, output): + """Convert a pickle file to a json file.""" + if not Path(input_file).exists(): + click.echo("Pickle file {} does not exist".format(input_file)) + return + + with open(input_file, 'rb') as picklefile: + input_contents = pickle.load(picklefile) + + with open(output, 'w') as outfile: + json.dump(input_contents, outfile, indent=4) + click.echo("Wrote from {} to {}.".format(input_file, output)) + +@click.command() +@click.option("-i", "--input-file", default="modbusMap.json", help="The modbus map json file to convert.") +@click.option("-o", "--output", default="modbusMap.p", help="The modbus map pickle file output filename.") +def json_to_pickle(input_file, output): + """Convert a pickle file to a json file.""" + if not Path(input_file).exists(): + click.echo("JSON file {} does not exist".format(input_file)) + return + + with open(input_file, 'rb') as json_file: + input_contents = json.load(json_file) + + with open(output, 'wb') as outfile: + pickle.dump(input_contents, outfile, protocol=0) + click.echo("Wrote from {} to {}.".format(input_file, output)) + + +cli.add_command(get_channel_csv) +cli.add_command(post_channel_csv) +cli.add_command(print_channel_options) +cli.add_command(create_modbusMap) +cli.add_command(pickle_to_json) +cli.add_command(json_to_pickle) + +if __name__ == '__main__': + cli() diff --git a/migration/migration.ipynb b/migration/migration.ipynb new file mode 100644 index 0000000..6406c8c --- /dev/null +++ b/migration/migration.ipynb @@ -0,0 +1,248 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "#migration script to rule them all\n", + "import boto3, json, lattice" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def convertToNewConfig(config):\n", + " #old form\n", + " #\"{'mainHP': 'https://hp-drivers.s3-us-west-2.amazonaws.com/mainMeshify/','m1': 'https://hp-drivers.s3-us-west-2.amazonaws.com/M1/','flowmonitor': 'https://s3.amazonaws.com/pocloud-drivers/flow-monitor/'}\"\n", + "\n", + " #new form\n", + " #\"{'mainHP': 'https://hp-thingsboard.s3.amazonaws.com/mainHP/','m1': 'https://hp-thingsboard.s3.amazonaws.com/M1/','flowmonitor': 'https://hp-thingsboard.s3.amazonaws.com/flowmonitor/'}\"\n", + " mainHPMapping = {\n", + " \"mainMeshify\": \"https://hp-thingsboard.s3.amazonaws.com/mainHP/\",\n", + " \"piflow\": \"https://hp-thingsboard.s3.amazonaws.com/mainHPRPI/\",\n", + " \"plcfresh\": \"https://hp-thingsboard.s3.amazonaws.com/mainHPPLCFRESH/\"\n", + " }\n", + "\n", + " configMapping = {\n", + " \"abbflow\": \"https://hp-thingsboard.s3.amazonaws.com/abbflow/\",\n", + " \"advvfdipp\": \"https://hp-thingsboard.s3.amazonaws.com/advvfdipp/\",\n", + " \"dhsensor\": \"https://hp-thingsboard.s3.amazonaws.com/dhsensor/\",\n", + " \"dual_flowmeter\": \"https://hp-thingsboard.s3.amazonaws.com/dual_flowmeter/\",\n", + " \"flowmeterskid\": \"https://hp-thingsboard.s3.amazonaws.com/flowmeterskid/\",\n", + " \"flowmonitor\": \"https://hp-thingsboard.s3.amazonaws.com/flowmonitor/\",\n", + " \"PiFlow\": \"https://hp-thingsboard.s3.amazonaws.com/PiFlow/\",\n", + " \"ipp\": \"https://hp-thingsboard.s3.amazonaws.com/ipp/\",\n", + " \"plcpond\": \"https://hp-thingsboard.s3.amazonaws.com/plcpond/\",\n", + " \"multisensor\": \"https://hp-thingsboard.s3.amazonaws.com/multisensor/\",\n", + " \"dualactuator\": \"https://hp-thingsboard.s3.amazonaws.com/dualactuator/\",\n", + " \"dualactuatorpri\": \"https://hp-thingsboard.s3.amazonaws.com/dualactuatorpri/\",\n", + " \"plcfreshwater\": \"https://hp-thingsboard.s3.amazonaws.com/plcfreshwater/\",\n", + " \"pondlevel\": \"https://hp-thingsboard.s3.amazonaws.com/pondlevel/\",\n", + " \"promagmbs\": \"https://hp-thingsboard.s3.amazonaws.com/promagmbs/\",\n", + " \"poc\": \"https://hp-thingsboard.s3.amazonaws.com/poc/\",\n", + " \"recycle_train\": \"https://hp-thingsboard.s3.amazonaws.com/recycle_train/\",\n", + " \"rigpump\": \"https://hp-thingsboard.s3.amazonaws.com/rigpump/\",\n", + " \"submonitor\": \"https://hp-thingsboard.s3.amazonaws.com/submonitor/\",\n", + " \"swdcontroller\": \"https://hp-thingsboard.s3.amazonaws.com/swdcontroller/\",\n", + " \"tankalarms\": \"https://hp-thingsboard.s3.amazonaws.com/tankalarms/\",\n", + " \"tanktransfer\": \"https://hp-thingsboard.s3.amazonaws.com/tanktransfer/\",\n", + " \"tenflowmeterskid\": \"https://hp-thingsboard.s3.amazonaws.com/tenflowmeterskid/\",\n", + " \"transferlite\": \"https://hp-thingsboard.s3.amazonaws.com/transferlite/\",\n", + " \"m1\": \"https://hp-thingsboard.s3.amazonaws.com/m1/\"\n", + " }\n", + " config = json.loads(config.replace(\"'\", '\"'))\n", + " for x in dict.keys(config):\n", + " if x == \"mainHP\":\n", + " config[x] = mainHPMapping.get(config[x].split(\"/\")[-2], \"bad_request\")\n", + " else:\n", + " config[x] = configMapping.get(x, \"bad_request\")\n", + " config = json.dumps(config).replace('\"', \"'\")\n", + " print(config)\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def updateConfig(mac, client, table):\n", + " resp = client.get_item(Key={'mac': {'S': mac}}, TableName=table)\n", + " oldconfig = resp[\"Item\"]['config']\n", + " print(oldconfig)\n", + " urls = convertToNewConfig(oldconfig)\n", + " newconfig = oldconfig.copy()\n", + " newconfig['S'] = urls\n", + " print(newconfig)\n", + " client.update_item(\n", + " TableName=table,\n", + " Key={'mac': {'S': mac}},\n", + " ExpressionAttributeNames={\"#C\": 'config'},\n", + " ExpressionAttributeValues={':c': newconfig},\n", + " ReturnValues='ALL_NEW',\n", + " UpdateExpression='SET #C = :c'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = boto3.client('dynamodb')\n", + "table = \"HPDeviceList\"\n", + "macs = ['C4:93:00:0C:68:F9']\n", + "for mac in macs:\n", + " updateConfig(mac, client, table)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def convertVanityToClientId(deviceName):\n", + " #Device Name form: Jones Holton FW #2\n", + " #MQTT clientID form: jones-holton-fw-2\n", + " mqttClientId = []\n", + " for c in deviceName:\n", + " if c == \" \":\n", + " mqttClientId.append(\"-\")\n", + " elif c.isalnum():\n", + " mqttClientId.append(c.lower())\n", + " elif c == '\"':\n", + " mqttClientId.append(\"in\")\n", + " elif c == '-':\n", + " mqttClientId.append(c)\n", + " mqttClientId = \"\".join(mqttClientId)\n", + " return mqttClientId" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "nodeTypes = {\n", + " \"tankalarms\": 95\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " {\n", + " \"id\": 73138,\n", + " \"parentNodeId\": 73138,\n", + " \"aliasNodeIds\": [],\n", + " \"folderId\": 35662,\n", + " \"nodeTypeId\": 95,\n", + " \"uniqueId\": \"b8:27:eb:3d:e9:11:01:99\",\n", + " \"vanity\": \"Fasken AW Battery Tank Alarms\",\n", + " \"tags\": [],\n", + " \"location\": null,\n", + " \"metadata\": {},\n", + " \"isActive\": true,\n", + " \"archiveNumber\": 0,\n", + " \"createdAt\": \"2021-08-19T17:54:44Z\",\n", + " \"updatedAt\": \"2021-08-19T17:54:44Z\",\n", + " \"folderName\": \"Fasken AW Battery\"\n", + " },\n", + " {\n", + " \"id\": 79215,\n", + " \"parentNodeId\": 79215,\n", + " \"aliasNodeIds\": [],\n", + " \"folderId\": 36624,\n", + " \"nodeTypeId\": 95,\n", + " \"uniqueId\": \"b8:27:eb:26:55:c0:01:99\",\n", + " \"vanity\": \"Fee BM Tank Alarms\",\n", + " \"tags\": [],\n", + " \"location\": null,\n", + " \"metadata\": {},\n", + " \"isActive\": true,\n", + " \"archiveNumber\": 0,\n", + " \"createdAt\": \"2022-03-17T23:01:35Z\",\n", + " \"updatedAt\": \"2022-03-17T23:01:35Z\",\n", + " \"folderName\": \"Fee BM\"\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "#Get nodes and filter by node type\n", + "nodes = lattice.getNodes()\n", + "folders = lattice.getFolders()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fasken AW Battery fasken-aw-battery\n", + "Fee BM fee-bm\n" + ] + } + ], + "source": [ + "desiredNodes = []\n", + "desiredType = nodeTypes[\"tankalarms\"]\n", + "for node in nodes:\n", + " if node[\"nodeTypeId\"] == desiredType:\n", + " for folder in folders:\n", + " if folder[\"id\"] == node[\"folderId\"]:\n", + " node[\"folderName\"] = folder[\"name\"]\n", + " break\n", + " desiredNodes.append(node)\n", + "\n", + "#print(json.dumps(desiredNodes, indent=2))\n", + "for n in desiredNodes:\n", + " print(n[\"folderName\"],convertVanityToClientId(n[\"folderName\"]))\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aws", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}