From a277e47fd3eecea4fef3c4125c89e75a26031a0f Mon Sep 17 00:00:00 2001 From: Patrick McDonagh Date: Tue, 26 Apr 2016 13:30:30 -0500 Subject: [PATCH] Add tag error fix, db optimization --- sampleData.py | 133 ++++++++++++++++++++++++++++++++++++ www/dbcreate_MySQL.sql | 50 ++++++++++++-- www/functions_MySQL.coffee | 2 +- www/functions_SQLite.coffee | 2 +- www/views/angularIndex.html | 2 +- 5 files changed, 179 insertions(+), 10 deletions(-) create mode 100644 sampleData.py diff --git a/sampleData.py b/sampleData.py new file mode 100644 index 0000000..35c481c --- /dev/null +++ b/sampleData.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +''' +MySQL Tag Server +Created on April 7, 2016 +@author: Patrick McDonagh +@description: Continuously loops through a list of tags to store values from a PLC into a MySQL database +''' +import mysql.connector as mysqlcon +import pickle +from tag.tag_mysql import Tag +import traceback +import time +import os +import random + + +class Sample(Tag): + def read(self, forceSend): + writeToDB = False + if self.tag: + v = 0.0 + if not (self.value is None): + v = [self.value + (10.0 * (random.random() - 0.5))] + else: + v = [random.random() * 100.0] + if v: + val = v[0] + if self.data_type == 'BOOL' or self.data_type == 'STRING': + if self.mapFn: + val = self.mapFn[val] + if (self.last_send_time == 0) or (self.value is None) or not (self.value == val) or ((time.time() - self.last_send_time) > self.guarantee_sec) or (forceSend is True): + self.last_value = self.value + self.value = val + writeToDB = True + else: + writeToDB = False + else: + if (self.last_send_time == 0) or (self.value is None) or (abs(self.value - v[0]) > self.chg_threshold) or ((time.time() - self.last_send_time) > self.guarantee_sec) or (forceSend is True): + self.last_value = self.value + self.value = v[0] + writeToDB = True + else: + writeToDB = False + if forceSend is False: + writeToDB = False + if writeToDB: + self.sendToDB() + return self.value + + +with open(os.path.realpath('.') + '/mysql_cfg.pickle', 'rb') as pickleconfig: + mysql_cfg = pickle.load(pickleconfig) + +if mysql_cfg: + db = mysqlcon.connect(**mysql_cfg) + +tag_store = {} +configProperties = {} + +def main(): + db.connect() + cur = db.cursor() + query = "SELECT * FROM tags WHERE class = 5 AND deleted = 0" + cur.execute(query) + tags = cur.fetchall() + print tags + # [(1, u'Century Counter Up', 5, u'Century_Counter_Up', u'REAL', 10.0, 3600, None, 0)] + db.disconnect() + + + configObj = {} + db.connect() + cur = db.cursor() + query = "SELECT parameter, val FROM config GROUP BY parameter;" + cur.execute(query) + config = cur.fetchall() + db.disconnect() + for x in config: + configObj[x[0]] = x[1] + + try: + configProperties['PLC_IP_ADDRESS'] = str(configObj['ip_address']) + print("FYI, using PLC IP Address from the database {0}".format(configProperties['PLC_IP_ADDRESS'])) + except KeyError: + print("FYI, there is no PLC IP Address stored in the database, defaulting to 192.168.1.10") + configProperties['PLC_IP_ADDRESS'] = "192.168.1.10" + + try: + configProperties['plc_type'] = str(configObj['plc_type']) + print("FYI, using PLC Type from the database {0}".format(configProperties['plc_type'])) + except KeyError: + print("FYI, there is no PLC Type stored in the database, defaulting to CLX") + configProperties['plc_type'] = "CLX" + + try: + configProperties['scan_rate'] = int(configObj['scan_rate']) + print("FYI, using Scan Rate from the database {0}".format(configProperties['scan_rate'])) + except KeyError: + print("FYI, there is no Scan Rate stored in the database, defaulting to 10 seconds") + configProperties['scan_rate'] = 10 + + try: + sa_test = str(configObj['save_all']) + if sa_test.lower() == "true": + configProperties['save_all'] = True + elif sa_test.lower() == "false": + configProperties['save_all'] = False + else: + configProperties['save_all'] = "test" + print("FYI, value for save_all is {0}".format(configProperties['save_all'])) + except KeyError: + print("FYI, there is no save_all value stored in the database, using 'test'") + configProperties['save_all'] = 'test' + + + + + for t in tags: + tag_store[t[1]] = Sample(t[1], t[3], t[0], t[5], t[6], t[7], mapFn=t[8], device_type=configProperties['plc_type'], ip_address=configProperties['PLC_IP_ADDRESS']) + + + while True: + for tag in tag_store: + try: + tag_store[tag].read(configProperties['save_all']) + except: + print("ERROR EVALUATING {}".format(tag)) + traceback.print_exc() + time.sleep(configProperties['scan_rate']) + +if __name__ == '__main__': + main() diff --git a/www/dbcreate_MySQL.sql b/www/dbcreate_MySQL.sql index 25f1512..039e62d 100644 --- a/www/dbcreate_MySQL.sql +++ b/www/dbcreate_MySQL.sql @@ -1,36 +1,66 @@ CREATE DATABASE poconsole; USE poconsole; CREATE TABLE IF NOT EXISTS tag_classes( - id int(11) NOT NULL AUTO_INCREMENT, + id INT NOT NULL AUTO_INCREMENT, tag_class varchar(64), description varchar(64), PRIMARY KEY (id) ); +CREATE TABLE IF NOT EXISTS device_types( + id INT NOT NULL AUTO_INCREMENT, + dType VARCHAR(64), + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS devices( + id INT NOT NULL AUTO_INCREMENT, + name varchar(64), + device_type INT, + address VARCHAR(64), + PRIMARY KEY (id), + INDEX device_type_ind (device_type), + FOREIGN KEY (device_type) + REFERENCES device_types(id) +); + CREATE TABLE IF NOT EXISTS tags( - id int(11) NOT NULL AUTO_INCREMENT, + id INT NOT NULL AUTO_INCREMENT, name varchar(128), - class int(11), + class INT, tag varchar(128), + deviceID INT, description varchar(128), data_type varchar(32), change_threshold float, - guarantee_sec integer(11), + guarantee_sec INT, map_function varchar(64), units varchar(64), minExpected varchar(64), maxExpected varchar(64), deleted INT NULL DEFAULT 0, - PRIMARY KEY (id) + PRIMARY KEY (id), + INDEX class_ind (class), + FOREIGN KEY (class) + REFERENCES tag_classes(id) + ON DELETE CASCADE, + INDEX deviceID_ind (deviceID), + FOREIGN KEY (deviceID) + REFERENCES devices(id) + ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS tag_vals( - id int(11) NOT NULL AUTO_INCREMENT, + id INT NOT NULL AUTO_INCREMENT, dtime datetime, tagID int, val float, - PRIMARY KEY (id) + PRIMARY KEY (id), + INDEX tagID_ind (tagID), + FOREIGN KEY (tagID) + REFERENCES tags(id) + ON DELETE CASCADE ); CREATE TABLE IF NOT EXISTS config ( @@ -47,6 +77,12 @@ INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (3, 'gauge INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (4, 'welltest', 'Well Test Data'); INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (5, 'custom', 'Custom tags'); +INSERT INTO poconsole.device_types (id, dType) VALUES (1, "CLX"); +INSERT INTO poconsole.device_types (id, dType) VALUES (2, "Micro800"); +INSERT INTO poconsole.device_types (id, dType) VALUES (3, "E300"); +-- INSERT INTO poconsole.device_types (id, dType) VALUES (4, "PF755"); + + CREATE USER 'website'@'localhost' IDENTIFIED BY 'henrypump'; GRANT ALL ON *.* TO 'website'@'localhost'; CREATE USER 'admin'@'localhost' IDENTIFIED BY 'henrypump'; diff --git a/www/functions_MySQL.coffee b/www/functions_MySQL.coffee index 50c1c8b..0fd92f0 100644 --- a/www/functions_MySQL.coffee +++ b/www/functions_MySQL.coffee @@ -66,7 +66,7 @@ exports.getAllTags = (req, res) -> exports.createTag = (req, res) -> req.app.locals.pool.getConnection (err, db) -> - query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?)' + query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?, ?)' db.query query, [req.body.tag, req.body.units, req.body.minExpected, req.body.maxExpected, req.body.name, req.body.description, req.body.guarantee_sec, req.body.change_threshold, req.body.data_type], (err, results) -> if err res.json diff --git a/www/functions_SQLite.coffee b/www/functions_SQLite.coffee index 6961acb..548624c 100644 --- a/www/functions_SQLite.coffee +++ b/www/functions_SQLite.coffee @@ -81,7 +81,7 @@ exports.createTag = (req, res) -> sqlite3 = require('sqlite3').verbose() db = new (sqlite3.Database)(dbFile) db.serialize -> - query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?)' + query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?, ?)' prepQuery = db.prepare(query) prepQuery.run req.body.tag, req.body.units, req.body.minExpected, req.body.maxExpected, req.body.name, req.body.description, req.body.guarantee_sec, req.body.change_threshold, req.body.data_type, (err) -> prepQuery.finalize() diff --git a/www/views/angularIndex.html b/www/views/angularIndex.html index 4ce77e1..cf7e895 100644 --- a/www/views/angularIndex.html +++ b/www/views/angularIndex.html @@ -18,7 +18,7 @@ - +