Add tag error fix, db optimization

2016-04-26 13:30:30 -05:00
parent 30ec348d48
commit a277e47fd3
5 changed files with 179 additions and 10 deletions
--- a/sampleData.py
+++ b/sampleData.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+'''
+MySQL Tag Server
+Created on April 7, 2016
+@author: Patrick McDonagh
+@description: Continuously loops through a list of tags to store values from a PLC into a MySQL database
+'''
+import mysql.connector as mysqlcon
+import pickle
+from tag.tag_mysql import Tag
+import traceback
+import time
+import os
+import random
+
+
+class Sample(Tag):
+    def read(self, forceSend):
+        writeToDB = False
+        if self.tag:
+            v = 0.0
+            if not (self.value is None):
+                v = [self.value + (10.0 * (random.random() - 0.5))]
+            else:
+                v = [random.random() * 100.0]
+            if v:
+                val = v[0]
+                if self.data_type == 'BOOL' or self.data_type == 'STRING':
+                    if self.mapFn:
+                        val = self.mapFn[val]
+                    if (self.last_send_time == 0) or (self.value is None) or not (self.value == val) or ((time.time() - self.last_send_time) > self.guarantee_sec) or (forceSend is True):
+                        self.last_value = self.value
+                        self.value = val
+                        writeToDB = True
+                    else:
+                        writeToDB = False
+                else:
+                    if (self.last_send_time == 0) or (self.value is None) or (abs(self.value - v[0]) > self.chg_threshold) or ((time.time() - self.last_send_time) > self.guarantee_sec) or (forceSend is True):
+                        self.last_value = self.value
+                        self.value = v[0]
+                        writeToDB = True
+                    else:
+                        writeToDB = False
+            if forceSend is False:
+                writeToDB = False
+        if writeToDB:
+            self.sendToDB()
+        return self.value
+
+
+with open(os.path.realpath('.') + '/mysql_cfg.pickle', 'rb') as pickleconfig:
+	mysql_cfg = pickle.load(pickleconfig)
+
+if mysql_cfg:
+	db = mysqlcon.connect(**mysql_cfg)
+
+tag_store = {}
+configProperties = {}
+
+def main():
+	db.connect()
+	cur = db.cursor()
+	query = "SELECT * FROM tags WHERE class = 5 AND deleted = 0"
+	cur.execute(query)
+	tags = cur.fetchall()
+	print tags
+	#  [(1, u'Century Counter Up', 5, u'Century_Counter_Up', u'REAL', 10.0, 3600, None, 0)]
+	db.disconnect()
+
+
+	configObj = {}
+	db.connect()
+	cur = db.cursor()
+	query = "SELECT parameter, val FROM config GROUP BY parameter;"
+	cur.execute(query)
+	config = cur.fetchall()
+	db.disconnect()
+	for x in config:
+		configObj[x[0]] = x[1]
+
+	try:
+		configProperties['PLC_IP_ADDRESS'] = str(configObj['ip_address'])
+		print("FYI, using PLC IP Address from the database {0}".format(configProperties['PLC_IP_ADDRESS']))
+	except KeyError:
+		print("FYI, there is no PLC IP Address stored in the database, defaulting to 192.168.1.10")
+		configProperties['PLC_IP_ADDRESS'] = "192.168.1.10"
+
+	try:
+		configProperties['plc_type'] = str(configObj['plc_type'])
+		print("FYI, using PLC Type from the database {0}".format(configProperties['plc_type']))
+	except KeyError:
+		print("FYI, there is no PLC Type stored in the database, defaulting to CLX")
+		configProperties['plc_type'] = "CLX"
+
+	try:
+		configProperties['scan_rate'] = int(configObj['scan_rate'])
+		print("FYI, using Scan Rate from the database {0}".format(configProperties['scan_rate']))
+	except KeyError:
+		print("FYI, there is no Scan Rate stored in the database, defaulting to 10 seconds")
+		configProperties['scan_rate'] = 10
+
+	try:
+		sa_test = str(configObj['save_all'])
+		if sa_test.lower() == "true":
+			configProperties['save_all'] = True
+		elif sa_test.lower() == "false":
+			configProperties['save_all'] = False
+		else:
+			configProperties['save_all'] = "test"
+		print("FYI, value for save_all is {0}".format(configProperties['save_all']))
+	except KeyError:
+		print("FYI, there is no save_all value stored in the database, using 'test'")
+		configProperties['save_all'] = 'test'
+
+
+
+
+	for t in tags:
+		tag_store[t[1]] = Sample(t[1], t[3], t[0], t[5], t[6], t[7], mapFn=t[8], device_type=configProperties['plc_type'], ip_address=configProperties['PLC_IP_ADDRESS'])
+
+
+	while True:
+		for tag in tag_store:
+			try:
+				tag_store[tag].read(configProperties['save_all'])
+			except:
+				print("ERROR EVALUATING {}".format(tag))
+				traceback.print_exc()
+		time.sleep(configProperties['scan_rate'])
+
+if __name__ == '__main__':
+	main()
--- a/www/dbcreate_MySQL.sql
+++ b/www/dbcreate_MySQL.sql
@@ -1,36 +1,66 @@
 CREATE DATABASE poconsole;
 USE poconsole;
 CREATE TABLE IF NOT EXISTS tag_classes(
-    id int(11) NOT NULL AUTO_INCREMENT,
+    id INT NOT NULL AUTO_INCREMENT,
    tag_class varchar(64),
    description varchar(64),
    PRIMARY KEY (id)
 );

+CREATE TABLE IF NOT EXISTS device_types(
+    id INT NOT NULL AUTO_INCREMENT,
+    dType VARCHAR(64),
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE IF NOT EXISTS devices(
+    id INT NOT NULL AUTO_INCREMENT,
+    name varchar(64),
+    device_type INT,
+    address VARCHAR(64),
+    PRIMARY KEY (id),
+    INDEX device_type_ind (device_type),
+    FOREIGN KEY (device_type)
+        REFERENCES device_types(id)
+);
+
 CREATE TABLE IF NOT EXISTS tags(
-    id int(11) NOT NULL AUTO_INCREMENT,
+    id INT NOT NULL AUTO_INCREMENT,
    name varchar(128),
-    class int(11),
+    class INT,
    tag varchar(128),
+    deviceID INT,
    description varchar(128),
    data_type varchar(32),
    change_threshold float,
-    guarantee_sec integer(11),
+    guarantee_sec INT,
    map_function varchar(64),
    units varchar(64),
    minExpected varchar(64),
    maxExpected varchar(64),
    deleted INT NULL DEFAULT 0,
-    PRIMARY KEY (id)
+    PRIMARY KEY (id),
+    INDEX class_ind (class),
+    FOREIGN KEY (class)
+        REFERENCES tag_classes(id)
+        ON DELETE CASCADE,
+    INDEX deviceID_ind (deviceID),
+    FOREIGN KEY (deviceID)
+        REFERENCES devices(id)
+        ON DELETE CASCADE
 );


 CREATE TABLE IF NOT EXISTS tag_vals(
-  id int(11) NOT NULL AUTO_INCREMENT,
+  id INT NOT NULL AUTO_INCREMENT,
  dtime datetime,
  tagID int,
  val float,
-  PRIMARY KEY (id)
+  PRIMARY KEY (id),
+  INDEX tagID_ind (tagID),
+  FOREIGN KEY (tagID)
+    REFERENCES tags(id)
+    ON DELETE CASCADE
 );

 CREATE TABLE IF NOT EXISTS config (
@@ -47,6 +77,12 @@ INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (3, 'gauge
 INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (4, 'welltest', 'Well Test Data');
 INSERT INTO poconsole.tag_classes (id, tag_class, description) VALUES (5, 'custom', 'Custom tags');

+INSERT INTO poconsole.device_types (id, dType) VALUES (1, "CLX");
+INSERT INTO poconsole.device_types (id, dType) VALUES (2, "Micro800");
+INSERT INTO poconsole.device_types (id, dType) VALUES (3, "E300");
+-- INSERT INTO poconsole.device_types (id, dType) VALUES (4, "PF755");
+
+
 CREATE USER 'website'@'localhost' IDENTIFIED BY 'henrypump';
 GRANT ALL ON *.* TO 'website'@'localhost';
 CREATE USER 'admin'@'localhost' IDENTIFIED BY 'henrypump';
--- a/www/functions_MySQL.coffee
+++ b/www/functions_MySQL.coffee
@@ -66,7 +66,7 @@ exports.getAllTags = (req, res) ->

 exports.createTag = (req, res) ->
    req.app.locals.pool.getConnection (err, db) ->
-        query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?)'
+        query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?, ?)'
        db.query query, [req.body.tag, req.body.units, req.body.minExpected, req.body.maxExpected, req.body.name, req.body.description, req.body.guarantee_sec, req.body.change_threshold, req.body.data_type], (err, results) ->
            if err
                res.json
--- a/www/functions_SQLite.coffee
+++ b/www/functions_SQLite.coffee
@@ -81,7 +81,7 @@ exports.createTag = (req, res) ->
    sqlite3 = require('sqlite3').verbose()
    db = new (sqlite3.Database)(dbFile)
    db.serialize ->
-        query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?)'
+        query = 'INSERT INTO tags (tag, units, minExpected, maxExpected, name, description, class, guarantee_sec, change_threshold, data_type) VALUES (?, ?, ?, ?, ?, ?, 5, ?, ?, ?)'
        prepQuery = db.prepare(query)
        prepQuery.run req.body.tag, req.body.units, req.body.minExpected, req.body.maxExpected, req.body.name, req.body.description, req.body.guarantee_sec, req.body.change_threshold, req.body.data_type, (err) ->
            prepQuery.finalize()
--- a/www/views/angularIndex.html
+++ b/www/views/angularIndex.html
@@ -18,7 +18,7 @@
 		<script src="/bower_components/justgage-toorshia/justgage.js"></script>
 		<script src="/bower_components/angular-justgage/ng-justgage.js"></script>

-		<script src="/node_modules/n3-charts/node_modules/d3/d3.min.js"></script>
+		<script src="/node_modules/d3/d3.min.js"></script>
 	    <script src="/node_modules/n3-charts/build/LineChart.js"></script>
 	    <link rel="stylesheet" href="/node_modules/n3-charts/build/LineChart.css">
 		<link rel="stylesheet" href="/bower_components/font-awesome/css/font-awesome.min.css">