adds security, changes measurement access

2017-04-13 18:09:07 -05:00
parent 2354026fdf
commit c58aacc502
9 changed files with 317 additions and 105 deletions
--- a/development.ini
+++ b/development.ini
@@ -18,7 +18,7 @@ pyramid.includes =
 # '127.0.0.1' and '::1'.
 # debugtoolbar.hosts = 127.0.0.1 ::1

-mongo_uri = mongodb://localhost:27017/poc
+mongo_uri = mongodb://poc_www:HenryPump1903@localhost:27017/poc
 # mongo_uri = mongodb://10.20.155.202:27017/poc


--- a/pocwww/init.py
+++ b/pocwww/init.py
@@ -96,13 +96,12 @@ def main(global_config, **settings):
    jinja2_env.filters['datestring'] = format_dateString

    db_url = urlparse(settings['mongo_uri'])
+
    config.registry.db = MongoClient(
        host=db_url.hostname,
        port=db_url.port,
    )

-    config.registry.db.poc.users.update_one({"username": "admin"}, {"$set": {"username": "admin", "password": "l3tm31n"}}, upsert=True)
-
    def add_db(request):
        db = config.registry.db[db_url.path[1:]]
        if db_url.username and db_url.password:
@@ -112,6 +111,10 @@ def main(global_config, **settings):
    config.add_request_method(add_db, 'db', reify=True)
    config.add_static_view('static', 'static', cache_max_age=3600)

+    # Add login for admin user
+    config.registry.db.poc.authenticate(db_url.username, db_url.password)
+    config.registry.db.poc.users.update_one({"username": "admin"}, {"$set": {"username": "admin", "password": "l3tm31n"}}, upsert=True)
+
    # CUSTOM JSON RENDERER
    prettyjson = JSON(indent=4)
    prettyjson.add_adapter(datetime, datetime_adapter)
--- a/pocwww/json.py
+++ b/pocwww/json.py
@@ -2,6 +2,7 @@ from pyramid.view import view_config
 from .view_helpers import *
 from bson import json_util
 import requests
+import pytz


 # JSON
@@ -63,35 +64,6 @@ def json_valuesdaterange(request):
    return {'first_date': date_limits['first'], 'last_date': date_limits['last']}


-@view_config(route_name="json_singlevaluebetween", renderer="prettyjson")
-@view_config(route_name="json_singlevaluebetween_wparams", renderer="prettyjson")
-def json_singlevaluebetween(request):
-    end = datetime.now()
-    try:  # Attempt to get a value from the request.
-        end = request.matchdict['enddt']
-        end = end.replace("T", " ")
-        end = datetime.strptime(end, "%Y-%m-%d %H:%M:%S.%fZ")
-    except KeyError:
-        pass
-
-    start = end - timedelta(days=7)
-    try:  # Attempt to get a value from the request.
-        start = request.matchdict['startdt']
-        start = start.replace("T", " ")
-        start = datetime.strptime(start, "%Y-%m-%d %H:%M:%S.%fZ")
-    except KeyError:
-        pass
-
-    tag_data = []
-    grouped_tags = request.db['wellData'].aggregate([
-        {"$match": {"tagname": request.matchdict['tagname'], 'timestamp': {'$gt': start, '$lte': end}}},
-        {
-            '$sort': {"timestamp": 1}
-        }
-    ])
-    return {'values': list(grouped_tags), 'start': start, 'end': end}
-
-
@view_config(route_name="json_singlevaluedaterange", renderer="prettyjson")
 def json_singlevaluedaterange(request):
    date_limits = list(request.db['wellData'].aggregate([
--- a/pocwww/measurements.py
+++ b/pocwww/measurements.py
@@ -0,0 +1,185 @@
+from pyramid.view import view_config
+from pyramid.httpexceptions import HTTPFound
+from pyramid.security import remember, forget
+from datetime import datetime, timedelta
+from functools import reduce
+import pytz
+from itertools import groupby
+
+
+def get_date_measurements(request, d, tagname=None):
+    find_datetime = datetime(d.year, d.month, d.day, 0, 0, 0)
+    if tagname:
+        return list(request.db['measurements'].find({"dateStored": find_datetime, 'tagname': tagname}))
+    return list(request.db['measurements'].find({"dateStored": find_datetime}))
+
+
+def expand_measurements_from_document(doc):
+    base_date = doc['dateStored']
+    value_list = []
+    values = doc['values']
+    for hour in values:
+        for minute in values[hour]:
+            measurement_datetime = datetime(base_date.year, base_date.month, base_date.day, int(hour), int(minute), tzinfo=pytz.utc)
+            # print("{} = {}".format(measurement_datetime, values[hour][minute]))
+            value_list.append({'timestamp': measurement_datetime, 'tagvalue': values[hour][minute]})
+    value_list.sort(key=lambda x: x['timestamp'], reverse=True)
+    doc['values'] = value_list
+    return doc
+
+
+def get_measurements_between(request, start_datetime, end_datetime, tagname=None):
+    start_date = datetime(start_datetime.year, start_datetime.month, start_datetime.day, 0, 0, 0, tzinfo=pytz.utc)
+    end_date = datetime(end_datetime.year, end_datetime.month, end_datetime.day, 0, 0, 0, tzinfo=pytz.utc)
+    found_measurements = []
+    if tagname:
+        found_measurements = list(map(expand_measurements_from_document, list(request.db['measurements'].find({"dateStored": {"$lte": end_date, "$gte": start_date}, 'tagname': tagname}))))
+    else:
+        found_measurements = list(map(expand_measurements_from_document, list(request.db['measurements'].find({"dateStored": {"$lte": end_date, "$gte": start_date}}))))
+
+    for i in range(0, len(found_measurements)):
+        found_measurements[i]['values'] = list(filter(lambda x: x['timestamp'] >= start_datetime and x['timestamp'] <= end_datetime, found_measurements[i]['values']))
+    return found_measurements
+
+
+def combine_measurements(m1, m2):
+    new_measurement = {
+        "averages": [],
+        "totals": [],
+        "maxes": [],
+        "mins": [],
+        "units": m1["units"],
+        "values": sorted(m2['values'] + m1['values'], key=lambda x: x['timestamp'], reverse=True),
+        "tagname": m1["tagname"],
+    }
+
+    # Average Value
+    try:
+        new_measurement['averages'] = m1['averages']
+        new_measurement['averages'].append({"timestamp": m2['dateStored'], "averageValue": m2['averageValue']})
+    except KeyError:
+        new_measurement['averages'].append({"timestamp": m1['dateStored'], "averageValue": m1['averageValue']})
+        new_measurement['averages'].append({"timestamp": m2['dateStored'], "averageValue": m2['averageValue']})
+
+    # Total Value
+    try:
+        new_measurement['totals'] = m1['totals']
+        new_measurement['totals'].append({"timestamp": m2['dateStored'], "totalValue": m2['totalValue']})
+    except KeyError:
+        new_measurement['totals'].append({"timestamp": m1['dateStored'], "totalValue": m1['totalValue']})
+        new_measurement['totals'].append({"timestamp": m2['dateStored'], "totalValue": m2['totalValue']})
+
+    # Max Value
+    try:
+        new_measurement['maxes'] = m1['maxes']
+        new_measurement['maxes'].append({"timestamp": m2['dateStored'], "maxValue": m2['maxValue']})
+    except KeyError:
+        new_measurement['maxes'].append({"timestamp": m1['dateStored'], "maxValue": m1['maxValue']})
+        new_measurement['maxes'].append({"timestamp": m2['dateStored'], "maxValue": m2['maxValue']})
+
+    # Min Value
+    try:
+        new_measurement['mins'] = m1['mins']
+        new_measurement['mins'].append({"timestamp": m2['dateStored'], "minValue": m2['minValue']})
+    except KeyError:
+        new_measurement['mins'].append({"timestamp": m1['dateStored'], "minValue": m1['minValue']})
+        new_measurement['mins'].append({"timestamp": m2['dateStored'], "minValue": m2['minValue']})
+
+    return new_measurement
+
+
+def first_measurement(m1):
+    new_measurement = {
+        "averages": [],
+        "totals": [],
+        "maxes": [],
+        "mins": [],
+        "units": m1["units"],
+        "values": sorted(m1['values'], key=lambda x: x['timestamp'], reverse=True),
+        "tagname": m1["tagname"],
+    }
+
+    # Average Value
+    try:
+        new_measurement['averages'] = m1['averages']
+    except KeyError:
+        new_measurement['averages'].append({"timestamp": m1['dateStored'], "averageValue": m1['averageValue']})
+
+    # Total Value
+    try:
+        new_measurement['totals'] = m1['totals']
+    except KeyError:
+        new_measurement['totals'].append({"timestamp": m1['dateStored'], "totalValue": m1['totalValue']})
+
+    # Max Value
+    try:
+        new_measurement['maxes'] = m1['maxes']
+    except KeyError:
+        new_measurement['maxes'].append({"timestamp": m1['dateStored'], "maxValue": m1['maxValue']})
+
+    # Min Value
+    try:
+        new_measurement['mins'] = m1['mins']
+    except KeyError:
+        new_measurement['mins'].append({"timestamp": m1['dateStored'], "minValue": m1['minValue']})
+
+    return new_measurement
+
+
+def get_grouped_measurements_between(request, start_datetime, end_datetime, tagname=None):
+    data = sorted(get_measurements_between(request, start_datetime, end_datetime, tagname=tagname), key=lambda x: x['tagname'])
+    tag_data = {}
+    for k, g in groupby(data, lambda x: x['tagname']):
+        group = list(g)
+        if len(group) > 1:
+            tag_data[k] = reduce(combine_measurements, group)
+        else:
+            tag_data[k] = first_measurement(group[0])
+
+    return tag_data
+
+
+@view_config(route_name='values_all', renderer="templates/valuesall.jinja2")
+@view_config(route_name='json_values_all', renderer='prettyjson')
+def values_all(request):
+    end = datetime.now(tz=pytz.utc)
+    try:  # Attempt to get a value from the request.
+        end = request.matchdict['enddt']
+        end = end.replace("T", " ")
+        end = pytz.utc.localize(datetime.strptime(end, "%Y-%m-%d %H:%M:%S.%fZ"))
+    except KeyError:
+        pass
+
+    start = end - timedelta(days=2)
+    try:  # Attempt to get a value from the request.
+        start = request.matchdict['startdt']
+        start = start.replace("T", " ")
+        start = pytz.utc.localize(datetime.strptime(start, "%Y-%m-%d %H:%M:%S.%fZ"))
+    except KeyError:
+        pass
+
+    all_values = get_grouped_measurements_between(request, start, end)
+    return {'navgroup': 'values', 'current_tag_values': all_values}
+
+
+@view_config(route_name="json_singlevaluebetween", renderer="prettyjson")
+@view_config(route_name="json_singlevaluebetween_wparams", renderer="prettyjson")
+def json_singlevaluebetween(request):
+    end = datetime.now(tz=pytz.utc)
+    try:  # Attempt to get a value from the request.
+        end = request.matchdict['enddt']
+        end = end.replace("T", " ")
+        end = pytz.utc.localize(datetime.strptime(end, "%Y-%m-%d %H:%M:%S.%fZ"))
+    except KeyError:
+        pass
+
+    start = end - timedelta(days=7)
+    try:  # Attempt to get a value from the request.
+        start = request.matchdict['startdt']
+        start = start.replace("T", " ")
+        start = pytz.utc.localize(datetime.strptime(start, "%Y-%m-%d %H:%M:%S.%fZ"))
+    except KeyError:
+        pass
+
+    all_values = get_grouped_measurements_between(request, start, end, tagname=request.matchdict['tagname'])
+    return {'tag': all_values, 'start': start, 'end': end, 'tagname': request.matchdict['tagname']}
--- a/pocwww/static/graphs.js
+++ b/pocwww/static/graphs.js
@@ -130,33 +130,95 @@ function drawSingleGraph(data){
        console.log("Destroying existing chart");
        scatterChart.destroy();
    }
+
+    var tag = data.tag[data.tagname];
+    console.log(tag);
+    console.log(tag.values.length);
+
    var graph_data = [];
-    var values = data.values;
    var ctx = document.getElementById("myChart");
-    var lines = ["currentValue", "maxDailyValue", "minDailyValue", "dailyAverage", "dailyTotal"];
-    for (var i = 0; i < lines.length; i++){
-        var baseObj = {
-            label: lines[i],
-            fill: false,
-            data: [],
-            lineTension: 0.05,
-            borderColor: color_scale[i % color_scale.length]
-        }
-        if (lines[i] == "dailyTotal"){
-            baseObj['hidden'] = true;
-        }
-        graph_data.push(baseObj);
+    var color_index = 0;
+
+    // Current Values
+    var currentValues = {
+        label: "Value",
+        fill: false,
+        data: [],
+        lineTension: 0.05,
+        borderColor: color_scale[color_index % color_scale.length]
    }

-    for(var i = 0; i < values.length; i++){
-        for(var j = 0; j < lines.length; j++){
-            var lineName = lines[j]
-            graph_data[lines.indexOf(lineName)].data.push({
-                x: values[i].timestamp,
-                y: values[i][lineName]
-            });
-        }
+    for (var i = 0; i < tag.values.length; i++){
+        currentValues.data.push({x: tag.values.timestamp, y: tag.values.tagvalue});
    }
+
+    graph_data.push(currentValues)
+    color_index++;
+
+    // Max Values
+    var maxValues = {
+        label: "Max",
+        fill: false,
+        data: [],
+        lineTension: 0.05,
+        borderColor: color_scale[color_index % color_scale.length]
+    }
+
+    for (var i = 0; i < tag.maxes.length; i++){
+        maxValues.data.push({x: tag.maxes.timestamp, y: tag.maxes.maxValue});
+    }
+
+    graph_data.push(maxValues)
+    color_index++;
+
+    // Min Values
+    var minValues = {
+        label: "Min",
+        fill: false,
+        data: [],
+        lineTension: 0.05,
+        borderColor: color_scale[color_index % color_scale.length]
+    }
+
+    for (var i = 0; i < tag.mins.length; i++){
+        minValues.data.push({x: tag.mins.timestamp, y: tag.mins.maxValue});
+    }
+
+    graph_data.push(minValues)
+    color_index++;
+
+    // Average Values
+    var averageValues = {
+        label: "Average",
+        fill: false,
+        data: [],
+        lineTension: 0.05,
+        borderColor: color_scale[color_index % color_scale.length]
+    }
+
+    for (var i = 0; i < tag.averages.length; i++){
+        averageValues.data.push({x: tag.averages.timestamp, y: tag.averages.maxValue});
+    }
+
+    graph_data.push(averageValues)
+    color_index++;
+
+    // Total Values
+    var totalValues = {
+        label: "Total",
+        fill: false,
+        data: [],
+        lineTension: 0.05,
+        borderColor: color_scale[color_index % color_scale.length]
+    }
+
+    for (var i = 0; i < tag.totals.length; i++){
+        totalValues.data.push({x: tag.totals.timestamp, y: tag.totals.maxValue});
+    }
+
+    graph_data.push(totalValues)
+    color_index++;
+
    scatterChart = new Chart(ctx, {
        type: 'line',
        data: {
--- a/pocwww/templates/dashboard.jinja2
+++ b/pocwww/templates/dashboard.jinja2
@@ -23,11 +23,6 @@
                            <th class="allcaps">Name</th>
                            <th class="allcaps"></th>
                            <th class="allcaps">Value</th>
-                            <th class="allcaps">Timestamp</th>
-                            <!-- <th>Max</th>
-                            <th>Min</th>
-                            <th>Average</th>
-                            <th>Total</th> -->
                        </tr>
                    </thead>
                    <tbody>
@@ -36,19 +31,18 @@
                        <tr>
                            <td><a href="/values/tag/{{t._id}}">{{t._id}}</a></td>
                            <td>
-                                <button type="button" class="btn btn-default" title="{{t._id}}" data-container="body" data-toggle="popover" data-trigger="focus" data-placement="right" data-content="Max: {{t.max | round(3)}}<br \>
-                                    Min: {{t.min | round(3)}}<br \>
-                                    Average: {{t.average | round(3)}}<br \>
-                                    Total: {{t.total | round(3)}}">
+                                <button type="button" class="btn btn-default" title="{{t._id}}" data-container="body" data-toggle="popover" data-trigger="focus" data-placement="right" data-content="Max: {{t.maxValue | round(3)}} {{t.units}}<br \>
+                                    Min: {{t.minValue | round(3)}} {{t.units}}<br \>
+                                    {% if t.useAverage %}
+                                        Average: {{t.averageValue | round(3)}} {{t.units}}<br \>
+                                    {% endif %}
+                                    {% if t.useTotal %}
+                                        Total: {{t.totalValue | round(3)}} {{t.units}}
+                                    {% endif %}">
                                    Details
                                </button>
                            </td>
-                            <td>{{t.value | round(3)}}</td>
-                            <td>{{t.timestamp | datetime('short')}}</td>
-                            <!-- <td>{{t.max | round(3)}}</td>
-                            <td>{{t.min | round(3)}}</td>
-                            <td>{{t.average | round(3)}}</td>
-                            <td>{{t.total | round(3)}}</td> -->
+                            <td>{{t.currentValue | round(3)}} {{t.units}}</td>
                        </tr>
                        {% endfor %}

--- a/pocwww/view_helpers.py
+++ b/pocwww/view_helpers.py
@@ -8,37 +8,6 @@ from_zone = tz.tzutc()
 to_zone = tz.tzlocal()


-def get_lastest_tag_values(request):
-    latest_tag_values = []
-    latesttag_agg = request.db['wellData'].aggregate([
-        {
-            '$sort': {"tagname": 1, "timestamp": 1}
-        },
-        {
-            '$group': {
-                '_id': "$tagname",
-                'timestamp': {'$last': "$timestamp"},
-                'value': {'$last': "$currentValue"},
-                'max': {'$last': "$maxDailyValue"},
-                'min': {'$last': "$minDailyValue"},
-                'average': {'$last': "$dailyAverage"},
-                'total': {'$last': "$dailyTotal"}
-            }
-        }
-    ])
-    for t in latesttag_agg:
-        latest_tag_values.append(t)
-    return latest_tag_values
-
-
-def get_latest_card(request):
-    try:
-        latest_card = list(request.db['cards'].find().sort("timestamp", -1).limit(1))[0]
-        return latest_card
-    except IndexError:
-        return []
-
-
 def card_page(request):
    page_num = 1
    try:
--- a/pocwww/views.py
+++ b/pocwww/views.py
@@ -9,7 +9,33 @@ from .view_helpers import *
@view_config(route_name='home', renderer='templates/dashboard.jinja2')
@view_config(route_name="json_snapshot", renderer="prettyjson")
 def my_view(request):
-    return {'project': 'POC Web Interface', 'navgroup': 'dashboard', 'tag_values': get_lastest_tag_values(request), 'card': get_latest_card(request)}
+    latest_tag_values = []
+    latesttag_agg = request.db['measurements'].aggregate([
+        {'$sort': {'dateStored': 1}},
+        {
+            '$group': {
+                '_id': '$tagname',
+                'currentValue': {'$last': '$currentValue'},
+                'units': {'$last': '$units'},
+                'maxValue': {'$last': '$maxValue'},
+                'minValue': {'$last': '$minValue'},
+                'totalValue': {'$last': '$totalValue'},
+                'averageValue': {'$last': '$averageValue'},
+                'useTotal': {'$last': '$useTotal'},
+                'useAverage': {'$last': '$useAverage'}
+            }
+        }
+    ])
+    for t in latesttag_agg:
+        latest_tag_values.append(t)
+
+    latest_card = []
+    try:
+        latest_card = list(request.db['cards'].find().sort("timestamp", -1).limit(1))[0]
+    except IndexError:
+        latest_card = []
+
+    return {'project': 'POC Web Interface', 'navgroup': 'dashboard', 'tag_values': latest_tag_values, 'card': latest_card}


@view_config(route_name='cards', renderer='templates/datelist.jinja2')
@@ -48,10 +74,10 @@ def card_single(request):
    return {"card": card, 'navgroup': 'cards', 'datelist': get_all_dates_with_cards(request), 'date': carddate, 'datepage': page_num}


-@view_config(route_name='values_all', renderer="templates/valuesall.jinja2")
-@view_config(route_name='json_values_all', renderer='prettyjson')
-def values_all(request):
-    return {'navgroup': 'values', 'current_tag_values': get_lastest_tag_values(request)}
+# @view_config(route_name='values_all', renderer="templates/valuesall.jinja2")
+# @view_config(route_name='json_values_all', renderer='prettyjson')
+# def values_all(request):
+#     return {'navgroup': 'values', 'current_tag_values': get_lastest_tag_values(request)}


@view_config(route_name="values_tag", renderer="templates/values_single.jinja2")
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,7 @@ requires = [
    'requests',
    'passlib',
    'python-dateutil',
+    'pytz',
 ]

 tests_require = [