Added thingsboard data

app.py

@@ -1,17 +1,53 @@
-import requests, os
+import requests, os, pytz
 import plotly.graph_objects as go
 import plotly.express as px
 from flask import Flask, render_template, jsonify, make_response
 from weasyprint import HTML
+import pandas as pd
+from datetime import datetime as dt
+from datetime import timedelta as td
+import datetime as dtf
+from tb_rest_client.rest_client_ce import *
+from tb_rest_client.rest import ApiException
 
 app = Flask(__name__)
 
 @app.route('/report')
 def report():
-    # Create Plotly graph
+    #Get data for graph
-    #fig = go.Figure(data=[go.Scatter(x=[1, 2, 3], y=[10, 20, 30], mode='lines')])
+    df = getData()
-    df = px.data.stocks()
+    # Create a figure with multiple y-axes
-    fig = px.line(df, x='date', y=['AAPL', 'GOOG'])
+    fig = go.Figure()
+
+    # Add the first line (INLET FLOW RATE) to the primary y-axis
+    fig.add_trace(go.Scatter(x=df.index, y=df['INLET FLOW RATE'], 
+                             mode='lines', name='Inlet Flow Rate',
+                             yaxis='y1'))
+
+    # Add the second line (INLET TURBIDITY) to the secondary y-axis
+    fig.add_trace(go.Scatter(x=df.index, y=df['SALES FLOW RATE'], 
+                             mode='lines', name='Sales Flow Rate',
+                             yaxis='y2'))
+
+    # Update layout for multiple y-axes
+    fig.update_layout(
+        #title='Graph with Multiple Y-Axes',
+        xaxis=dict(title='Time'),
+        yaxis=dict(
+            title='Flow Rate',
+            titlefont=dict(color='blue'),
+            tickfont=dict(color='blue')
+        ),
+        yaxis2=dict(
+            title='Turbidity',
+            titlefont=dict(color='red'),
+            tickfont=dict(color='red'),
+            anchor='x',
+            overlaying='y',
+            side='right'
+        ),
+        #legend=dict(x=0, y=1)
+    )
     # Save the graph as an image
     image_path = os.path.join('static', 'graph.svg')
     fig.write_image(image_path)
@@ -47,5 +83,134 @@ def download_pdf():
     response.headers['Content-Disposition'] = 'inline; filename=report.pdf'
     return response
 
+def getDataFrame(telemetry, ignore_keys, time):    
+    df = pd.DataFrame()
+    #for location in telemetry.keys():
+    # Iterate through each datapoint within each location
+    for datapoint in telemetry.keys():
+        # Convert the datapoint list of dictionaries to a DataFrame
+        if datapoint not in ignore_keys:
+            temp_df = pd.DataFrame(telemetry[datapoint])
+            temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None)
+            # Set 'ts' as the index
+            temp_df.set_index('ts', inplace=True)
+            temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce")
+            # Rename 'value' column to the name of the datapoint
+            temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)
+            
+            # Join the temp_df to the main DataFrame
+            df = df.join(temp_df, how='outer')
+            df.ffill()
+            #df = df.fillna(method='ffill', limit=2)
+     # Rename index to 'Date'
+    df.rename_axis('Date', inplace=True)
+    return df
+
+def formatColumnName(telemetryName):
+   name = " ".join([x.capitalize() for x in telemetryName.split("_")])
+   label_mapping = {
+    "Lit 116b Level": "WASTE TANK 1",
+    "Lit 116a Level": "WASTE TANK 2",
+    "Fit 100 Flow Rate": "INLET FLOW RATE",
+    "Fit 109b Flow Rate": "SALES FLOW RATE",
+    "Outlet Turbidity Temp": "OUTLET TURBIDITY TEMP",
+    "Outlet Orp Temp": "OUTLET ORP TEMP",
+    "Inlet Turbidity Temp": "INLET TURBIDITY TEMP",
+    "Inlet Ph Temp": "INLET PH TEMP",
+    "Ait 102b H2s": "INLET H₂S",
+    "At 109b H2s": "OUTLET H₂S",
+    "At 109c Oil In Water": "OUTLET OIL IN WATER",
+    "Ait 102a Turbitity": "INLET TURBIDITY",
+    "At 109a Turbidity": "OUTLET TURBIDITY",
+    "At 109e Orp": "OUTLET ORP"
+   }
+   return label_mapping.get(name)
+
+def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
+    # Creating the REST client object with context manager to get auto token refresh
+    with RestClientCE(base_url=url) as rest_client:
+        try:
+            # Auth with credentials
+            rest_client.login(username=username, password=password)
+            # Get customers > get devices under a target customer > get keys for devices > get data for devices
+            customers = rest_client.get_customers(page_size="100", page="0")
+            devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)
+            telemetry = {}
+            for d in devices.data:
+                #print(d.name)
+                device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)
+                start_ts, end_ts = getTime(timeRequest)
+                #print(keys)
+                telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)
+            return telemetry
+        except ApiException as e:
+            print(e)
+            return False
+def getTime(timeRequest):
+    start_ts, end_ts = 0,0
+    if timeRequest["type"] == "last":
+        now = dt.now()
+        delta = td(days=timeRequest["days"], seconds=timeRequest["seconds"], microseconds=timeRequest["microseconds"], milliseconds=timeRequest["milliseconds"], minutes=timeRequest["minutes"], hours=timeRequest["hours"], weeks=timeRequest["weeks"])
+        start_ts = str(int(dt.timestamp(now - delta) * 1000))
+        end_ts = str(int(dt.timestamp(now) * 1000))
+    elif timeRequest["type"] == "midnight-midnight":
+        timezone = pytz.timezone(timeRequest["timezone"])
+        today = dtf.date.today()
+        yesterday_midnight = dtf.datetime.combine(today - dtf.timedelta(days=1), dtf.time())
+        today_midnight = dtf.datetime.combine(today, dtf.time())
+        yesterday_midnight = timezone.localize(yesterday_midnight)
+        today_midnight = timezone.localize(today_midnight)
+        start_ts = int(yesterday_midnight.timestamp()) * 1000
+        end_ts = int(today_midnight.timestamp()) * 1000
+    elif timeRequest["type"] == "range":
+        start_ts = timeRequest["ts_start"]
+        end_ts = timeRequest["ts_end"]
+    return (start_ts, end_ts)
+def getTelemetry(rest_client, device, keys, start_ts, end_ts,limit):
+    try:
+        return rest_client.get_timeseries(entity_id=device.id, keys=keys, start_ts=start_ts, end_ts=end_ts, limit=limit) #entity_type=entity_type, 
+    except Exception as e:
+        print("Something went wrong in getTelemetry")
+        print(e)
+        return False
+def getDevices(rest_client, customers,target_customer, page=0, pageSize=500):
+    for c in customers.data:
+        if c.name == target_customer:
+            cid = c.id.id
+    devices = rest_client.get_customer_devices(customer_id=cid, page_size=pageSize, page=page)
+    return devices #.to_dict()
+def getDeviceKeys(rest_client, devices,target_device):
+        try:
+            for d in devices.data:
+                if d.name == target_device:
+                    device = d
+                    keys = rest_client.get_timeseries_keys_v1(d.id)
+                    return device, keys, None
+            return None, None,"Device Not Found"
+        except Exception as e:
+            print("Something went wrong in getDeviceKeys")
+            print(e)
+            return (None, None, e)
+
+def getData():
+    # ThingsBoard REST API URL
+    url = "https://www.enxlekkocloud.com" #"https://hp.henrypump.cloud"
+    # Default Tenant Administrator credentials
+    username = "nico.a.melone@gmail.com" #"henry.pump.automation@gmail.com"
+    password = "9EE#mqb*b6bXV9hJrPYGm&w3q5Y@3acumvvb5isQ" #"Henry Pump @ 2022"
+    time = {
+        "type": "range",
+        "timezone": "US/Alaska" ,
+        "ts_start": 1728115200000,
+        "ts_end": 1728201600000
+    }
+    telemetry = getThingsBoardData(url, username, password, "Thunderbird Field Services", time)
+    ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm',  'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate',  'at_109d_o2_in_water',  'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm']
+
+    #Create a Sheet for each Device
+    for device in telemetry.keys():
+        df = getDataFrame(telemetry[device], ignore_keys, time)
+    return df
+
 if __name__ == '__main__':
     app.run(debug=True)

requirements.txt

@@ -2,4 +2,6 @@ flask
 requests
 weasyprint
 plotly
-kaleido
+kaleido
+pandas==2.2.2
+tb-rest-client==3.7.0

templates/report.html

@@ -14,13 +14,13 @@
         </tr>
       </thead>
       <tbody>
-        {% for index,row in table_data.iterrows() %}
+        <!-- {% for index,row in table_data.iterrows() %}
         <tr>
-          <td>{{ row['date'] }}</td>
+          <td>{{ row['Name'] }}</td>
-          <td>{{ row['GOOG'] }}</td>
+          <td>{{ row['INLET TURBIDITY'] }}</td>
-          <td>{{ row['AAPL'] }}</td>
+          <td>{{ row['INLET FLOW RATE'] }}</td>
         </tr>
-        {% endfor %}
+        {% endfor %} -->
       </tbody>
     </table>