nmelone/ThingsBoard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated git ignore

Browse Source
This commit is contained in:
Nico Melone
2025-03-05 17:43:46 -06:00
parent ab27d7fb93
commit 113f1fb405
818 changed files with 295 additions and 79815 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
EKKO Reports/buildReportEKKO2.ipynb
View File

@@ -241,107 +241,6 @@
" return \" \".join([x.upper() for x in telemetryName.split(\"_\")])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def getDataFrame(telemetry, keys, time): \n",
" df = pd.DataFrame()\n",
" #for location in telemetry.keys():\n",
" # Iterate through each datapoint within each location\n",
" for datapoint in telemetry.keys():\n",
" # Convert the datapoint list of dictionaries to a DataFrame\n",
" if datapoint in keys:\n",
" temp_df = pd.DataFrame(telemetry[datapoint])\n",
" temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n",
" # Set 'ts' as the index\n",
" temp_df.set_index('ts', inplace=True)\n",
" temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n",
" # Rename 'value' column to the name of the datapoint\n",
" temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n",
" \n",
" # Join the temp_df to the main DataFrame\n",
" df = df.join(temp_df, how='outer')\n",
" df.ffill()\n",
" #df = df.fillna(method='ffill', limit=2)\n",
" df = df.reindex(sorted(df.columns), axis=1)\n",
" # Rename index to 'Date'\n",
" df.rename_axis('Date', inplace=True)\n",
" return df\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def getManualDataFrame(telemetry, keys, time): \n",
" df = pd.DataFrame()\n",
" for key in keys:\n",
" if key not in telemetry.keys():\n",
" telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}]\n",
" for datapoint in telemetry.keys():\n",
" if datapoint in keys:\n",
" temp_df = pd.DataFrame(telemetry[datapoint])\n",
" temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n",
" temp_df.set_index('ts', inplace=True)\n",
" if datapoint in [\"manual_next_pigging_scheduled\"]:\n",
" temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n",
" print(temp_df)\n",
" elif datapoint in [\"manual_equipment_description\",\"manual_issues_concerns\"]:\n",
" temp_df[\"value\"] = temp_df[\"value\"].astype(str)\n",
" else:\n",
" temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n",
" temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n",
"\n",
" df = df.join(temp_df, how='outer')\n",
"\n",
" # Take the latest non-null value for each column\n",
" latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None)\n",
"\n",
" # Convert to a single-row DataFrame\n",
" df = pd.DataFrame([latest_values])\n",
"\n",
" df = df.reindex(sorted(df.columns), axis=1)\n",
" df.rename_axis('Date', inplace=True)\n",
"\n",
" return df\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def getSampleDataFrame(telemetry, keys, time): \n",
" df = pd.DataFrame()\n",
"\n",
" for datapoint in telemetry.keys():\n",
" if datapoint in keys:\n",
" temp_df = pd.DataFrame(telemetry[datapoint])\n",
" temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n",
" temp_df.set_index('ts', inplace=True)\n",
" if datapoint in [\"manual_sample_time\"]:\n",
" temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n",
" print(temp_df)\n",
" elif datapoint in [\"manual_sample_datapoint\", \"manual_sample_lab\", \"manual_sample_location\"]:\n",
" temp_df[\"value\"] = temp_df[\"value\"].astype(str)\n",
" else:\n",
" temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n",
" temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n",
"\n",
" df = df.join(temp_df, how='outer')\n",
"\n",
" df = df.reindex(sorted(df.columns), axis=1)\n",
" df.rename_axis('Date', inplace=True)\n",
"\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 13,
@@ -412,15 +311,6 @@
" )\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": 14,

2
EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml
View File

@@ -5,7 +5,7 @@ codeuri = "/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-d
runtime = "python3.9"
architecture = "x86_64"
handler = "thunderbirdfsreport.lambda_handler"
source_hash = "34ee8699205f989b1c9503c33660f39caa456af608fb961c68f13ce0fdacf18c"
source_hash = "93b500c168f513a7a25cab048f85e186f10d9e9cd5e16384ba4c71c272648df0"
manifest_hash = ""
packagetype = "Zip"
functions = ["ThunderbirdFSReport"]

32
EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py vendored
View File

@@ -1,3 +1,4 @@
import json
import logging, boto3, pytz, os, shutil
from openpyxl.utils.datetime import to_excel
import pandas as pd
@@ -53,7 +54,8 @@ def getTelemetry(rest_client, device, keys, start_ts, end_ts,limit):
def getLatestTelemetry(rest_client, device, keys):
try:
return rest_client.get_latest_timeseries(entity_id=device.id, keys=keys) #entity_type=entity_type,
latest = rest_client.get_latest_timeseries(entity_id=device.id, keys=keys)
return latest
except Exception as e:
logging.error("Something went wrong in getTelemetry")
logging.error(e)
@@ -89,7 +91,7 @@ def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
# Auth with credentials
rest_client.login(username=username, password=password)
# Get customers > get devices under a target customer > get keys for devices > get data for devices
customers = rest_client.get_customers(page_size="100", page="0")
customers = rest_client.get_customers(page_size="500", page="0")
devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)
telemetry = {}
for d in devices.data:
@@ -105,27 +107,6 @@ def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
logging.error(e)
return False
def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
# Creating the REST client object with context manager to get auto token refresh
with RestClientCE(base_url=url) as rest_client:
try:
# Auth with credentials
rest_client.login(username=username, password=password)
# Get customers > get devices under a target customer > get keys for devices > get data for devices
customers = rest_client.get_customers(page_size="100", page="0")
devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)
telemetry = {}
for d in devices.data:
#print(d.name)
device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)
start_ts, end_ts = getTime(timeRequest)
#print(keys)
telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)
return telemetry
except ApiException as e:
logging.error(e)
return False
def getMaxWidth():
label_mapping = {
"Lit 116b Level": "WASTE TANK 1",
@@ -313,7 +294,7 @@ def lambda_handler(event, context):
}
telemetry = getThingsBoardData(url, os.environ["username"], os.environ["password"], "Thunderbird Field Services", time)
if telemetry.get("ACW #1", None):
shutil.copyfile('./ACW Daily Report Template.xlsx', f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx")
# Create a Pandas Excel writer using XlsxWriter as the engine.
@@ -372,14 +353,13 @@ def lambda_handler(event, context):
# Close the Pandas Excel writer and output the Excel file.
writer.close()
# Create an AWS SES client
ses_client = boto3.client('ses', region_name='us-east-1')
s3 = boto3.resource('s3')
BUCKET_NAME = "thingsboard-email-reports"
s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb'))
# Create a new SES resource
# Create an email message
emails = [
"dvaught@thunderbirdfs.com",
"rkamper@thunderbirdfs.com",

BIN
EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/~$ACW Daily Report Template.xlsx vendored
View File

Binary file not shown.

31
EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/test_thunderbirdfsreport.py Normal file
View File

@@ -0,0 +1,31 @@
# thunderbirdfsreport/tests/test_thunderbirdfsreport.py (24-30)
import unittest
from unittest.mock import MagicMock
from thunderbirdfsreport import getDevices
from tb_rest_client.rest_client_ce import Customer, CustomerId, PageDataCustomer, PageDataDevice
class TestGetDevices(unittest.TestCase):
def setUp(self):
self.rest_client = MagicMock()
customer1Id = CustomerId(id="cid1",entity_type="CUSTOMER")
customer2Id = CustomerId(id="cid2",entity_type="CUSTOMER")
self.customer1 = Customer(id=customer1Id, name='customer1', title="customer1")
self.target_customer = Customer(id=customer2Id, name='target_customer', title="customer2")
self.customers = PageDataCustomer(data=[self.customer1, self.target_customer])
def test_get_devices_found_customer(self):
self.rest_client.get_customer_devices.return_value = {'devices': []}
devices = getDevices(self.rest_client, self.customers, 'target_customer')
self.assertEqual(devices, {"devices": []})
self.rest_client.get_customer_devices.assert_called_once_with(customer_id='cid2', page_size=500, page=0)
def test_get_devices_not_found_customer(self):
self.rest_client.get_customer_devices.return_value = {'devices': []}
devices = getDevices(self.rest_client, self.customers, 'non_target_customer')
self.assertEqual(devices, PageDataDevice())
self.assertFalse(self.rest_client.get_customer_devices.called)
if __name__ == '__main__':
unittest.main()

68
EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py
View File

@@ -22,11 +22,18 @@ url = "https://www.enxlekkocloud.com" #"https://hp.henrypump.cloud"
def getDevices(rest_client, customers,target_customer, page=0, pageSize=500):
for c in customers.data:
if c.name == target_customer:
cid = c.id.id
devices = rest_client.get_customer_devices(customer_id=cid, page_size=pageSize, page=page)
return devices #.to_dict()
try:
cid = ""
for c in customers.data:
if c.name == target_customer:
cid = c.id.id
devices = rest_client.get_customer_devices(customer_id=cid, page_size=pageSize, page=page)
return devices #.to_dict()
return PageDataDevice()
except Exception as e:
logging.error(f"Error occured in getDevices: {e}")
return PageDataDevice()
def getDeviceKeys(rest_client, devices,target_device):
@@ -53,7 +60,8 @@ def getTelemetry(rest_client, device, keys, start_ts, end_ts,limit):
def getLatestTelemetry(rest_client, device, keys):
try:
return rest_client.get_latest_timeseries(entity_id=device.id, keys=keys) #entity_type=entity_type,
latest = rest_client.get_latest_timeseries(entity_id=device.id, keys=keys)
return latest
except Exception as e:
logging.error("Something went wrong in getTelemetry")
logging.error(e)
@@ -89,39 +97,20 @@ def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
# Auth with credentials
rest_client.login(username=username, password=password)
# Get customers > get devices under a target customer > get keys for devices > get data for devices
customers = rest_client.get_customers(page_size="100", page="0")
customers = rest_client.get_customers(page_size="500", page="0")
devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)
telemetry = {}
for d in devices.data:
#print(d.name)
device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)
start_ts, end_ts = getTime(timeRequest)
#print(keys)
telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)
latest = getLatestTelemetry(rest_client=rest_client, device=device, keys="manual_well,manual_tfs_lead,manual_copa_lead,manual_job_name")
telemetry[d.name].update(latest)
return telemetry
except ApiException as e:
logging.error(e)
return False
def getThingsBoardData(url, username, password, targetCustomer, timeRequest):
# Creating the REST client object with context manager to get auto token refresh
with RestClientCE(base_url=url) as rest_client:
try:
# Auth with credentials
rest_client.login(username=username, password=password)
# Get customers > get devices under a target customer > get keys for devices > get data for devices
customers = rest_client.get_customers(page_size="100", page="0")
devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)
telemetry = {}
for d in devices.data:
#print(d.name)
device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)
start_ts, end_ts = getTime(timeRequest)
#print(keys)
telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)
return telemetry
if devices.data:
for d in devices.data:
#print(d.name)
device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)
start_ts, end_ts = getTime(timeRequest)
#print(keys)
telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)
latest = getLatestTelemetry(rest_client=rest_client, device=device, keys="manual_well,manual_tfs_lead,manual_copa_lead,manual_job_name")
telemetry[d.name].update(latest)
return telemetry
return {}
except ApiException as e:
logging.error(e)
return False
@@ -313,7 +302,7 @@ def lambda_handler(event, context):
}
telemetry = getThingsBoardData(url, os.environ["username"], os.environ["password"], "Thunderbird Field Services", time)
if telemetry.get("ACW #1", None):
shutil.copyfile('./ACW Daily Report Template.xlsx', f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx")
# Create a Pandas Excel writer using XlsxWriter as the engine.
@@ -372,14 +361,13 @@ def lambda_handler(event, context):
# Close the Pandas Excel writer and output the Excel file.
writer.close()
# Create an AWS SES client
ses_client = boto3.client('ses', region_name='us-east-1')
s3 = boto3.resource('s3')
BUCKET_NAME = "thingsboard-email-reports"
s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb'))
# Create a new SES resource
# Create an email message
emails = [
"dvaught@thunderbirdfs.com",
"rkamper@thunderbirdfs.com",