nmelone/ThingsBoard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added monthly reports and sheet name changes

Browse Source
This commit is contained in:
Nico Melone
2025-03-19 11:02:08 -05:00
parent cbdbccc699
commit 83eb9555fb
4 changed files with 201 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
Report Generator/email-reports-v2.ipynb
View File

@@ -181,7 +181,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -194,45 +194,55 @@
" reportToList = {}\n",
" # Loop through each item in config, each item represents a report\n",
" for report in config:\n",
" reportToList[report[\"name\"]] = report[\"emails\"]\n",
" for customer in report[\"customers\"].keys():\n",
" reportDeviceTypes = report[\"customers\"][customer][\"deviceTypes\"]\n",
" devices = get_all_customer_devices(token, customer)\n",
" if report[\"filterDevicesIn\"]:\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] in report[\"filterDevicesIn\"]]\n",
" if report[\"filterDevicesOut\"]:\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] not in report[\"filterDevicesOut\"]]\n",
" if not reportData.get(report[\"name\"], None):\n",
" reportData[report[\"name\"]] = {}\n",
" for device in devices:\n",
" deviceId = device[\"id\"][\"id\"]\n",
" deviceType = device[\"type\"]\n",
" deviceName = device[\"name\"]\n",
" for reportDeviceType in reportDeviceTypes:\n",
" if reportDeviceType[\"deviceType\"] == deviceType:\n",
" keys = get_timeseries_keys(token, deviceId)\n",
" keys = list(filter(lambda x: x in reportDeviceType[\"dataPoints\"], keys))\n",
" #Check for report customer\n",
" if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n",
" #Check for device type in config\n",
" if deviceType in list(map(lambda x: x[\"deviceType\"], reportDeviceTypes)):\n",
" #Check if deviceType in report\n",
" if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(deviceType, None):\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType] = {}\n",
" if keys:\n",
" deviceData = get_latest_telemetry(token, deviceId, keys)\n",
" for x in reportDeviceTypes:\n",
" if x[\"deviceType\"] == deviceType:\n",
" labels = x[\"labels\"]\n",
" labelled_data = {}\n",
" for k,v in labels.items():\n",
" labelled_data[v] = {}\n",
" for k,v in deviceData.items():\n",
" labelled_data[labels[k]] = v\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType][deviceName] = labelled_data\n",
" else:\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType][deviceName] = {} \n",
" runThisReport = True\n",
" if report[\"period\"] == \"Monthly\" and dt.now().day != 1:\n",
" runThisReport = False\n",
" if not report[\"emails\"]:\n",
" runThisReport = False\n",
" if runThisReport:\n",
" reportToList[report[\"name\"]] = report[\"emails\"]\n",
" for customer in report[\"customers\"].keys():\n",
" reportDeviceTypes = report[\"customers\"][customer][\"deviceTypes\"]\n",
" devices = get_all_customer_devices(token, customer)\n",
" if report[\"filterDevicesIn\"]:\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] in report[\"filterDevicesIn\"]]\n",
" if report[\"filterDevicesOut\"]:\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] not in report[\"filterDevicesOut\"]]\n",
" if not reportData.get(report[\"name\"], None):\n",
" reportData[report[\"name\"]] = {}\n",
" for device in devices:\n",
" deviceId = device[\"id\"][\"id\"]\n",
" deviceType = device[\"type\"]\n",
" deviceName = device[\"name\"]\n",
" sheetName = deviceType\n",
" for x in reportDeviceTypes:\n",
" if x[\"deviceType\"] == deviceType:\n",
" sheetName = x[\"sheetName\"]\n",
" for reportDeviceType in reportDeviceTypes:\n",
" if reportDeviceType[\"deviceType\"] == deviceType:\n",
" keys = get_timeseries_keys(token, deviceId)\n",
" keys = list(filter(lambda x: x in reportDeviceType[\"dataPoints\"], keys))\n",
" #Check for report customer\n",
" if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n",
" #Check for device type in config\n",
" if deviceType in list(map(lambda x: x[\"deviceType\"], reportDeviceTypes)):\n",
" #Check if deviceType in report\n",
" if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(sheetName, None):\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName] = {}\n",
" if keys:\n",
" deviceData = get_latest_telemetry(token, deviceId, keys)\n",
" for x in reportDeviceTypes:\n",
" if x[\"deviceType\"] == deviceType:\n",
" labels = x[\"labels\"]\n",
" labelled_data = {}\n",
" for k,v in labels.items():\n",
" labelled_data[v] = {}\n",
" for k,v in deviceData.items():\n",
" labelled_data[labels[k]] = v\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = labelled_data\n",
" else:\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = {} \n",
" #Sort Data\n",
" reportDataSorted = sort_dict_keys(reportData)\n",
" #print(json.dumps(reportDataSorted,indent=4))\n",
@@ -246,7 +256,79 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"RRig-Energy-Water-Production-Monthly\": {\n",
" \"RRig-Energy\": {\n",
" \"Water Wells\": {\n",
" \"RRE #1 Water Well #1\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"264738.0\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #2\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"268991.0\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #3\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"264242.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #4\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"262723.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #5\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"269209.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #6\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"6203670.5\"\n",
" }\n",
" ]\n",
" }\n",
" }\n",
" }\n",
" }\n",
"}\n"
]
}
],
"source": [
"print(json.dumps(reportDataSorted, indent=4))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -258,7 +340,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -321,6 +403,13 @@
" print(spreadsheets)\n",
" \"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {

31
Report Generator/lambda-python3.12/tbreport/config.json
View File

@@ -9,6 +9,7 @@
"deviceTypes": [
{
"deviceType": "rigpump",
"sheetName": "Rig Pump",
"dataPoints": [
"vfd_current",
"vfd_frequency"
@@ -20,6 +21,7 @@
},
{
"deviceType": "cqwatertanks",
"sheetName": "Water Tanks",
"dataPoints": [
"fm_1_flow_rate",
"tank_1_level",
@@ -33,6 +35,7 @@
},
{
"deviceType": "piflow",
"sheetName": "Water Wells",
"dataPoints": [
"avgFrequency30Days",
"percentRunTime30Days",
@@ -48,6 +51,7 @@
},
{
"deviceType": "advvfdipp",
"sheetName": "Adv VFD IPP",
"dataPoints": [
"flowtotalyesterday",
"fluidlevel",
@@ -68,32 +72,37 @@
},
"filterDevicesIn": [],
"filterDevicesOut": [],
"name": "OxyRock-Daily-Report"
"name": "OxyRock-Daily-Report",
"period": "Daily"
},
{
"emails": [
"chris@rrigenergy.com",
"corey@rrigenergy.com",
"parker@rrigenergy.com",
"zack@rrigenergy.com",
"claire@rrigenergy.com"
],
"customers": {
"81083430-6988-11ec-a919-556e8dbef35c": {
"name": "Henry-Petroleum",
"4d4058a0-b25d-11ef-861c-8dbe77c636e1": {
"name": "RRig-Energy",
"deviceTypes": [
{
"deviceType": "abbflow",
"deviceType": "rr_well",
"sheetName": "Water Wells",
"dataPoints": [
"accumulated_volume"
"last_month_total"
],
"labels":{
"accumulated_volume": "Accumulated Volume"
"last_month_total": "Month Volume"
}
}
]
}
},
"filterDevicesIn": [
"0090dbd0-abb0-11ec-90c2-ad8278896f52"
],
"filterDevicesIn": [],
"filterDevicesOut": [],
"name": "Henry-Petroleum-Daily-Report"
"name": "RRig-Energy-Water-Production-Monthly",
"period": "Monthly"
}
]

88
Report Generator/lambda-python3.12/tbreport/tbreport.py
View File

@@ -134,45 +134,55 @@ def lambda_handler(event, context):
reportToList = {}
# Loop through each item in config, each item represents a report
for report in config:
reportToList[report["name"]] = report["emails"]
for customer in report["customers"].keys():
reportDeviceTypes = report["customers"][customer]["deviceTypes"]
devices = get_all_customer_devices(token, customer)
if report["filterDevicesIn"]:
devices = [device for device in devices if device["id"]["id"] in report["filterDevicesIn"]]
if report["filterDevicesOut"]:
devices = [device for device in devices if device["id"]["id"] not in report["filterDevicesOut"]]
if not reportData.get(report["name"], None):
reportData[report["name"]] = {}
for device in devices:
deviceId = device["id"]["id"]
deviceType = device["type"]
deviceName = device["name"]
for reportDeviceType in reportDeviceTypes:
if reportDeviceType["deviceType"] == deviceType:
keys = get_timeseries_keys(token, deviceId)
keys = list(filter(lambda x: x in reportDeviceType["dataPoints"], keys))
#Check for report customer
if not reportData[report["name"]].get(report["customers"][customer]["name"], None):
reportData[report["name"]][report["customers"][customer]["name"]] = {}
#Check for device type in config
if deviceType in list(map(lambda x: x["deviceType"], reportDeviceTypes)):
#Check if deviceType in report
if not reportData[report["name"]][report["customers"][customer]["name"]].get(deviceType, None):
reportData[report["name"]][report["customers"][customer]["name"]][deviceType] = {}
if keys:
deviceData = get_latest_telemetry(token, deviceId, keys)
for x in reportDeviceTypes:
if x["deviceType"] == deviceType:
labels = x["labels"]
labelled_data = {}
for k,v in labels.items():
labelled_data[v] = {}
for k,v in deviceData.items():
labelled_data[labels[k]] = v
reportData[report["name"]][report["customers"][customer]["name"]][deviceType][deviceName] = labelled_data
else:
reportData[report["name"]][report["customers"][customer]["name"]][deviceType][deviceName] = {}
runThisReport = True
if report["period"] == "Monthly" and dt.now().day != 1:
runThisReport = False
if not report["emails"]:
runThisReport = False
if runThisReport:
reportToList[report["name"]] = report["emails"]
for customer in report["customers"].keys():
reportDeviceTypes = report["customers"][customer]["deviceTypes"]
devices = get_all_customer_devices(token, customer)
if report["filterDevicesIn"]:
devices = [device for device in devices if device["id"]["id"] in report["filterDevicesIn"]]
if report["filterDevicesOut"]:
devices = [device for device in devices if device["id"]["id"] not in report["filterDevicesOut"]]
if not reportData.get(report["name"], None):
reportData[report["name"]] = {}
for device in devices:
deviceId = device["id"]["id"]
deviceType = device["type"]
deviceName = device["name"]
sheetName = deviceType
for x in reportDeviceTypes:
if x["deviceType"] == deviceType:
sheetName = x["sheetName"]
for reportDeviceType in reportDeviceTypes:
if reportDeviceType["deviceType"] == deviceType:
keys = get_timeseries_keys(token, deviceId)
keys = list(filter(lambda x: x in reportDeviceType["dataPoints"], keys))
#Check for report customer
if not reportData[report["name"]].get(report["customers"][customer]["name"], None):
reportData[report["name"]][report["customers"][customer]["name"]] = {}
#Check for device type in config
if deviceType in list(map(lambda x: x["deviceType"], reportDeviceTypes)):
#Check if deviceType in report
if not reportData[report["name"]][report["customers"][customer]["name"]].get(sheetName, None):
reportData[report["name"]][report["customers"][customer]["name"]][sheetName] = {}
if keys:
deviceData = get_latest_telemetry(token, deviceId, keys)
for x in reportDeviceTypes:
if x["deviceType"] == deviceType:
labels = x["labels"]
labelled_data = {}
for k,v in labels.items():
labelled_data[v] = {}
for k,v in deviceData.items():
labelled_data[labels[k]] = v
reportData[report["name"]][report["customers"][customer]["name"]][sheetName][deviceName] = labelled_data
else:
reportData[report["name"]][report["customers"][customer]["name"]][sheetName][deviceName] = {}
#Sort Data
reportDataSorted = sort_dict_keys(reportData)
#print(json.dumps(reportDataSorted,indent=4))

2
Report Generator/lambda-python3.12/template.yaml
View File

@@ -13,7 +13,7 @@ Resources:
TBReport:
Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
Properties:
MemorySize: 128
MemorySize: 512
Timeout: 300
Environment:
Variables: