nmelone/ThingsBoard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added monthly reports and sheet name changes

Browse Source
This commit is contained in:
Nico Melone
2025-03-19 11:02:08 -05:00
parent cbdbccc699
commit 83eb9555fb
4 changed files with 201 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
Report Generator/email-reports-v2.ipynb
View File

@@ -181,7 +181,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -194,45 +194,55 @@
" reportToList = {}\n", " reportToList = {}\n",
" # Loop through each item in config, each item represents a report\n", " # Loop through each item in config, each item represents a report\n",
" for report in config:\n", " for report in config:\n",
" reportToList[report[\"name\"]] = report[\"emails\"]\n", " runThisReport = True\n",
" for customer in report[\"customers\"].keys():\n", " if report[\"period\"] == \"Monthly\" and dt.now().day != 1:\n",
" reportDeviceTypes = report[\"customers\"][customer][\"deviceTypes\"]\n", " runThisReport = False\n",
" devices = get_all_customer_devices(token, customer)\n", " if not report[\"emails\"]:\n",
" if report[\"filterDevicesIn\"]:\n", " runThisReport = False\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] in report[\"filterDevicesIn\"]]\n", " if runThisReport:\n",
" if report[\"filterDevicesOut\"]:\n", " reportToList[report[\"name\"]] = report[\"emails\"]\n",
" devices = [device for device in devices if device[\"id\"][\"id\"] not in report[\"filterDevicesOut\"]]\n", " for customer in report[\"customers\"].keys():\n",
" if not reportData.get(report[\"name\"], None):\n", " reportDeviceTypes = report[\"customers\"][customer][\"deviceTypes\"]\n",
" reportData[report[\"name\"]] = {}\n", " devices = get_all_customer_devices(token, customer)\n",
" for device in devices:\n", " if report[\"filterDevicesIn\"]:\n",
" deviceId = device[\"id\"][\"id\"]\n", " devices = [device for device in devices if device[\"id\"][\"id\"] in report[\"filterDevicesIn\"]]\n",
" deviceType = device[\"type\"]\n", " if report[\"filterDevicesOut\"]:\n",
" deviceName = device[\"name\"]\n", " devices = [device for device in devices if device[\"id\"][\"id\"] not in report[\"filterDevicesOut\"]]\n",
" for reportDeviceType in reportDeviceTypes:\n", " if not reportData.get(report[\"name\"], None):\n",
" if reportDeviceType[\"deviceType\"] == deviceType:\n", " reportData[report[\"name\"]] = {}\n",
" keys = get_timeseries_keys(token, deviceId)\n", " for device in devices:\n",
" keys = list(filter(lambda x: x in reportDeviceType[\"dataPoints\"], keys))\n", " deviceId = device[\"id\"][\"id\"]\n",
" #Check for report customer\n", " deviceType = device[\"type\"]\n",
" if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n", " deviceName = device[\"name\"]\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n", " sheetName = deviceType\n",
" #Check for device type in config\n", " for x in reportDeviceTypes:\n",
" if deviceType in list(map(lambda x: x[\"deviceType\"], reportDeviceTypes)):\n", " if x[\"deviceType\"] == deviceType:\n",
" #Check if deviceType in report\n", " sheetName = x[\"sheetName\"]\n",
" if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(deviceType, None):\n", " for reportDeviceType in reportDeviceTypes:\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType] = {}\n", " if reportDeviceType[\"deviceType\"] == deviceType:\n",
" if keys:\n", " keys = get_timeseries_keys(token, deviceId)\n",
" deviceData = get_latest_telemetry(token, deviceId, keys)\n", " keys = list(filter(lambda x: x in reportDeviceType[\"dataPoints\"], keys))\n",
" for x in reportDeviceTypes:\n", " #Check for report customer\n",
" if x[\"deviceType\"] == deviceType:\n", " if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n",
" labels = x[\"labels\"]\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n",
" labelled_data = {}\n", " #Check for device type in config\n",
" for k,v in labels.items():\n", " if deviceType in list(map(lambda x: x[\"deviceType\"], reportDeviceTypes)):\n",
" labelled_data[v] = {}\n", " #Check if deviceType in report\n",
" for k,v in deviceData.items():\n", " if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(sheetName, None):\n",
" labelled_data[labels[k]] = v\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName] = {}\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType][deviceName] = labelled_data\n", " if keys:\n",
" else:\n", " deviceData = get_latest_telemetry(token, deviceId, keys)\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][deviceType][deviceName] = {} \n", " for x in reportDeviceTypes:\n",
" if x[\"deviceType\"] == deviceType:\n",
" labels = x[\"labels\"]\n",
" labelled_data = {}\n",
" for k,v in labels.items():\n",
" labelled_data[v] = {}\n",
" for k,v in deviceData.items():\n",
" labelled_data[labels[k]] = v\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = labelled_data\n",
" else:\n",
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = {} \n",
" #Sort Data\n", " #Sort Data\n",
" reportDataSorted = sort_dict_keys(reportData)\n", " reportDataSorted = sort_dict_keys(reportData)\n",
" #print(json.dumps(reportDataSorted,indent=4))\n", " #print(json.dumps(reportDataSorted,indent=4))\n",
@@ -246,7 +256,79 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{\n",
" \"RRig-Energy-Water-Production-Monthly\": {\n",
" \"RRig-Energy\": {\n",
" \"Water Wells\": {\n",
" \"RRE #1 Water Well #1\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"264738.0\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #2\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"268991.0\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #3\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"264242.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #4\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"262723.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #5\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"269209.5\"\n",
" }\n",
" ]\n",
" },\n",
" \"RRE #1 Water Well #6\": {\n",
" \"Month Volume\": [\n",
" {\n",
" \"ts\": 1742399400000,\n",
" \"value\": \"6203670.5\"\n",
" }\n",
" ]\n",
" }\n",
" }\n",
" }\n",
" }\n",
"}\n"
]
}
],
"source": [
"print(json.dumps(reportDataSorted, indent=4))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -258,7 +340,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 13,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -321,6 +403,13 @@
" print(spreadsheets)\n", " print(spreadsheets)\n",
" \"\"\"" " \"\"\""
] ]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {

31
Report Generator/lambda-python3.12/tbreport/config.json
View File

@@ -9,6 +9,7 @@
"deviceTypes": [ "deviceTypes": [
{ {
"deviceType": "rigpump", "deviceType": "rigpump",
"sheetName": "Rig Pump",
"dataPoints": [ "dataPoints": [
"vfd_current", "vfd_current",
"vfd_frequency" "vfd_frequency"
@@ -20,6 +21,7 @@
}, },
{ {
"deviceType": "cqwatertanks", "deviceType": "cqwatertanks",
"sheetName": "Water Tanks",
"dataPoints": [ "dataPoints": [
"fm_1_flow_rate", "fm_1_flow_rate",
"tank_1_level", "tank_1_level",
@@ -33,6 +35,7 @@
}, },
{ {
"deviceType": "piflow", "deviceType": "piflow",
"sheetName": "Water Wells",
"dataPoints": [ "dataPoints": [
"avgFrequency30Days", "avgFrequency30Days",
"percentRunTime30Days", "percentRunTime30Days",
@@ -48,6 +51,7 @@
}, },
{ {
"deviceType": "advvfdipp", "deviceType": "advvfdipp",
"sheetName": "Adv VFD IPP",
"dataPoints": [ "dataPoints": [
"flowtotalyesterday", "flowtotalyesterday",
"fluidlevel", "fluidlevel",
@@ -68,32 +72,37 @@
}, },
"filterDevicesIn": [], "filterDevicesIn": [],
"filterDevicesOut": [], "filterDevicesOut": [],
"name": "OxyRock-Daily-Report" "name": "OxyRock-Daily-Report",
"period": "Daily"
}, },
{ {
"emails": [ "emails": [
"chris@rrigenergy.com",
"corey@rrigenergy.com",
"parker@rrigenergy.com",
"zack@rrigenergy.com",
"claire@rrigenergy.com"
], ],
"customers": { "customers": {
"81083430-6988-11ec-a919-556e8dbef35c": { "4d4058a0-b25d-11ef-861c-8dbe77c636e1": {
"name": "Henry-Petroleum", "name": "RRig-Energy",
"deviceTypes": [ "deviceTypes": [
{ {
"deviceType": "abbflow", "deviceType": "rr_well",
"sheetName": "Water Wells",
"dataPoints": [ "dataPoints": [
"accumulated_volume" "last_month_total"
], ],
"labels":{ "labels":{
"accumulated_volume": "Accumulated Volume" "last_month_total": "Month Volume"
} }
} }
] ]
} }
}, },
"filterDevicesIn": [ "filterDevicesIn": [],
"0090dbd0-abb0-11ec-90c2-ad8278896f52"
],
"filterDevicesOut": [], "filterDevicesOut": [],
"name": "Henry-Petroleum-Daily-Report" "name": "RRig-Energy-Water-Production-Monthly",
"period": "Monthly"
} }
] ]

88
Report Generator/lambda-python3.12/tbreport/tbreport.py
View File

@@ -134,45 +134,55 @@ def lambda_handler(event, context):
reportToList = {} reportToList = {}
# Loop through each item in config, each item represents a report # Loop through each item in config, each item represents a report
for report in config: for report in config:
reportToList[report["name"]] = report["emails"] runThisReport = True
for customer in report["customers"].keys(): if report["period"] == "Monthly" and dt.now().day != 1:
reportDeviceTypes = report["customers"][customer]["deviceTypes"] runThisReport = False
devices = get_all_customer_devices(token, customer) if not report["emails"]:
if report["filterDevicesIn"]: runThisReport = False
devices = [device for device in devices if device["id"]["id"] in report["filterDevicesIn"]] if runThisReport:
if report["filterDevicesOut"]: reportToList[report["name"]] = report["emails"]
devices = [device for device in devices if device["id"]["id"] not in report["filterDevicesOut"]] for customer in report["customers"].keys():
if not reportData.get(report["name"], None): reportDeviceTypes = report["customers"][customer]["deviceTypes"]
reportData[report["name"]] = {} devices = get_all_customer_devices(token, customer)
for device in devices: if report["filterDevicesIn"]:
deviceId = device["id"]["id"] devices = [device for device in devices if device["id"]["id"] in report["filterDevicesIn"]]
deviceType = device["type"] if report["filterDevicesOut"]:
deviceName = device["name"] devices = [device for device in devices if device["id"]["id"] not in report["filterDevicesOut"]]
for reportDeviceType in reportDeviceTypes: if not reportData.get(report["name"], None):
if reportDeviceType["deviceType"] == deviceType: reportData[report["name"]] = {}
keys = get_timeseries_keys(token, deviceId) for device in devices:
keys = list(filter(lambda x: x in reportDeviceType["dataPoints"], keys)) deviceId = device["id"]["id"]
#Check for report customer deviceType = device["type"]
if not reportData[report["name"]].get(report["customers"][customer]["name"], None): deviceName = device["name"]
reportData[report["name"]][report["customers"][customer]["name"]] = {} sheetName = deviceType
#Check for device type in config for x in reportDeviceTypes:
if deviceType in list(map(lambda x: x["deviceType"], reportDeviceTypes)): if x["deviceType"] == deviceType:
#Check if deviceType in report sheetName = x["sheetName"]
if not reportData[report["name"]][report["customers"][customer]["name"]].get(deviceType, None): for reportDeviceType in reportDeviceTypes:
reportData[report["name"]][report["customers"][customer]["name"]][deviceType] = {} if reportDeviceType["deviceType"] == deviceType:
if keys: keys = get_timeseries_keys(token, deviceId)
deviceData = get_latest_telemetry(token, deviceId, keys) keys = list(filter(lambda x: x in reportDeviceType["dataPoints"], keys))
for x in reportDeviceTypes: #Check for report customer
if x["deviceType"] == deviceType: if not reportData[report["name"]].get(report["customers"][customer]["name"], None):
labels = x["labels"] reportData[report["name"]][report["customers"][customer]["name"]] = {}
labelled_data = {} #Check for device type in config
for k,v in labels.items(): if deviceType in list(map(lambda x: x["deviceType"], reportDeviceTypes)):
labelled_data[v] = {} #Check if deviceType in report
for k,v in deviceData.items(): if not reportData[report["name"]][report["customers"][customer]["name"]].get(sheetName, None):
labelled_data[labels[k]] = v reportData[report["name"]][report["customers"][customer]["name"]][sheetName] = {}
reportData[report["name"]][report["customers"][customer]["name"]][deviceType][deviceName] = labelled_data if keys:
else: deviceData = get_latest_telemetry(token, deviceId, keys)
reportData[report["name"]][report["customers"][customer]["name"]][deviceType][deviceName] = {} for x in reportDeviceTypes:
if x["deviceType"] == deviceType:
labels = x["labels"]
labelled_data = {}
for k,v in labels.items():
labelled_data[v] = {}
for k,v in deviceData.items():
labelled_data[labels[k]] = v
reportData[report["name"]][report["customers"][customer]["name"]][sheetName][deviceName] = labelled_data
else:
reportData[report["name"]][report["customers"][customer]["name"]][sheetName][deviceName] = {}
#Sort Data #Sort Data
reportDataSorted = sort_dict_keys(reportData) reportDataSorted = sort_dict_keys(reportData)
#print(json.dumps(reportDataSorted,indent=4)) #print(json.dumps(reportDataSorted,indent=4))

2
Report Generator/lambda-python3.12/template.yaml
View File

@@ -13,7 +13,7 @@ Resources:
TBReport: TBReport:
Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
Properties: Properties:
MemorySize: 128 MemorySize: 512
Timeout: 300 Timeout: 300
Environment: Environment:
Variables: Variables: