297 lines
13 KiB
Plaintext
297 lines
13 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"ename": "ModuleNotFoundError",
|
|
"evalue": "No module named 'tb_rest_client'",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
|
"Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtb_rest_client\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrest_client_pe\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;241m*\u001b[39m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtb_rest_client\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrest\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m ApiException\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mjson\u001b[39;00m\u001b[38;5;241m,\u001b[39m \u001b[38;5;21;01mxlsxwriter\u001b[39;00m\u001b[38;5;241m,\u001b[39m \u001b[38;5;21;01mboto3\u001b[39;00m\u001b[38;5;241m,\u001b[39m \u001b[38;5;21;01mos\u001b[39;00m\u001b[38;5;241m,\u001b[39m \u001b[38;5;21;01mtime\u001b[39;00m\n",
|
|
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tb_rest_client'"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from tb_rest_client.rest_client_pe import *\n",
|
|
"from tb_rest_client.rest import ApiException\n",
|
|
"import json, xlsxwriter, boto3, os, time\n",
|
|
"from threading import Lock\n",
|
|
"from datetime import datetime as dt\n",
|
|
"from email.mime.multipart import MIMEMultipart\n",
|
|
"from email.mime.text import MIMEText\n",
|
|
"from email import encoders\n",
|
|
"from email.mime.base import MIMEBase"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Define a rate limiter class\n",
|
|
"class RateLimiter:\n",
|
|
" def __init__(self, max_calls, period):\n",
|
|
" self.max_calls = max_calls\n",
|
|
" self.period = period\n",
|
|
" self.call_times = []\n",
|
|
" self.lock = Lock()\n",
|
|
"\n",
|
|
" def acquire(self):\n",
|
|
" with self.lock:\n",
|
|
" current_time = time.time()\n",
|
|
" # Remove expired calls\n",
|
|
" self.call_times = [t for t in self.call_times if t > current_time - self.period]\n",
|
|
" if len(self.call_times) >= self.max_calls:\n",
|
|
" # Wait for the oldest call to expire\n",
|
|
" time_to_wait = self.period - (current_time - self.call_times[0])\n",
|
|
" time.sleep(time_to_wait)\n",
|
|
" # Register the current call\n",
|
|
" self.call_times.append(time.time())\n",
|
|
"\n",
|
|
"# Initialize a rate limiter\n",
|
|
"rate_limiter = RateLimiter(max_calls=10, period=2) # Adjust `max_calls` and `period` as needed\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"def sort_dict_keys(d):\n",
|
|
" \"\"\"Sorts the keys of all nested dictionaries in a given dictionary.\n",
|
|
"\n",
|
|
" Args:\n",
|
|
" d: The input dictionary.\n",
|
|
"\n",
|
|
" Returns:\n",
|
|
" A new dictionary with sorted keys at each level.\n",
|
|
" \"\"\"\n",
|
|
" sorted_d = {}\n",
|
|
" for k, v in d.items():\n",
|
|
" if isinstance(v, dict):\n",
|
|
" sorted_d[k] = sort_dict_keys(v)\n",
|
|
" else:\n",
|
|
" sorted_d[k] = v\n",
|
|
" return dict(sorted(sorted_d.items()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"url = \"https://hp.henrypump.cloud\"\n",
|
|
"username = \"nmelone@henry-pump.com\"\n",
|
|
"password = \"gzU6$26v42mU%3jDzTJf\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"with RestClientBase(base_url=url) as rest_client:\n",
|
|
" devices = rest_client.get_customer_devices(customer_id=\"ec691940-52e2-11ec-a919-556e8dbef35c\", page=0, page_size=100)\n",
|
|
" print(devices)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Creating Rest Client for ThingsBoard\n",
|
|
"with RestClientPE(base_url=url) as rest_client:\n",
|
|
" try:\n",
|
|
" rest_client.login(username=username, password=password)\n",
|
|
" # Loading Config from file\n",
|
|
" with open('/Users/nico/Documents/GitHub/ThingsBoard/Report Generator/lambda-python3.12/tbreport/config.json') as f:\n",
|
|
" config = json.load(f)\n",
|
|
" reportData = {}\n",
|
|
" reportToList = {}\n",
|
|
"\n",
|
|
" # Loop through each item in config, each item represents a report\n",
|
|
" for report in config:\n",
|
|
" reportToList[report[\"name\"]] = report[\"emails\"]\n",
|
|
" for customer in report[\"customers\"].keys():\n",
|
|
" # Apply rate limiting for API calls\n",
|
|
" rate_limiter.acquire()\n",
|
|
" devices = rest_client.get_customer_devices(customer_id=customer, page=0, page_size=1000)\n",
|
|
" \n",
|
|
" if report[\"filterDevicesIn\"]:\n",
|
|
" devices.data = [device for device in devices.data if device.id.id in report[\"filterDevicesIn\"]]\n",
|
|
" if report[\"filterDevicesOut\"]:\n",
|
|
" devices.data = [device for device in devices.data if device.id.id not in report[\"filterDevicesOut\"]]\n",
|
|
" \n",
|
|
" if not reportData.get(report[\"name\"], None):\n",
|
|
" reportData[report[\"name\"]] = {}\n",
|
|
"\n",
|
|
" for device in devices.data:\n",
|
|
" for deviceType in report[\"customers\"][customer][\"deviceTypes\"]:\n",
|
|
" if device.type == deviceType[\"deviceType\"]:\n",
|
|
" rate_limiter.acquire()\n",
|
|
" keys = rest_client.get_timeseries_keys_v1(device.id)\n",
|
|
" keys = list(filter(lambda x: x in deviceType[\"dataPoints\"], keys))\n",
|
|
" #Check for report customer\n",
|
|
" if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n",
|
|
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n",
|
|
" #Check for device type in config\n",
|
|
" if device.type in list(map(lambda x: x[\"deviceType\"], report[\"customers\"][customer][\"deviceTypes\"])):\n",
|
|
" #Check if deviceType in report\n",
|
|
" if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(device.type, None):\n",
|
|
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][device.type] = {}\n",
|
|
" if keys:\n",
|
|
" rate_limiter.acquire()\n",
|
|
" deviceData = rest_client.get_latest_timeseries(entity_id=device.id, keys=\",\".join(keys))\n",
|
|
" for x in report[\"customers\"][customer][\"deviceTypes\"]:\n",
|
|
" if x[\"deviceType\"] == device.type:\n",
|
|
" labels = x[\"labels\"]\n",
|
|
" labelled_data = {}\n",
|
|
" for k,v in labels.items():\n",
|
|
" labelled_data[v] = {}\n",
|
|
" for k,v in deviceData.items():\n",
|
|
" labelled_data[labels[k]] = v\n",
|
|
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][device.type][device.name] = labelled_data\n",
|
|
" else:\n",
|
|
" reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][device.type][device.name] = {} \n",
|
|
" #Sort Data\n",
|
|
" reportDataSorted = sort_dict_keys(reportData)\n",
|
|
" #print(json.dumps(reportDataSorted,indent=4))\n",
|
|
" except ApiException as e:\n",
|
|
" print(f\"API Exception: {e}\")\n",
|
|
" except Exception as e:\n",
|
|
" print(f\"Other Exception in getting data:\\n{e}\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"print(json.dumps(reportData, indent=4))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"print(json.dumps(reportToList, indent=4))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Create an AWS SES client\n",
|
|
"ses_client = boto3.client('ses', region_name='us-east-1')\n",
|
|
"s3 = boto3.resource('s3')\n",
|
|
"BUCKET_NAME = \"thingsboard-email-reports\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
" # Create a workbook for each report\n",
|
|
"for report_name, report_data in reportDataSorted.items():\n",
|
|
" #will generate an email lower down\n",
|
|
" spreadsheets = []\n",
|
|
" # Create a worksheet for each company\n",
|
|
" for company_name, company_data in report_data.items():\n",
|
|
" workbook = xlsxwriter.Workbook(f\"/Users/nico/Documents/test/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx\",{'strings_to_numbers': True})\n",
|
|
" bold = workbook.add_format({'bold': True})\n",
|
|
" # Create a sheet for each device type\n",
|
|
" for device_type, device_data in company_data.items():\n",
|
|
" worksheet = workbook.add_worksheet(device_type)\n",
|
|
" \n",
|
|
" # Set the header column with device types\n",
|
|
" device_names = list(device_data.keys())\n",
|
|
" worksheet.write_column(1, 0, device_names,bold)\n",
|
|
" #TODO Fix header row and ensure data is put in correct column\n",
|
|
" # Write the data to the sheet\n",
|
|
" for i, (telemetry_name, telemetry_data) in enumerate(device_data.items()):\n",
|
|
" # Set the header row with telemetry names\n",
|
|
" telemetry_names = list(telemetry_data.keys())\n",
|
|
" worksheet.write_row(0, 1, telemetry_names, bold)\n",
|
|
" for j, (data_name, data) in enumerate(telemetry_data.items()):\n",
|
|
" values = [d[\"value\"] for d in data]\n",
|
|
" worksheet.write_row(i + 1, j+ 1, values)\n",
|
|
" worksheet.autofit()\n",
|
|
" workbook.close()\n",
|
|
" spreadsheets.append(workbook)\n",
|
|
" \n",
|
|
" \"\"\"# Store the generated report in S3.\n",
|
|
" s3.Object(BUCKET_NAME, f'{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx').put(Body=open(f\"/Users/nico/Documents/test/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx\", 'rb'))\n",
|
|
" # Create an email message\n",
|
|
" msg = MIMEMultipart()\n",
|
|
" msg['Subject'] = report_name\n",
|
|
" msg['From'] = 'alerts@henry-pump.com'\n",
|
|
" msg['To'] = \", \".join(reportToList[report_name])\n",
|
|
"\n",
|
|
" # Add a text body to the message (optional)\n",
|
|
" body_text = 'Please find the attached spreadsheets.'\n",
|
|
" msg.attach(MIMEText(body_text, 'plain'))\n",
|
|
"\n",
|
|
" # Attach each workbook in the spreadsheets array\n",
|
|
" for spreadsheet in spreadsheets:\n",
|
|
" # Attach the file to the email message\n",
|
|
" attachment = MIMEBase('application', 'octet-stream')\n",
|
|
" attachment.set_payload(open(spreadsheet.filename, \"rb\").read())\n",
|
|
" encoders.encode_base64(attachment)\n",
|
|
" attachment.add_header('Content-Disposition', 'attachment', filename=spreadsheet.filename[5:])\n",
|
|
"\n",
|
|
" msg.attach(attachment)\n",
|
|
" # Send the email using AWS SES\n",
|
|
" response = ses_client.send_raw_email(\n",
|
|
" \n",
|
|
" RawMessage={'Data': msg.as_string()}\n",
|
|
" )\n",
|
|
"\n",
|
|
" print(response)\n",
|
|
" print(spreadsheets)\n",
|
|
" \"\"\""
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "tbreport",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.1"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|