{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import requests, json, time, traceback, boto3, xlsxwriter\n", "from threading import Lock\n", "from datetime import datetime as dt" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "THINGSBOARD_URL = \"https://hp.henrypump.cloud\"\n", "USERNAME = \"nmelone@henry-pump.com\"\n", "PASSWORD = \"gzU6$26v42mU%3jDzTJf\"\n", "CONFIG_PATH = '/Users/nico/Documents/GitHub/ThingsBoard/Report Generator/lambda-python3.12/tbreport/config.json'" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# Define a rate limiter class\n", "class RateLimiter:\n", " def __init__(self, max_calls, period):\n", " self.max_calls = max_calls\n", " self.period = period\n", " self.call_times = []\n", " self.lock = Lock()\n", "\n", " def acquire(self):\n", " with self.lock:\n", " current_time = time.time()\n", " # Remove expired calls\n", " self.call_times = [t for t in self.call_times if t > current_time - self.period]\n", " if len(self.call_times) >= self.max_calls:\n", " # Wait for the oldest call to expire\n", " time_to_wait = self.period - (current_time - self.call_times[0])\n", " time.sleep(time_to_wait)\n", " # Register the current call\n", " self.call_times.append(time.time())\n", "\n", "# Initialize a rate limiter\n", "RATE_LIMITER = RateLimiter(max_calls=10, period=2) # Adjust `max_calls` and `period` as needed" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def sort_dict_keys(d):\n", " \"\"\"Sorts the keys of all nested dictionaries in a given dictionary.\n", "\n", " Args:\n", " d: The input dictionary.\n", "\n", " Returns:\n", " A new dictionary with sorted keys at each level.\n", " \"\"\"\n", " sorted_d = {}\n", " for k, v in d.items():\n", " if isinstance(v, dict):\n", " sorted_d[k] = sort_dict_keys(v)\n", " else:\n", " sorted_d[k] = v\n", " return dict(sorted(sorted_d.items()))" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# Authenticate to get the JWT token\n", "def get_jwt_token():\n", " url = f\"{THINGSBOARD_URL}/api/auth/login\"\n", " payload = {\"username\": USERNAME, \"password\": PASSWORD}\n", " response = requests.post(url, json=payload)\n", " \n", " if response.status_code == 200:\n", " return response.json().get(\"token\")\n", " else:\n", " raise Exception(f\"Authentication failed: {response.text}\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def get_all_customer_devices(jwt_token, customer_id):\n", " \"\"\"Retrieve all devices for a customer, handling pagination.\"\"\"\n", " devices = []\n", " page = 0\n", " page_size = 100 # Adjust if needed\n", " \n", " while True:\n", " url = f\"{THINGSBOARD_URL}/api/customer/{customer_id}/devices?pageSize={page_size}&page={page}\"\n", " headers = {\"X-Authorization\": f\"Bearer {jwt_token}\"}\n", " RATE_LIMITER.acquire()\n", " response = requests.get(url, headers=headers)\n", " \n", " if response.status_code == 200:\n", " data = response.json()\n", " devices.extend(data.get(\"data\", [])) # Add devices from current page\n", " \n", " if page >= data.get(\"totalPages\", 1) - 1:\n", " break # Exit loop if this is the last page\n", " \n", " page += 1 # Move to next page\n", " else:\n", " raise Exception(f\"Failed to get customer devices: {response.text}\")\n", " \n", " return devices\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "def get_timeseries_keys(jwt_token, device_id):\n", " \"\"\"Retrieve available time-series keys (telemetry keys) for a given device ID.\"\"\"\n", " url = f\"{THINGSBOARD_URL}/api/plugins/telemetry/DEVICE/{device_id}/keys/timeseries\"\n", " headers = {\"X-Authorization\": f\"Bearer {jwt_token}\"}\n", " RATE_LIMITER.acquire()\n", " response = requests.get(url, headers=headers)\n", " \n", " if response.status_code == 200:\n", " return response.json() # Returns a list of telemetry keys\n", " else:\n", " raise Exception(f\"Failed to get timeseries keys: {response.text}\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def get_latest_telemetry(jwt_token, device_id, keys=None):\n", " \"\"\"Retrieve the latest telemetry data for a given device ID.\n", " \n", " Args:\n", " jwt_token (str): The authentication token.\n", " device_id (str): The ID of the device.\n", " keys (list, optional): A list of telemetry keys to fetch. Defaults to None (fetch all).\n", " \n", " Returns:\n", " dict: The latest telemetry data for the device.\n", " \"\"\"\n", " if keys:\n", " key_str = \",\".join(keys) # Convert list to comma-separated string\n", " url = f\"{THINGSBOARD_URL}/api/plugins/telemetry/DEVICE/{device_id}/values/timeseries?keys={key_str}\"\n", " else:\n", " url = f\"{THINGSBOARD_URL}/api/plugins/telemetry/DEVICE/{device_id}/values/timeseries\"\n", " \n", " headers = {\"X-Authorization\": f\"Bearer {jwt_token}\"}\n", " RATE_LIMITER.acquire()\n", " response = requests.get(url, headers=headers)\n", " \n", " if response.status_code == 200:\n", " return response.json() # Returns a dictionary with keys and their latest values\n", " else:\n", " raise Exception(f\"Failed to get latest telemetry: {response.text}\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Main execution\n", "try:\n", " token = get_jwt_token()\n", " with open(CONFIG_PATH) as f:\n", " config = json.load(f)\n", " reportData = {}\n", " reportToList = {}\n", " # Loop through each item in config, each item represents a report\n", " for report in config:\n", " runThisReport = True\n", " if report[\"period\"] == \"Monthly\" and dt.now().day != 1:\n", " runThisReport = False\n", " if not report[\"emails\"]:\n", " runThisReport = False\n", " if runThisReport:\n", " reportToList[report[\"name\"]] = report[\"emails\"]\n", " for customer in report[\"customers\"].keys():\n", " reportDeviceTypes = report[\"customers\"][customer][\"deviceTypes\"]\n", " devices = get_all_customer_devices(token, customer)\n", " if report[\"filterDevicesIn\"]:\n", " devices = [device for device in devices if device[\"id\"][\"id\"] in report[\"filterDevicesIn\"]]\n", " if report[\"filterDevicesOut\"]:\n", " devices = [device for device in devices if device[\"id\"][\"id\"] not in report[\"filterDevicesOut\"]]\n", " if not reportData.get(report[\"name\"], None):\n", " reportData[report[\"name\"]] = {}\n", " for device in devices:\n", " deviceId = device[\"id\"][\"id\"]\n", " deviceType = device[\"type\"]\n", " deviceName = device[\"name\"]\n", " sheetName = deviceType\n", " for x in reportDeviceTypes:\n", " if x[\"deviceType\"] == deviceType:\n", " sheetName = x[\"sheetName\"]\n", " for reportDeviceType in reportDeviceTypes:\n", " if reportDeviceType[\"deviceType\"] == deviceType:\n", " keys = get_timeseries_keys(token, deviceId)\n", " keys = list(filter(lambda x: x in reportDeviceType[\"dataPoints\"], keys))\n", " #Check for report customer\n", " if not reportData[report[\"name\"]].get(report[\"customers\"][customer][\"name\"], None):\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]] = {}\n", " #Check for device type in config\n", " if deviceType in list(map(lambda x: x[\"deviceType\"], reportDeviceTypes)):\n", " #Check if deviceType in report\n", " if not reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]].get(sheetName, None):\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName] = {}\n", " if keys:\n", " deviceData = get_latest_telemetry(token, deviceId, keys)\n", " for x in reportDeviceTypes:\n", " if x[\"deviceType\"] == deviceType:\n", " labels = x[\"labels\"]\n", " labelled_data = {}\n", " for k,v in labels.items():\n", " labelled_data[v] = {}\n", " for k,v in deviceData.items():\n", " labelled_data[labels[k]] = v\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = labelled_data\n", " else:\n", " reportData[report[\"name\"]][report[\"customers\"][customer][\"name\"]][sheetName][deviceName] = {} \n", " #Sort Data\n", " reportDataSorted = sort_dict_keys(reportData)\n", " #print(json.dumps(reportDataSorted,indent=4))\n", "except KeyError as ke:\n", " print(\"KeyError:\", ke)\n", " traceback.print_exc()\n", "except Exception as e:\n", " print(\"Error:\", e)\n", " traceback.print_exc()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{\n", " \"RRig-Energy-Water-Production-Monthly\": {\n", " \"RRig-Energy\": {\n", " \"Water Wells\": {\n", " \"RRE #1 Water Well #1\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"264738.0\"\n", " }\n", " ]\n", " },\n", " \"RRE #1 Water Well #2\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"268991.0\"\n", " }\n", " ]\n", " },\n", " \"RRE #1 Water Well #3\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"264242.5\"\n", " }\n", " ]\n", " },\n", " \"RRE #1 Water Well #4\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"262723.5\"\n", " }\n", " ]\n", " },\n", " \"RRE #1 Water Well #5\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"269209.5\"\n", " }\n", " ]\n", " },\n", " \"RRE #1 Water Well #6\": {\n", " \"Month Volume\": [\n", " {\n", " \"ts\": 1742399400000,\n", " \"value\": \"6203670.5\"\n", " }\n", " ]\n", " }\n", " }\n", " }\n", " }\n", "}\n" ] } ], "source": [ "print(json.dumps(reportDataSorted, indent=4))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Create an AWS SES client\n", "ses_client = boto3.client('ses', region_name='us-east-1')\n", "s3 = boto3.resource('s3')\n", "BUCKET_NAME = \"thingsboard-email-reports\"" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ " # Create a workbook for each report\n", "for report_name, report_data in reportDataSorted.items():\n", " #will generate an email lower down\n", " spreadsheets = []\n", " # Create a worksheet for each company\n", " for company_name, company_data in report_data.items():\n", " workbook = xlsxwriter.Workbook(f\"/Users/nico/Documents/test/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx\",{'strings_to_numbers': True})\n", " bold = workbook.add_format({'bold': True})\n", " # Create a sheet for each device type\n", " for device_type, device_data in company_data.items():\n", " worksheet = workbook.add_worksheet(device_type)\n", " \n", " # Set the header column with device types\n", " device_names = list(device_data.keys())\n", " worksheet.write_column(1, 0, device_names,bold)\n", " #TODO Fix header row and ensure data is put in correct column\n", " # Write the data to the sheet\n", " for i, (telemetry_name, telemetry_data) in enumerate(device_data.items()):\n", " # Set the header row with telemetry names\n", " telemetry_names = list(telemetry_data.keys())\n", " worksheet.write_row(0, 1, telemetry_names, bold)\n", " for j, (data_name, data) in enumerate(telemetry_data.items()):\n", " values = [d[\"value\"] for d in data]\n", " worksheet.write_row(i + 1, j+ 1, values)\n", " worksheet.autofit()\n", " workbook.close()\n", " spreadsheets.append(workbook)\n", " \n", " \"\"\"# Store the generated report in S3.\n", " s3.Object(BUCKET_NAME, f'{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx').put(Body=open(f\"/Users/nico/Documents/test/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx\", 'rb'))\n", " # Create an email message\n", " msg = MIMEMultipart()\n", " msg['Subject'] = report_name\n", " msg['From'] = 'alerts@henry-pump.com'\n", " msg['To'] = \", \".join(reportToList[report_name])\n", "\n", " # Add a text body to the message (optional)\n", " body_text = 'Please find the attached spreadsheets.'\n", " msg.attach(MIMEText(body_text, 'plain'))\n", "\n", " # Attach each workbook in the spreadsheets array\n", " for spreadsheet in spreadsheets:\n", " # Attach the file to the email message\n", " attachment = MIMEBase('application', 'octet-stream')\n", " attachment.set_payload(open(spreadsheet.filename, \"rb\").read())\n", " encoders.encode_base64(attachment)\n", " attachment.add_header('Content-Disposition', 'attachment', filename=spreadsheet.filename[5:])\n", "\n", " msg.attach(attachment)\n", " # Send the email using AWS SES\n", " response = ses_client.send_raw_email(\n", " \n", " RawMessage={'Data': msg.as_string()}\n", " )\n", "\n", " print(response)\n", " print(spreadsheets)\n", " \"\"\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "tbreport", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.1" } }, "nbformat": 4, "nbformat_minor": 2 }