{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import logging, json, boto3, pytz, math, os, shutil\n", "from openpyxl.utils import get_column_letter\n", "from openpyxl.utils.datetime import CALENDAR_WINDOWS_1900, to_excel\n", "import pandas as pd\n", "from datetime import datetime as dt\n", "from datetime import timedelta as td\n", "import datetime as dtf\n", "from tb_rest_client.rest_client_ce import *\n", "from tb_rest_client.rest import ApiException\n", "from email.mime.multipart import MIMEMultipart\n", "from email.mime.text import MIMEText\n", "from email import encoders\n", "from email.mime.base import MIMEBase" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "logging.basicConfig(level=logging.DEBUG,\n", " format='%(asctime)s - %(levelname)s - %(module)s - %(lineno)d - %(message)s',\n", " datefmt='%Y-%m-%d %H:%M:%S')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# ThingsBoard REST API URL\n", "url = \"https://www.enxlekkocloud.com\" #\"https://hp.henrypump.cloud\"\n", "# Default Tenant Administrator credentials\n", "username = \"nico.a.melone@gmail.com\" #\"henry.pump.automation@gmail.com\"\n", "password = \"9EE#mqb*b6bXV9hJrPYGm&w3q5Y@3acumvvb5isQ\" #\"Henry Pump @ 2022\"" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def getDevices(rest_client, customers,target_customer, page=0, pageSize=500):\n", " for c in customers.data:\n", " if c.name == target_customer:\n", " cid = c.id.id\n", " devices = rest_client.get_customer_devices(customer_id=cid, page_size=pageSize, page=page)\n", " return devices #.to_dict()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def getDeviceKeys(rest_client, devices,target_device):\n", " try:\n", " for d in devices.data:\n", " if d.name == target_device:\n", " device = d\n", " keys = rest_client.get_timeseries_keys_v1(d.id)\n", " return device, keys, None\n", " return None, None,\"Device Not Found\"\n", " except Exception as e:\n", " logging.error(\"Something went wrong in getDeviceKeys\")\n", " logging.error(e)\n", " return (None, None, e)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def getTelemetry(rest_client, device, keys, start_ts, end_ts,limit):\n", " try:\n", " return rest_client.get_timeseries(entity_id=device.id, keys=keys, start_ts=start_ts, end_ts=end_ts, limit=limit) #entity_type=entity_type, \n", " except Exception as e:\n", " logging.error(\"Something went wrong in getTelemetry\")\n", " logging.error(e)\n", " return False\n", " " ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "def getTime(timeRequest):\n", " start_ts, end_ts = 0,0\n", " if timeRequest[\"type\"] == \"last\":\n", " now = dt.now()\n", " delta = td(days=timeRequest[\"days\"], seconds=timeRequest[\"seconds\"], microseconds=timeRequest[\"microseconds\"], milliseconds=timeRequest[\"milliseconds\"], minutes=timeRequest[\"minutes\"], hours=timeRequest[\"hours\"], weeks=timeRequest[\"weeks\"])\n", " start_ts = str(int(dt.timestamp(now - delta) * 1000))\n", " end_ts = str(int(dt.timestamp(now) * 1000))\n", " elif timeRequest[\"type\"] == \"midnight-midnight\":\n", " timezone = pytz.timezone(timeRequest[\"timezone\"])\n", " today = dtf.date.today()\n", " yesterday_midnight = dtf.datetime.combine(today - dtf.timedelta(days=1), dtf.time())\n", " today_midnight = dtf.datetime.combine(today, dtf.time())\n", " yesterday_midnight = timezone.localize(yesterday_midnight)\n", " today_midnight = timezone.localize(today_midnight)\n", " start_ts = int(yesterday_midnight.timestamp()) * 1000\n", " end_ts = int(today_midnight.timestamp()) * 1000\n", " elif timeRequest[\"type\"] == \"range\":\n", " start_ts = timeRequest[\"ts_start\"]\n", " end_ts = timeRequest[\"ts_end\"]\n", " return (start_ts, end_ts)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def getThingsBoardData(url, username, password, targetCustomer, timeRequest):\n", " # Creating the REST client object with context manager to get auto token refresh\n", " with RestClientCE(base_url=url) as rest_client:\n", " try:\n", " # Auth with credentials\n", " rest_client.login(username=username, password=password)\n", " # Get customers > get devices under a target customer > get keys for devices > get data for devices\n", " customers = rest_client.get_customers(page_size=\"100\", page=\"0\")\n", " devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)\n", " telemetry = {}\n", " for d in devices.data:\n", " #print(d.name)\n", " device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)\n", " start_ts, end_ts = getTime(timeRequest)\n", " #print(keys)\n", " telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)\n", " return telemetry\n", " except ApiException as e:\n", " logging.error(e)\n", " return False\n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def getMaxWidth():\n", " label_mapping = {\n", " \"Lit 116b Level\": \"WASTE TANK 1\",\n", " \"Lit 116a Level\": \"WASTE TANK 2\",\n", " \"Fit 100 Flow Rate\": \"INLET FLOW RATE\",\n", " \"Fit 109b Flow Rate\": \"SALES FLOW RATE\",\n", " \"Outlet Turbidity Temp\": \"OUTLET TURBIDITY TEMP\",\n", " \"Outlet Orp Temp\": \"OUTLET ORP TEMP\",\n", " \"Inlet Turbidity Temp\": \"INLET TURBIDITY TEMP\",\n", " \"Inlet Ph Temp\": \"INLET PH TEMP\",\n", " \"Ait 102b H2s\": \"INLET H₂S\",\n", " \"At 109b H2s\": \"OUTLET H₂S\",\n", " \"At 109c Oil In Water\": \"OUTLET OIL IN WATER\",\n", " \"Ait 102a Turbitity\": \"INLET TURBIDITY\",\n", " \"At 109a Turbidity\": \"OUTLET TURBIDITY\",\n", " \"At 109e Orp\": \"OUTLET ORP\"\n", " }\n", " width = 0\n", " for key,value in label_mapping.items():\n", " if(len(value) > width):\n", " width = len(value)\n", "\n", " return width" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def formatColumnName(telemetryName):\n", " name = \" \".join([x.capitalize() for x in telemetryName.split(\"_\")])\n", " label_mapping = {\n", " \"Lit 116b Level\": \"WASTE TANK 1\",\n", " \"Lit 116a Level\": \"WASTE TANK 2\",\n", " \"Fit 100 Flow Rate\": \"INLET FLOW RATE\",\n", " \"Fit 109b Flow Rate\": \"SALES FLOW RATE\",\n", " \"Outlet Turbidity Temp\": \"OUTLET TURBIDITY TEMP\",\n", " \"Outlet Orp Temp\": \"OUTLET ORP TEMP\",\n", " \"Inlet Turbidity Temp\": \"INLET TURBIDITY TEMP\",\n", " \"Inlet Ph Temp\": \"INLET PH TEMP\",\n", " \"Ait 102b H2s\": \"INLET H₂S\",\n", " \"At 109b H2s\": \"OUTLET H₂S\",\n", " \"At 109c Oil In Water\": \"OUTLET OIL IN WATER\",\n", " \"Ait 102a Turbitity\": \"INLET TURBIDITY\",\n", " \"At 109a Turbidity\": \"OUTLET TURBIDITY\",\n", " \"At 109e Orp\": \"OUTLET ORP\"\n", " }\n", " return label_mapping.get(name)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "def formatChartName(telemetryName):\n", " return \" \".join([x.upper() for x in telemetryName.split(\"_\")])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "def getDataFrame(telemetry, ignore_keys, time): \n", " df = pd.DataFrame()\n", " #for location in telemetry.keys():\n", " # Iterate through each datapoint within each location\n", " for datapoint in telemetry.keys():\n", " # Convert the datapoint list of dictionaries to a DataFrame\n", " if datapoint not in ignore_keys:\n", " temp_df = pd.DataFrame(telemetry[datapoint])\n", " temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", " # Set 'ts' as the index\n", " temp_df.set_index('ts', inplace=True)\n", " temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n", " # Rename 'value' column to the name of the datapoint\n", " temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n", " \n", " # Join the temp_df to the main DataFrame\n", " df = df.join(temp_df, how='outer')\n", " df.ffill()\n", " #df = df.fillna(method='ffill', limit=2)\n", " # Rename index to 'Date'\n", " df.rename_axis('Date', inplace=True)\n", " return df" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "def get_last_data_row(ws):\n", " # Start from the bottom row and work up to find the last row with data\n", " for row in range(ws.max_row, 0, -1):\n", " if any(cell.value is not None for cell in ws[row]):\n", " return row\n", " return 0 # If no data is found, return 0" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "time = {\n", " \"type\": \"last\",\n", " \"days\":3,\n", " \"seconds\":0,\n", " \"microseconds\":0,\n", " \"milliseconds\":0,\n", " \"minutes\":0,\n", " \"hours\":0,\n", " \"weeks\":0,\n", " \"timezone\": \"US/Central\"\n", " }\n", "time = {\n", " \"type\": \"midnight-midnight\",\n", " \"timezone\": \"US/Alaska\" \n", "}\n", "time = {\n", " \"type\": \"range\",\n", " \"timezone\": \"US/Alaska\" ,\n", " \"ts_start\": 1728115200000,\n", " \"ts_end\": 1728201600000\n", "}\n", "telemetry = getThingsBoardData(url, username, password, \"Thunderbird Field Services\", time)" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# Create a Pandas Excel writer using XlsxWriter as the engine.\n", "shutil.copyfile('/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-daily-report/ACW Daily Report Template.xlsx', f\"/Users/nico/Documents/test/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\")\n", "writer = pd.ExcelWriter(\n", " f\"/Users/nico/Documents/test/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\", \n", " engine=\"openpyxl\",\n", " datetime_format=\"yyyy-mm-dd hh:mm:ss\",\n", " date_format=\"yyyy-mm-dd\",\n", " #engine_kwargs={'options': {'strings_to_numbers': True}},\n", " mode=\"a\",\n", " if_sheet_exists=\"overlay\")\n", "reportsheet = writer.book.worksheets[0]\n", "\n", "ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm', 'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate', 'at_109d_o2_in_water', 'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm']\n", "\n", "#Create a Sheet for each Device\n", "for device in telemetry.keys():\n", " df = getDataFrame(telemetry[device], ignore_keys, time)\n", " \n", " # Write the dataframe data to XlsxWriter. Turn off the default header and\n", " # index and skip one row to allow us to insert a user defined header.\n", " df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format=\"%.2f\")\n", "\n", " # Get the xlsxwriter workbook and worksheet objects.\n", " workbook = writer.book\n", " worksheet = writer.sheets[device]\n", " for row in worksheet.iter_rows(min_row=2, max_col=1):\n", " for cell in row:\n", " cell.number_format = 'yyyy-mm-dd hh:mm:ss'\n", "\n", "#Getting the data sheet for ACW #1 to access date range actually available\n", "datasheet = writer.book.worksheets[1]\n", "datetime_min = datasheet[\"A2\"].value\n", "last_data_row = get_last_data_row(datasheet)\n", "datetime_max = datasheet[f\"A{last_data_row}\"].value\n", "#Convert to excel number\n", "datetime_min = to_excel(datetime_min)\n", "datetime_max = round(to_excel(datetime_max))\n", "#Change the range of the chart\n", "chart = reportsheet._charts[0]\n", "chart.x_axis.scaling.min = datetime_min\n", "chart.x_axis.scaling.max = datetime_max\n", "chart.x_axis.number_format = 'hh:mm'\n", "reportsheet[\"B3\"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y')\n", "reportsheet[\"B4\"] = \"Test Well Name\"\n", "reportsheet[\"B5\"] = \"Test Well Lead\"\n", "reportsheet[\"B6\"] = \"Test COPA Lead\"\n", "reportsheet[\"B7\"] = \"Test Job Name\"\n", "\n", "reportsheet[\"B10\"]= \"Test Events or Spills\"\n", "reportsheet[\"B12\"] = \"Test Issues\"\n", "\n", "reportsheet[\"E4\"] = \"A very large summary test text to put into perspective the amount\\n of work that is having to be done to this sheet\\n for this to work\"\n", "# Close the Pandas Excel writer and output the Excel file.\n", "writer.close()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Create an AWS SES client\n", "ses_client = boto3.client('ses', region_name='us-east-1')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Create an email message\n", "emails = [\"nmelone@henry-pump.com\"]\n", "\"\"\"emails = [\n", " \"dvaught@thunderbirdfs.com\", \n", " \"rkamper@thunderbirdfs.com\", \n", " \"john.griffin@acaciaes.com\", \n", " \"Joshua.Fine@fineelectricalservices2018.com\"\n", "]\"\"\"\n", "msg = MIMEMultipart()\n", "msg['Subject'] = \"Thunderbird Field Services\"\n", "msg['From'] = 'alerts@henry-pump.com'\n", "msg['To'] = \", \".join(emails)\n", "\n", "# Add a text body to the message (optional)\n", "body_text = 'Please find the attached spreadsheets.'\n", "msg.attach(MIMEText(body_text, 'plain'))\n", "\n", "\n", "# Attach the file to the email message\n", "attachment = MIMEBase('application', 'octet-stream')\n", "attachment.set_payload(open(f\"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\", \"rb\").read())\n", "encoders.encode_base64(attachment)\n", "attachment.add_header('Content-Disposition', 'attachment', filename=f\"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\")\n", "msg.attach(attachment)\n", "\n", "# Send the email using AWS SES\n", "response = ses_client.send_raw_email(\n", " \n", " RawMessage={'Data': msg.as_string()}\n", ")" ] } ], "metadata": { "kernelspec": { "display_name": "tbreport", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.4" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }