diff --git a/EKKO Reports/buildReportEKKO2.ipynb b/EKKO Reports/buildReportEKKO2.ipynb index 2d7c17a..32e87b4 100644 --- a/EKKO Reports/buildReportEKKO2.ipynb +++ b/EKKO Reports/buildReportEKKO2.ipynb @@ -136,15 +136,15 @@ " # Auth with credentials\n", " rest_client.login(username=username, password=password)\n", " # Get customers > get devices under a target customer > get keys for devices > get data for devices\n", - " customers = rest_client.get_customers(page_size=\"100\", page=\"0\")\n", + " customers = rest_client.get_customers(page_size=\"500\", page=\"0\")\n", " devices = getDevices(rest_client=rest_client, customers=customers, target_customer=targetCustomer)\n", " telemetry = {}\n", " for d in devices.data:\n", " #print(d.name)\n", " device, keys, err = getDeviceKeys(rest_client=rest_client, devices=devices, target_device=d.name)\n", " start_ts, end_ts = getTime(timeRequest)\n", - " #print(keys)\n", - " telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=25000)\n", + " print(len(keys), keys)\n", + " telemetry[d.name] = getTelemetry(rest_client=rest_client, device=device, keys=','.join(keys), start_ts=start_ts, end_ts=end_ts, limit=50000)\n", " return telemetry\n", " except ApiException as e:\n", " logging.error(e)\n", @@ -201,12 +201,13 @@ " \"Inlet Ph Temp\": \"INLET PH TEMP\",\n", " \"Ait 102b H2s\": \"INLET H₂S\",\n", " \"At 109b H2s\": \"OUTLET H₂S\",\n", - " \"At 109c Oil In Water\": \"OUTLET OIL IN WATER\",\n", + " \"At 109c Oil In Water\": \"OUTLET DENSITY\",\n", " \"Ait 102a Turbitity\": \"INLET TURBIDITY\",\n", " \"At 109a Turbidity\": \"OUTLET TURBIDITY\",\n", - " \"At 109e Orp\": \"OUTLET ORP\"\n", + " \"At 109e Orp\": \"OUTLET ORP\",\n", + " \"Ait 102d Oil In Water\": \"INLET DENSITY\"\n", " }\n", - " return label_mapping.get(name)" + " return label_mapping.get(name, name)" ] }, { @@ -221,17 +222,17 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def getDataFrame(telemetry, ignore_keys, time): \n", + "def getDataFrame(telemetry, keys, time): \n", " df = pd.DataFrame()\n", " #for location in telemetry.keys():\n", " # Iterate through each datapoint within each location\n", " for datapoint in telemetry.keys():\n", " # Convert the datapoint list of dictionaries to a DataFrame\n", - " if datapoint not in ignore_keys:\n", + " if datapoint in keys:\n", " temp_df = pd.DataFrame(telemetry[datapoint])\n", " temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", " # Set 'ts' as the index\n", @@ -244,11 +245,146 @@ " df = df.join(temp_df, how='outer')\n", " df.ffill()\n", " #df = df.fillna(method='ffill', limit=2)\n", - " # Rename index to 'Date'\n", + " df = df.reindex(sorted(df.columns), axis=1)\n", + " # Rename index to 'Date'\n", " df.rename_axis('Date', inplace=True)\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def getManualDataFrame(telemetry, keys, time): \n", + " df = pd.DataFrame()\n", + " for key in keys:\n", + " if key not in telemetry.keys():\n", + " telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}]\n", + " for datapoint in telemetry.keys():\n", + " if datapoint in keys:\n", + " temp_df = pd.DataFrame(telemetry[datapoint])\n", + " temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " temp_df.set_index('ts', inplace=True)\n", + " if datapoint in [\"manual_next_pigging_scheduled\"]:\n", + " temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " print(temp_df)\n", + " elif datapoint in [\"manual_equipment_description\",\"manual_issues_concerns\"]:\n", + " temp_df[\"value\"] = temp_df[\"value\"].astype(str)\n", + " else:\n", + " temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n", + " temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n", + "\n", + " df = df.join(temp_df, how='outer')\n", + "\n", + " # Take the latest non-null value for each column\n", + " latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None)\n", + "\n", + " # Convert to a single-row DataFrame\n", + " df = pd.DataFrame([latest_values])\n", + "\n", + " df = df.reindex(sorted(df.columns), axis=1)\n", + " df.rename_axis('Date', inplace=True)\n", + "\n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def getSampleDataFrame(telemetry, keys, time): \n", + " df = pd.DataFrame()\n", + "\n", + " for datapoint in telemetry.keys():\n", + " if datapoint in keys:\n", + " temp_df = pd.DataFrame(telemetry[datapoint])\n", + " temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " temp_df.set_index('ts', inplace=True)\n", + " if datapoint in [\"manual_sample_time\"]:\n", + " temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " print(temp_df)\n", + " elif datapoint in [\"manual_sample_datapoint\", \"manual_sample_lab\", \"manual_sample_location\"]:\n", + " temp_df[\"value\"] = temp_df[\"value\"].astype(str)\n", + " else:\n", + " temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n", + " temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n", + "\n", + " df = df.join(temp_df, how='outer')\n", + "\n", + " df = df.reindex(sorted(df.columns), axis=1)\n", + " df.rename_axis('Date', inplace=True)\n", + "\n", " return df" ] }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def process_dataframe(telemetry, keys, time, special_handling=None, latest_only=False): \n", + " df = pd.DataFrame()\n", + "\n", + " # If latest_only is True, ensure missing keys are initialized\n", + " if latest_only:\n", + " for key in keys:\n", + " if key not in telemetry:\n", + " telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}]\n", + "\n", + " for datapoint in telemetry.keys():\n", + " if datapoint in keys:\n", + " temp_df = pd.DataFrame(telemetry[datapoint])\n", + " temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " temp_df.set_index('ts', inplace=True)\n", + "\n", + " if special_handling and datapoint in special_handling.get(\"datetime\", []):\n", + " temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + " elif special_handling and datapoint in special_handling.get(\"string\", []):\n", + " temp_df[\"value\"] = temp_df[\"value\"].astype(str)\n", + " else:\n", + " temp_df[\"value\"] = pd.to_numeric(temp_df[\"value\"], errors=\"coerce\")\n", + "\n", + " temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)\n", + " df = df.join(temp_df, how='outer')\n", + "\n", + " if latest_only:\n", + " latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None)\n", + " df = pd.DataFrame([latest_values])\n", + "\n", + " df = df.reindex(sorted(df.columns), axis=1)\n", + " df.rename_axis('Date', inplace=True)\n", + "\n", + " return df\n", + "\n", + "# Usage\n", + "def getDataFrame(telemetry, keys, time):\n", + " return process_dataframe(telemetry, keys, time)\n", + "\n", + "def getManualDataFrame(telemetry, keys, time):\n", + " return process_dataframe(\n", + " telemetry, keys, time, \n", + " special_handling={\n", + " \"datetime\": [\"manual_next_pigging_scheduled\"],\n", + " \"string\": [\"manual_equipment_description\", \"manual_issues_concerns\"]\n", + " }, \n", + " latest_only=True\n", + " )\n", + "\n", + "def getSampleDataFrame(telemetry, keys, time):\n", + " return process_dataframe(\n", + " telemetry, keys, time, \n", + " special_handling={\n", + " \"datetime\": [\"manual_sample_time\"],\n", + " \"string\": [\"manual_sample_datapoint\", \"manual_sample_lab\", \"manual_sample_location\"]\n", + " }\n", + " )\n" + ] + }, { "cell_type": "code", "execution_count": 13, @@ -265,77 +401,81 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "261 ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm', 'ait_102a_turbitity', 'ait_102b_h2s', 'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109d_o2_in_water', 'at_109e_orp', 'fit_109a_flow_rate', 'fit_100_flow_rate', 'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109b_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_level', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_level', 'lit_116a_hihi_alm', 'lit_116a_hi_alm', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'n2_run_time_lifetime', 'compressor_lifetime_run_hours', 'ef_vfd_1_fault_description', 'ef_vfd_1_n2_frequency', 'ef_vfd_2_running', 'ef_vfd_2_n2_hand_spt', 'ef_vfd_2_n2_frequency', 'ef_vfd_2_n2_faulted_alm', 'ef_vfd_2_n2_auto_room_spt', 'ef_vfd_2_n2_auto', 'ef_vfd_2_fault_description', 'ef_vfd_1_running', 'ef_vfd_1_n2_hand_spt', 'ef_vfd_1_n2_faulted_alm', 'ef_vfd_1_n2_auto_room_spt', 'ef_vfd_1_n2_auto', 'n2_inlet_dew_point', 'manual_water_to_tanks_time', 'manual_sample_time', 'manual_sample_value', 'manual_sample_lab', 'manual_sample_datapoint', 'manual_sample_location', 'manual_equipment_description', 'manual_water_events', 'manual_diverted_water_time', 'manual_standby_time', 'manual_equipment_time', 'manual_unit_uptime', 'manual_water_events_time', 'manual_clean_water_sold_per_job', 'manual_skim_oil_discharged_per_job', 'manual_h202_on_hand', 'manual_coagulant_on_hand', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_cartridge_filter_changes', 'manual_bag_filter_changes', 'outlet_ph', 'lit_110a_level', 'fit_106b_yesterday', 'fit_106b_today', 'fit_106b_this_month', 'fit_106b_lifetime', 'fit_106b_last_month', 'fit_106b_job', 'fcv_101a_position', 'outlet_o2']\n" + ] + } + ], "source": [ - "time = {\n", + "\"\"\"time = {\n", " \"type\": \"last\",\n", - " \"days\":3,\n", + " \"days\":1,\n", " \"seconds\":0,\n", " \"microseconds\":0,\n", " \"milliseconds\":0,\n", " \"minutes\":0,\n", " \"hours\":0,\n", " \"weeks\":0,\n", - " \"timezone\": \"US/Central\"\n", + " \"timezone\": \"US/Alaska\"\n", " }\n", + " \"\"\"\n", "time = {\n", " \"type\": \"midnight-midnight\",\n", " \"timezone\": \"US/Alaska\" \n", "}\n", - "time = {\n", + "\"\"\"\n", + " time = {\n", " \"type\": \"range\",\n", " \"timezone\": \"US/Alaska\" ,\n", " \"ts_start\": 1728115200000,\n", " \"ts_end\": 1728201600000\n", - "}\n", + "} \"\"\"\n", "telemetry = getThingsBoardData(url, username, password, \"Thunderbird Field Services\", time)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(telemetry[\"ACW #1\"].keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "telemetry['ACW #1']['manual_clean_water_sold_per_job']" + ] + }, + { + "cell_type": "code", + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'sRGB' 41 1\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'gAMA' 54 4\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 70 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 91 17733\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 41 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'iTXt' 62 1489\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 1563 8215\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'sRGB' 41 1\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'eXIf' 54 132\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 198 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 219 16384\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'sRGB' 41 1\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'gAMA' 54 4\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 70 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 91 17733\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 41 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'iTXt' 62 1489\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 1563 8215\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IHDR' 16 13\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'sRGB' 41 1\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'eXIf' 54 132\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'pHYs' 198 9\n", - "2025-01-21 16:40:11 - DEBUG - PngImagePlugin - 198 - STREAM b'IDAT' 219 16384\n" + "/var/folders/dd/glmkqm595_n53prmxzd7qh980000gn/T/ipykernel_98236/3556243203.py:17: FutureWarning: The behavior of 'to_datetime' with 'unit' when parsing strings is deprecated. In a future version, strings will be parsed as datetime strings, matching the behavior without a 'unit'. To retain the old behavior, explicitly cast ints or floats to numeric type before calling to_datetime.\n", + " temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n", + "/var/folders/dd/glmkqm595_n53prmxzd7qh980000gn/T/ipykernel_98236/3556243203.py:17: FutureWarning: The behavior of 'to_datetime' with 'unit' when parsing strings is deprecated. In a future version, strings will be parsed as datetime strings, matching the behavior without a 'unit'. To retain the old behavior, explicitly cast ints or floats to numeric type before calling to_datetime.\n", + " temp_df[\"value\"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time[\"timezone\"]).dt.tz_localize(None)\n" ] } ], "source": [ "# Create a Pandas Excel writer using XlsxWriter as the engine.\n", - "shutil.copyfile('/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-daily-report/ACW Daily Report Template.xlsx', f\"/Users/nico/Documents/test/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\")\n", + "shutil.copyfile('/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/ACW Daily Report Template.xlsx', f\"/Users/nico/Documents/test/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\")\n", "writer = pd.ExcelWriter(\n", " f\"/Users/nico/Documents/test/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx\", \n", " engine=\"openpyxl\",\n", @@ -346,16 +486,19 @@ " if_sheet_exists=\"overlay\")\n", "reportsheet = writer.book.worksheets[0]\n", "\n", - "ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm', 'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate', 'at_109d_o2_in_water', 'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm']\n", - "\n", + "keys = ['ait_102a_turbitity','ait_102b_h2s', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109e_orp', 'fit_100_flow_rate', 'fit_109b_flow_rate', 'lit_116b_level', 'lit_116a_level', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'ait_102d_oil_in_water', 'outlet_ph']\n", + "manual_keys = ['manual_bag_filter_changes', 'manual_cartridge_filter_changes', 'manual_clean_water_sold_per_job', 'manual_coagulant_on_hand', 'manual_diverted_water_time', 'manual_equipment_description', 'manual_equipment_time', 'manual_h202_on_hand', 'manual_issues_concerns', 'manual_next_pigging_scheduled', 'manual_skim_oil_discharged_per_job', 'manual_standby_time', 'manual_unit_uptime', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_water_events', 'manual_water_events_time', 'manual_water_to_tanks_time']\n", + "sample_keys = ['manual_sample_datapoint', 'manual_sample_lab', 'manual_sample_location', 'manual_sample_time', 'manual_sample_value'] \n", "#Create a Sheet for each Device\n", "for device in telemetry.keys():\n", - " df = getDataFrame(telemetry[device], ignore_keys, time)\n", - " \n", + " df = getDataFrame(telemetry[device], keys, time)\n", + " dfm = getManualDataFrame(telemetry[device], manual_keys, time)\n", + " dfs = getSampleDataFrame(telemetry[device], sample_keys, time)\n", " # Write the dataframe data to XlsxWriter. Turn off the default header and\n", " # index and skip one row to allow us to insert a user defined header.\n", " df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format=\"%.2f\")\n", - "\n", + " dfm.to_excel(writer, sheet_name=device+\" Manual Entry\", startrow=0, header=True, index=True, float_format=\"%.2f\")\n", + " dfs.to_excel(writer, sheet_name=device+\" Manual Samples\", startrow=0, header=True, index=True, float_format=\"%.2f\")\n", " # Get the xlsxwriter workbook and worksheet objects.\n", " workbook = writer.book\n", " worksheet = writer.sheets[device]\n", @@ -371,21 +514,17 @@ "#Convert to excel number\n", "datetime_min = to_excel(datetime_min)\n", "datetime_max = round(to_excel(datetime_max))\n", - "#Change the range of the chart\n", - "chart = reportsheet._charts[0]\n", - "chart.x_axis.scaling.min = datetime_min\n", - "chart.x_axis.scaling.max = datetime_max\n", - "chart.x_axis.number_format = 'hh:mm'\n", + "for chart in reportsheet._charts:\n", + " #Change the range of the chart\n", + " #chart = reportsheet._charts[0]\n", + " chart.x_axis.scaling.min = datetime_min\n", + " chart.x_axis.scaling.max = datetime_max\n", + " chart.x_axis.number_format = 'hh:mm'\n", + "\n", "reportsheet[\"B4\"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y')\n", + "\"\"\"\n", "reportsheet[\"B5\"] = \"Test Well Name\"\n", - "reportsheet[\"B6\"] = \"Test Well Lead\"\n", - "reportsheet[\"B7\"] = \"Test COPA Lead\"\n", - "reportsheet[\"B8\"] = \"Test Job Name\"\n", - "\n", - "reportsheet[\"B11\"]= \"Test Events or Spills\"\n", - "reportsheet[\"B13\"] = \"Test Issues\"\n", - "\n", - "reportsheet[\"E5\"] = \"A very large summary test text to put into perspective the amount\\n of work that is having to be done to this sheet\\n for this to work\"\n", + "\"\"\"\n", "# Close the Pandas Excel writer and output the Excel file.\n", "writer.close()\n" ] @@ -396,7 +535,7 @@ "metadata": {}, "outputs": [], "source": [ - "df" + "dfm" ] }, { @@ -464,7 +603,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.13.1" }, "orig_nbformat": 4 }, diff --git a/EKKO Reports/test.html b/EKKO Reports/test.html new file mode 100644 index 0000000..8294f92 --- /dev/null +++ b/EKKO Reports/test.html @@ -0,0 +1,40 @@ +
+
+
+
+
+
+
+
+ +
+ +
+
+
\ No newline at end of file diff --git a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml index 0cd5ff8..1e42cb2 100644 --- a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml +++ b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml @@ -5,7 +5,7 @@ codeuri = "/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-d runtime = "python3.9" architecture = "x86_64" handler = "thunderbirdfsreport.lambda_handler" -source_hash = "470cf4aeb6b1a4872891b288a63843be543d3c07583f5f99de6198845430a4cc" +source_hash = "96f09690c748fbb53cc41f4396ed0d83dfb501a3cb2710f29c4149645fc9c9fb" manifest_hash = "" packagetype = "Zip" functions = ["ThunderbirdFSReport"] diff --git a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/ACW Daily Report Template.xlsx b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/ACW Daily Report Template.xlsx index 84eec46..424fb4b 100644 Binary files a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/ACW Daily Report Template.xlsx and b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/ACW Daily Report Template.xlsx differ diff --git a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py index 8ac01c3..f37618e 100644 --- a/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py +++ b/EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py @@ -136,41 +136,77 @@ def formatColumnName(telemetryName): "Inlet Ph Temp": "INLET PH TEMP", "Ait 102b H2s": "INLET H₂S", "At 109b H2s": "OUTLET H₂S", - "At 109c Oil In Water": "OUTLET OIL IN WATER", + "At 109c Oil In Water": "OUTLET DENSITY", "Ait 102a Turbitity": "INLET TURBIDITY", "At 109a Turbidity": "OUTLET TURBIDITY", - "At 109e Orp": "OUTLET ORP" + "At 109e Orp": "OUTLET ORP", + "Ait 102d Oil In Water": "INLET DENSITY" } - return label_mapping.get(name) + return label_mapping.get(name, name) def formatChartName(telemetryName): return " ".join([x.upper() for x in telemetryName.split("_")]) -def getDataFrame(telemetry, ignore_keys, time): +def process_dataframe(telemetry, keys, time, special_handling=None, latest_only=False): df = pd.DataFrame() - #for location in telemetry.keys(): - # Iterate through each datapoint within each location + + # If latest_only is True, ensure missing keys are initialized + if latest_only: + for key in keys: + if key not in telemetry: + telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}] + for datapoint in telemetry.keys(): - # Convert the datapoint list of dictionaries to a DataFrame - if datapoint not in ignore_keys: + if datapoint in keys: temp_df = pd.DataFrame(telemetry[datapoint]) temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None) - # Set 'ts' as the index temp_df.set_index('ts', inplace=True) - temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce") - # Rename 'value' column to the name of the datapoint + + if special_handling and datapoint in special_handling.get("datetime", []): + temp_df["value"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None) + elif special_handling and datapoint in special_handling.get("string", []): + temp_df["value"] = temp_df["value"].astype(str) + else: + temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce") + temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True) - - # Join the temp_df to the main DataFrame df = df.join(temp_df, how='outer') - df.ffill() - #df = df.fillna(method='ffill', limit=2) - # Rename index to 'Date' + + if latest_only: + latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None) + df = pd.DataFrame([latest_values]) + + df = df.reindex(sorted(df.columns), axis=1) df.rename_axis('Date', inplace=True) + return df +# Usage +def getDataFrame(telemetry, keys, time): + return process_dataframe(telemetry, keys, time) + +def getManualDataFrame(telemetry, keys, time): + return process_dataframe( + telemetry, keys, time, + special_handling={ + "datetime": ["manual_next_pigging_scheduled"], + "string": ["manual_equipment_description", "manual_issues_concerns"] + }, + latest_only=True + ) + +def getSampleDataFrame(telemetry, keys, time): + return process_dataframe( + telemetry, keys, time, + special_handling={ + "datetime": ["manual_sample_time"], + "string": ["manual_sample_datapoint", "manual_sample_lab", "manual_sample_location"] + } + ) + + def get_last_data_row(ws): # Start from the bottom row and work up to find the last row with data @@ -203,16 +239,19 @@ def lambda_handler(event, context): ) reportsheet = writer.book.worksheets[0] - ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm', 'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate', 'at_109d_o2_in_water', 'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm'] - + keys = ['ait_102a_turbitity','ait_102b_h2s', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109e_orp', 'fit_100_flow_rate', 'fit_109b_flow_rate', 'lit_116b_level', 'lit_116a_level', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'ait_102d_oil_in_water','outlet_ph'] + manual_keys = ['manual_bag_filter_changes', 'manual_cartridge_filter_changes', 'manual_clean_water_sold_per_job', 'manual_coagulant_on_hand', 'manual_diverted_water_time', 'manual_equipment_description', 'manual_equipment_time', 'manual_h202_on_hand', 'manual_issues_concerns', 'manual_next_pigging_scheduled', 'manual_skim_oil_discharged_per_job', 'manual_standby_time', 'manual_unit_uptime', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_water_events', 'manual_water_events_time', 'manual_water_to_tanks_time'] + sample_keys = ['manual_sample_datapoint', 'manual_sample_lab', 'manual_sample_location', 'manual_sample_time', 'manual_sample_value'] #Create a Sheet for each Device for device in telemetry.keys(): - df = getDataFrame(telemetry[device], ignore_keys, time) - + df = getDataFrame(telemetry[device], keys, time) + dfm = getManualDataFrame(telemetry[device], manual_keys, time) + dfs = getSampleDataFrame(telemetry[device], sample_keys, time) # Write the dataframe data to XlsxWriter. Turn off the default header and # index and skip one row to allow us to insert a user defined header. df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format="%.2f") - + dfm.to_excel(writer, sheet_name=device+" Manual Entry", startrow=0, header=True, index=True, float_format="%.2f") + dfs.to_excel(writer, sheet_name=device+" Manual Samples", startrow=0, header=True, index=True, float_format="%.2f") # Get the xlsxwriter workbook and worksheet objects. workbook = writer.book worksheet = writer.sheets[device] @@ -228,22 +267,16 @@ def lambda_handler(event, context): #Convert to excel number datetime_min = to_excel(datetime_min) datetime_max = round(to_excel(datetime_max)) - #Change the range of the chart - chart = reportsheet._charts[0] - chart.x_axis.scaling.min = datetime_min - chart.x_axis.scaling.max = datetime_max - chart.x_axis.number_format = 'hh:mm' + for chart in reportsheet._charts: + #Change the range of the chart + #chart = reportsheet._charts[0] + chart.x_axis.scaling.min = datetime_min + chart.x_axis.scaling.max = datetime_max + chart.x_axis.number_format = 'hh:mm' reportsheet["B4"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y') """ + Just a reminder of how to manipulate a single cell reportsheet["B5"] = "Test Well Name" - reportsheet["B6"] = "Test Well Lead" - reportsheet["B7"] = "Test COPA Lead" - reportsheet["B8"] = "Test Job Name" - - reportsheet["B11"]= "Test Events or Spills" - reportsheet["B13"] = "Test Issues" - - reportsheet["E5"] = "A very large summary test text to put into perspective the amount\n of work that is having to be done to this sheet\n for this to work" """ # Close the Pandas Excel writer and output the Excel file. writer.close() @@ -251,8 +284,9 @@ def lambda_handler(event, context): # Create an AWS SES client ses_client = boto3.client('ses', region_name='us-east-1') - - + s3 = boto3.resource('s3') + BUCKET_NAME = "thingsboard-email-reports" + s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb')) # Create an email message emails = [ @@ -260,7 +294,10 @@ def lambda_handler(event, context): "rkamper@thunderbirdfs.com", "john.griffin@acaciaes.com", "Bruce@enxl.us", - "Joshua.Fine@fineelectricalservices2018.com" + "Joshua.Fine@fineelectricalservices2018.com", + "choice.luster@thunderbirdfs.com", + "rvaught@thunderbirdfs.com", + "sterling.smith@enxl.us" ] msg = MIMEMultipart() msg['Subject'] = "Thunderbird Field Services" diff --git a/EKKO Reports/thunderbirdfs-daily-report/.vscode/launch.json b/EKKO Reports/thunderbirdfs-daily-report/.vscode/launch.json new file mode 100644 index 0000000..117d135 --- /dev/null +++ b/EKKO Reports/thunderbirdfs-daily-report/.vscode/launch.json @@ -0,0 +1,10 @@ +{ + "configurations": [ + { + "type": "chrome", + "name": "http://127.0.0.1:3001/Users/nico/Documents/GitHub/ThingsBoard/EKKO%20Reports/test.html", + "request": "launch", + "url": "http://127.0.0.1:3001/Users/nico/Documents/GitHub/ThingsBoard/EKKO%20Reports/test.html" + } + ] +} \ No newline at end of file diff --git a/EKKO Reports/thunderbirdfs-daily-report/template.yaml b/EKKO Reports/thunderbirdfs-daily-report/template.yaml index 723c591..58dc8da 100644 --- a/EKKO Reports/thunderbirdfs-daily-report/template.yaml +++ b/EKKO Reports/thunderbirdfs-daily-report/template.yaml @@ -8,7 +8,7 @@ Description: > # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst Globals: Function: - Timeout: 3 + Timeout: 6 MemorySize: 256 # You can add LoggingConfig parameters such as the Logformat, Log Group, and SystemLogLevel or ApplicationLogLevel. Learn more here https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html#sam-function-loggingconfig. @@ -31,6 +31,15 @@ Resources: Layers: - !Ref TFSReportLayer - arn:aws:lambda:us-east-1:668099181075:layer:AWSLambda-Python36-SciPy1x:115 + Policies: + - AmazonSESFullAccess + - Statement: + - Effect: Allow + Action: + - s3:PutObject + Resource: + - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket + - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket/* TFSReportLayer: Type: AWS::Serverless::LayerVersion Properties: @@ -45,7 +54,7 @@ Resources: Schedule: Type: AWS::Scheduler::Schedule Properties: - ScheduleExpression: cron(0 1 * * ? *) + ScheduleExpression: cron(0 2 * * ? *) FlexibleTimeWindow: Mode: 'OFF' ScheduleExpressionTimezone: America/Juneau @@ -69,4 +78,7 @@ Resources: Statement: - Effect: Allow Action: lambda:InvokeFunction - Resource: !GetAtt ThunderbirdFSReport.Arn + Resource: + - !GetAtt ThunderbirdFSReport.Arn + - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket + - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket/* diff --git a/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/ACW Daily Report Template.xlsx b/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/ACW Daily Report Template.xlsx index 84eec46..424fb4b 100644 Binary files a/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/ACW Daily Report Template.xlsx and b/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/ACW Daily Report Template.xlsx differ diff --git a/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py b/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py index 8ac01c3..f37618e 100644 --- a/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py +++ b/EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py @@ -136,41 +136,77 @@ def formatColumnName(telemetryName): "Inlet Ph Temp": "INLET PH TEMP", "Ait 102b H2s": "INLET H₂S", "At 109b H2s": "OUTLET H₂S", - "At 109c Oil In Water": "OUTLET OIL IN WATER", + "At 109c Oil In Water": "OUTLET DENSITY", "Ait 102a Turbitity": "INLET TURBIDITY", "At 109a Turbidity": "OUTLET TURBIDITY", - "At 109e Orp": "OUTLET ORP" + "At 109e Orp": "OUTLET ORP", + "Ait 102d Oil In Water": "INLET DENSITY" } - return label_mapping.get(name) + return label_mapping.get(name, name) def formatChartName(telemetryName): return " ".join([x.upper() for x in telemetryName.split("_")]) -def getDataFrame(telemetry, ignore_keys, time): +def process_dataframe(telemetry, keys, time, special_handling=None, latest_only=False): df = pd.DataFrame() - #for location in telemetry.keys(): - # Iterate through each datapoint within each location + + # If latest_only is True, ensure missing keys are initialized + if latest_only: + for key in keys: + if key not in telemetry: + telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}] + for datapoint in telemetry.keys(): - # Convert the datapoint list of dictionaries to a DataFrame - if datapoint not in ignore_keys: + if datapoint in keys: temp_df = pd.DataFrame(telemetry[datapoint]) temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None) - # Set 'ts' as the index temp_df.set_index('ts', inplace=True) - temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce") - # Rename 'value' column to the name of the datapoint + + if special_handling and datapoint in special_handling.get("datetime", []): + temp_df["value"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None) + elif special_handling and datapoint in special_handling.get("string", []): + temp_df["value"] = temp_df["value"].astype(str) + else: + temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce") + temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True) - - # Join the temp_df to the main DataFrame df = df.join(temp_df, how='outer') - df.ffill() - #df = df.fillna(method='ffill', limit=2) - # Rename index to 'Date' + + if latest_only: + latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None) + df = pd.DataFrame([latest_values]) + + df = df.reindex(sorted(df.columns), axis=1) df.rename_axis('Date', inplace=True) + return df +# Usage +def getDataFrame(telemetry, keys, time): + return process_dataframe(telemetry, keys, time) + +def getManualDataFrame(telemetry, keys, time): + return process_dataframe( + telemetry, keys, time, + special_handling={ + "datetime": ["manual_next_pigging_scheduled"], + "string": ["manual_equipment_description", "manual_issues_concerns"] + }, + latest_only=True + ) + +def getSampleDataFrame(telemetry, keys, time): + return process_dataframe( + telemetry, keys, time, + special_handling={ + "datetime": ["manual_sample_time"], + "string": ["manual_sample_datapoint", "manual_sample_lab", "manual_sample_location"] + } + ) + + def get_last_data_row(ws): # Start from the bottom row and work up to find the last row with data @@ -203,16 +239,19 @@ def lambda_handler(event, context): ) reportsheet = writer.book.worksheets[0] - ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm', 'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate', 'at_109d_o2_in_water', 'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm'] - + keys = ['ait_102a_turbitity','ait_102b_h2s', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109e_orp', 'fit_100_flow_rate', 'fit_109b_flow_rate', 'lit_116b_level', 'lit_116a_level', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'ait_102d_oil_in_water','outlet_ph'] + manual_keys = ['manual_bag_filter_changes', 'manual_cartridge_filter_changes', 'manual_clean_water_sold_per_job', 'manual_coagulant_on_hand', 'manual_diverted_water_time', 'manual_equipment_description', 'manual_equipment_time', 'manual_h202_on_hand', 'manual_issues_concerns', 'manual_next_pigging_scheduled', 'manual_skim_oil_discharged_per_job', 'manual_standby_time', 'manual_unit_uptime', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_water_events', 'manual_water_events_time', 'manual_water_to_tanks_time'] + sample_keys = ['manual_sample_datapoint', 'manual_sample_lab', 'manual_sample_location', 'manual_sample_time', 'manual_sample_value'] #Create a Sheet for each Device for device in telemetry.keys(): - df = getDataFrame(telemetry[device], ignore_keys, time) - + df = getDataFrame(telemetry[device], keys, time) + dfm = getManualDataFrame(telemetry[device], manual_keys, time) + dfs = getSampleDataFrame(telemetry[device], sample_keys, time) # Write the dataframe data to XlsxWriter. Turn off the default header and # index and skip one row to allow us to insert a user defined header. df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format="%.2f") - + dfm.to_excel(writer, sheet_name=device+" Manual Entry", startrow=0, header=True, index=True, float_format="%.2f") + dfs.to_excel(writer, sheet_name=device+" Manual Samples", startrow=0, header=True, index=True, float_format="%.2f") # Get the xlsxwriter workbook and worksheet objects. workbook = writer.book worksheet = writer.sheets[device] @@ -228,22 +267,16 @@ def lambda_handler(event, context): #Convert to excel number datetime_min = to_excel(datetime_min) datetime_max = round(to_excel(datetime_max)) - #Change the range of the chart - chart = reportsheet._charts[0] - chart.x_axis.scaling.min = datetime_min - chart.x_axis.scaling.max = datetime_max - chart.x_axis.number_format = 'hh:mm' + for chart in reportsheet._charts: + #Change the range of the chart + #chart = reportsheet._charts[0] + chart.x_axis.scaling.min = datetime_min + chart.x_axis.scaling.max = datetime_max + chart.x_axis.number_format = 'hh:mm' reportsheet["B4"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y') """ + Just a reminder of how to manipulate a single cell reportsheet["B5"] = "Test Well Name" - reportsheet["B6"] = "Test Well Lead" - reportsheet["B7"] = "Test COPA Lead" - reportsheet["B8"] = "Test Job Name" - - reportsheet["B11"]= "Test Events or Spills" - reportsheet["B13"] = "Test Issues" - - reportsheet["E5"] = "A very large summary test text to put into perspective the amount\n of work that is having to be done to this sheet\n for this to work" """ # Close the Pandas Excel writer and output the Excel file. writer.close() @@ -251,8 +284,9 @@ def lambda_handler(event, context): # Create an AWS SES client ses_client = boto3.client('ses', region_name='us-east-1') - - + s3 = boto3.resource('s3') + BUCKET_NAME = "thingsboard-email-reports" + s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb')) # Create an email message emails = [ @@ -260,7 +294,10 @@ def lambda_handler(event, context): "rkamper@thunderbirdfs.com", "john.griffin@acaciaes.com", "Bruce@enxl.us", - "Joshua.Fine@fineelectricalservices2018.com" + "Joshua.Fine@fineelectricalservices2018.com", + "choice.luster@thunderbirdfs.com", + "rvaught@thunderbirdfs.com", + "sterling.smith@enxl.us" ] msg = MIMEMultipart() msg['Subject'] = "Thunderbird Field Services"