added additional datapoints and charts

EKKO Reports/test.html

@@ -0,0 +1,40 @@
+<div class="mat-mdc-text-field-wrapper mdc-text-field ng-tns-c508571215-19 mdc-text-field--outlined"><!---->
+    <div class="mat-mdc-form-field-flex ng-tns-c508571215-19">
+        <div matformfieldnotchedoutline=""
+            class="mdc-notched-outline ng-tns-c508571215-19 mdc-notched-outline--upgraded ng-star-inserted">
+            <div class="mat-mdc-notch-piece mdc-notched-outline__leading"></div>
+            <div class="mat-mdc-notch-piece mdc-notched-outline__notch" style=""><label matformfieldfloatinglabel=""
+                    class="mdc-floating-label mat-mdc-floating-label ng-tns-c508571215-19 ng-star-inserted"
+                    id="mat-mdc-form-field-label-20" for="mat-input-10" style="transform: var(
+        --mat-mdc-form-field-label-transform,
+        translateY(-50%) translateX(calc(1 * (52px + var(--mat-mdc-form-field-label-offset-x, 0px))))
+    );"><mat-label _ngcontent-ng-c109547480="" class="ng-tns-c508571215-19">Next Pigging Run Scheduled
+                        (date)</mat-label><!----></label><!----><!----><!----></div>
+            <div class="mat-mdc-notch-piece mdc-notched-outline__trailing"></div>
+        </div><!---->
+        <div class="mat-mdc-form-field-icon-prefix ng-tns-c508571215-19 ng-star-inserted">
+            <mat-datetimepicker-toggle _ngcontent-ng-c109547480="" matprefix=""
+                class="mat-datetimepicker-toggle ng-tns-c508571215-19"><button mat-icon-button="" type="button"
+                    mat-ripple-loader-uninitialized="" mat-ripple-loader-class-name="mat-mdc-button-ripple"
+                    class="mdc-icon-button mat-mdc-icon-button mat-unthemed mat-mdc-button-base"
+                    mat-ripple-loader-centered="" aria-label="Open calendar">
+                    <span class="mat-mdc-button-persistent-ripple mdc-icon-button__ripple"></span><mat-icon role="img"
+                        class="mat-icon notranslate material-icons mat-ligature-font mat-icon-no-color"
+                        aria-hidden="true" data-mat-icon-type="font"><!---->
+                        <svg fill="currentColor" focusable="false"
+                            height="100%" viewBox="0 0 24 24" width="100%" style="vertical-align: top;"
+                            class="ng-star-inserted">
+                            <path
+                                d="M15,13H16.5V15.82L18.94,17.23L18.19,18.53L15,16.69V13M19,8H5V19H9.67C9.24,18.09 9,17.07 9,16A7,7 0 0,1 16,9C17.07,9 18.09,9.24 19,9.67V8M5,21C3.89,21 3,20.1 3,19V5C3,3.89 3.89,3 5,3H6V1H8V3H16V1H18V3H19A2,2 0 0,1 21,5V11.1C22.24,12.36 23,14.09 23,16A7,7 0 0,1 16,23C14.09,23 12.36,22.24 11.1,21H5M16,11.15A4.85,4.85 0 0,0 11.15,16C11.15,18.68 13.32,20.85 16,20.85A4.85,4.85 0 0,0 20.85,16C20.85,13.32 18.68,11.15 16,11.15Z">
+                            </path>
+                        </svg><!----><!----></mat-icon><span class="mat-mdc-focus-indicator"></span><span
+                        class="mat-mdc-button-touch-target"></span></button></mat-datetimepicker-toggle>
+        </div>
+        <!----><!---->
+        <div class="mat-mdc-form-field-infix ng-tns-c508571215-19"><!----><mat-datetimepicker
+                _ngcontent-ng-c109547480="" openonfocus="true"
+                class="ng-tns-c508571215-19"></mat-datetimepicker><!----><input _ngcontent-ng-c109547480="" matinput=""
+                class="mat-mdc-input-element ng-tns-c508571215-19 mat-mdc-form-field-input-control mdc-text-field__input ng-pristine ng-valid cdk-text-field-autofill-monitored ng-touched"
+                id="mat-input-10" aria-invalid="false" aria-required="false" aria-haspopup="true"></div><!----><!---->
+    </div><!---->
+</div>

EKKO Reports/thunderbirdfs-daily-report/.aws-sam/build.toml

@@ -5,7 +5,7 @@ codeuri = "/Users/nico/Documents/GitHub/ThingsBoard/EKKO Reports/thunderbirdfs-d
 runtime = "python3.9"
 architecture = "x86_64"
 handler = "thunderbirdfsreport.lambda_handler"
-source_hash = "470cf4aeb6b1a4872891b288a63843be543d3c07583f5f99de6198845430a4cc"
+source_hash = "96f09690c748fbb53cc41f4396ed0d83dfb501a3cb2710f29c4149645fc9c9fb"
 manifest_hash = ""
 packagetype = "Zip"
 functions = ["ThunderbirdFSReport"]

EKKO Reports/thunderbirdfs-daily-report/.aws-sam/cache/67ab70ce-dfbc-432e-90ee-b104415061ba/thunderbirdfsreport.py

@@ -136,41 +136,77 @@ def formatColumnName(telemetryName):
     "Inlet Ph Temp": "INLET PH TEMP",
     "Ait 102b H2s": "INLET H₂S",
     "At 109b H2s": "OUTLET H₂S",
-    "At 109c Oil In Water": "OUTLET OIL IN WATER",
+    "At 109c Oil In Water": "OUTLET DENSITY",
     "Ait 102a Turbitity": "INLET TURBIDITY",
     "At 109a Turbidity": "OUTLET TURBIDITY",
-    "At 109e Orp": "OUTLET ORP"
+    "At 109e Orp": "OUTLET ORP",
+    "Ait 102d Oil In Water": "INLET DENSITY"
    }
-   return label_mapping.get(name)
+   return label_mapping.get(name, name)
 
 
 def formatChartName(telemetryName):
    return " ".join([x.upper() for x in telemetryName.split("_")])
 
 
-def getDataFrame(telemetry, ignore_keys, time):    
+def process_dataframe(telemetry, keys, time, special_handling=None, latest_only=False):    
     df = pd.DataFrame()
-    #for location in telemetry.keys():
+
-    # Iterate through each datapoint within each location
+    # If latest_only is True, ensure missing keys are initialized
+    if latest_only:
+        for key in keys:
+            if key not in telemetry:
+                telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}]
+
     for datapoint in telemetry.keys():
-        # Convert the datapoint list of dictionaries to a DataFrame
+        if datapoint in keys:
-        if datapoint not in ignore_keys:
             temp_df = pd.DataFrame(telemetry[datapoint])
             temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None)
-            # Set 'ts' as the index
             temp_df.set_index('ts', inplace=True)
-            temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce")
-            # Rename 'value' column to the name of the datapoint
-            temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)
 
-            # Join the temp_df to the main DataFrame
+            if special_handling and datapoint in special_handling.get("datetime", []):
+                temp_df["value"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None)
+            elif special_handling and datapoint in special_handling.get("string", []):
+                temp_df["value"] = temp_df["value"].astype(str)
+            else:
+                temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce")
+
+            temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)
             df = df.join(temp_df, how='outer')
-            df.ffill()
+
-            #df = df.fillna(method='ffill', limit=2)
+    if latest_only:
-     # Rename index to 'Date'
+        latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None)
+        df = pd.DataFrame([latest_values])
+
+    df = df.reindex(sorted(df.columns), axis=1)
     df.rename_axis('Date', inplace=True)
+
     return df
 
+# Usage
+def getDataFrame(telemetry, keys, time):
+    return process_dataframe(telemetry, keys, time)
+
+def getManualDataFrame(telemetry, keys, time):
+    return process_dataframe(
+        telemetry, keys, time, 
+        special_handling={
+            "datetime": ["manual_next_pigging_scheduled"],
+            "string": ["manual_equipment_description", "manual_issues_concerns"]
+        }, 
+        latest_only=True
+    )
+
+def getSampleDataFrame(telemetry, keys, time):
+    return process_dataframe(
+        telemetry, keys, time, 
+        special_handling={
+            "datetime": ["manual_sample_time"],
+            "string": ["manual_sample_datapoint", "manual_sample_lab", "manual_sample_location"]
+        }
+    )
+
+
 
 def get_last_data_row(ws):
     # Start from the bottom row and work up to find the last row with data
@@ -203,16 +239,19 @@ def lambda_handler(event, context):
             )
 
         reportsheet = writer.book.worksheets[0]
-        ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm',  'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate',  'at_109d_o2_in_water',  'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm']
+        keys = ['ait_102a_turbitity','ait_102b_h2s', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109e_orp', 'fit_100_flow_rate', 'fit_109b_flow_rate', 'lit_116b_level', 'lit_116a_level', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'ait_102d_oil_in_water','outlet_ph']
-
+        manual_keys = ['manual_bag_filter_changes', 'manual_cartridge_filter_changes', 'manual_clean_water_sold_per_job', 'manual_coagulant_on_hand', 'manual_diverted_water_time', 'manual_equipment_description', 'manual_equipment_time', 'manual_h202_on_hand', 'manual_issues_concerns', 'manual_next_pigging_scheduled', 'manual_skim_oil_discharged_per_job', 'manual_standby_time', 'manual_unit_uptime', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_water_events', 'manual_water_events_time', 'manual_water_to_tanks_time']
+        sample_keys = ['manual_sample_datapoint', 'manual_sample_lab', 'manual_sample_location', 'manual_sample_time', 'manual_sample_value']
         #Create a Sheet for each Device
         for device in telemetry.keys():
-            df = getDataFrame(telemetry[device], ignore_keys, time)
+            df = getDataFrame(telemetry[device], keys, time)
-            
+            dfm = getManualDataFrame(telemetry[device], manual_keys, time)
+            dfs = getSampleDataFrame(telemetry[device], sample_keys, time)
             # Write the dataframe data to XlsxWriter. Turn off the default header and
             # index and skip one row to allow us to insert a user defined header.
             df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format="%.2f")
-
+            dfm.to_excel(writer, sheet_name=device+" Manual Entry", startrow=0, header=True, index=True, float_format="%.2f")
+            dfs.to_excel(writer, sheet_name=device+" Manual Samples", startrow=0, header=True, index=True, float_format="%.2f")
             # Get the xlsxwriter workbook and worksheet objects.
             workbook = writer.book
             worksheet = writer.sheets[device]
@@ -228,22 +267,16 @@ def lambda_handler(event, context):
         #Convert to excel number
         datetime_min = to_excel(datetime_min)
         datetime_max = round(to_excel(datetime_max))
+        for chart in reportsheet._charts:
             #Change the range of the chart
-        chart = reportsheet._charts[0]
+            #chart = reportsheet._charts[0]
             chart.x_axis.scaling.min = datetime_min
             chart.x_axis.scaling.max = datetime_max
             chart.x_axis.number_format = 'hh:mm'
         reportsheet["B4"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y')
         """ 
+        Just a reminder of how to manipulate a single cell
         reportsheet["B5"] = "Test Well Name"
-        reportsheet["B6"] = "Test Well Lead"
-        reportsheet["B7"] = "Test COPA Lead"
-        reportsheet["B8"] = "Test Job Name"
-
-        reportsheet["B11"]= "Test Events or Spills"
-        reportsheet["B13"] = "Test Issues"
-
-        reportsheet["E5"] = "A very large summary test text to put into perspective the amount\n of work that is having to be done to this sheet\n for this to work"
         """
         # Close the Pandas Excel writer and output the Excel file.
         writer.close()
@@ -251,8 +284,9 @@ def lambda_handler(event, context):
 
         # Create an AWS SES client
         ses_client = boto3.client('ses', region_name='us-east-1')
-
+        s3 = boto3.resource('s3')
-
+        BUCKET_NAME = "thingsboard-email-reports"
+        s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb'))
         # Create an email message
 
         emails = [
@@ -260,7 +294,10 @@ def lambda_handler(event, context):
             "rkamper@thunderbirdfs.com", 
             "john.griffin@acaciaes.com",
             "Bruce@enxl.us", 
-            "Joshua.Fine@fineelectricalservices2018.com"
+            "Joshua.Fine@fineelectricalservices2018.com",
+            "choice.luster@thunderbirdfs.com",
+            "rvaught@thunderbirdfs.com",
+            "sterling.smith@enxl.us"
         ]
         msg = MIMEMultipart()
         msg['Subject'] = "Thunderbird Field Services"

EKKO Reports/thunderbirdfs-daily-report/.vscode/launch.json

@@ -0,0 +1,10 @@
+{
+    "configurations": [
+        {
+            "type": "chrome",
+            "name": "http://127.0.0.1:3001/Users/nico/Documents/GitHub/ThingsBoard/EKKO%20Reports/test.html",
+            "request": "launch",
+            "url": "http://127.0.0.1:3001/Users/nico/Documents/GitHub/ThingsBoard/EKKO%20Reports/test.html"
+        }
+    ]
+}

EKKO Reports/thunderbirdfs-daily-report/template.yaml

@@ -8,7 +8,7 @@ Description: >
 # More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst
 Globals:
   Function:
-    Timeout: 3
+    Timeout: 6
     MemorySize: 256
 
     # You can add LoggingConfig parameters such as the Logformat, Log Group, and SystemLogLevel or ApplicationLogLevel. Learn more here https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-resource-function.html#sam-function-loggingconfig.
@@ -31,6 +31,15 @@ Resources:
       Layers:
         - !Ref TFSReportLayer
         - arn:aws:lambda:us-east-1:668099181075:layer:AWSLambda-Python36-SciPy1x:115
+      Policies:
+        - AmazonSESFullAccess
+        - Statement:
+            - Effect: Allow
+              Action:
+                - s3:PutObject
+              Resource:
+                - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket
+                - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket/*
   TFSReportLayer:
     Type: AWS::Serverless::LayerVersion
     Properties:
@@ -45,7 +54,7 @@ Resources:
   Schedule:
     Type: AWS::Scheduler::Schedule
     Properties:
-      ScheduleExpression: cron(0 1 * * ? *)
+      ScheduleExpression: cron(0 2 * * ? *)
       FlexibleTimeWindow:
         Mode: 'OFF'
       ScheduleExpressionTimezone: America/Juneau
@@ -69,4 +78,7 @@ Resources:
             Statement:
               - Effect: Allow
                 Action: lambda:InvokeFunction
-                Resource: !GetAtt ThunderbirdFSReport.Arn    
+                Resource: 
+                  - !GetAtt ThunderbirdFSReport.Arn
+                  - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket
+                  - !Sub arn:${AWS::Partition}:s3:::!ImportValue TBReportBucket/*    

EKKO Reports/thunderbirdfs-daily-report/thunderbirdfsreport/thunderbirdfsreport.py

@@ -136,41 +136,77 @@ def formatColumnName(telemetryName):
     "Inlet Ph Temp": "INLET PH TEMP",
     "Ait 102b H2s": "INLET H₂S",
     "At 109b H2s": "OUTLET H₂S",
-    "At 109c Oil In Water": "OUTLET OIL IN WATER",
+    "At 109c Oil In Water": "OUTLET DENSITY",
     "Ait 102a Turbitity": "INLET TURBIDITY",
     "At 109a Turbidity": "OUTLET TURBIDITY",
-    "At 109e Orp": "OUTLET ORP"
+    "At 109e Orp": "OUTLET ORP",
+    "Ait 102d Oil In Water": "INLET DENSITY"
    }
-   return label_mapping.get(name)
+   return label_mapping.get(name, name)
 
 
 def formatChartName(telemetryName):
    return " ".join([x.upper() for x in telemetryName.split("_")])
 
 
-def getDataFrame(telemetry, ignore_keys, time):    
+def process_dataframe(telemetry, keys, time, special_handling=None, latest_only=False):    
     df = pd.DataFrame()
-    #for location in telemetry.keys():
+
-    # Iterate through each datapoint within each location
+    # If latest_only is True, ensure missing keys are initialized
+    if latest_only:
+        for key in keys:
+            if key not in telemetry:
+                telemetry[key] = [{'ts': dt.timestamp(dt.now()), 'value': '0'}]
+
     for datapoint in telemetry.keys():
-        # Convert the datapoint list of dictionaries to a DataFrame
+        if datapoint in keys:
-        if datapoint not in ignore_keys:
             temp_df = pd.DataFrame(telemetry[datapoint])
             temp_df['ts'] = pd.to_datetime(temp_df['ts'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None)
-            # Set 'ts' as the index
             temp_df.set_index('ts', inplace=True)
-            temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce")
-            # Rename 'value' column to the name of the datapoint
-            temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)
 
-            # Join the temp_df to the main DataFrame
+            if special_handling and datapoint in special_handling.get("datetime", []):
+                temp_df["value"] = pd.to_datetime(temp_df['value'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(time["timezone"]).dt.tz_localize(None)
+            elif special_handling and datapoint in special_handling.get("string", []):
+                temp_df["value"] = temp_df["value"].astype(str)
+            else:
+                temp_df["value"] = pd.to_numeric(temp_df["value"], errors="coerce")
+
+            temp_df.rename(columns={'value': formatColumnName(datapoint)}, inplace=True)
             df = df.join(temp_df, how='outer')
-            df.ffill()
+
-            #df = df.fillna(method='ffill', limit=2)
+    if latest_only:
-     # Rename index to 'Date'
+        latest_values = df.apply(lambda x: x.dropna().iloc[-1] if not x.dropna().empty else None)
+        df = pd.DataFrame([latest_values])
+
+    df = df.reindex(sorted(df.columns), axis=1)
     df.rename_axis('Date', inplace=True)
+
     return df
 
+# Usage
+def getDataFrame(telemetry, keys, time):
+    return process_dataframe(telemetry, keys, time)
+
+def getManualDataFrame(telemetry, keys, time):
+    return process_dataframe(
+        telemetry, keys, time, 
+        special_handling={
+            "datetime": ["manual_next_pigging_scheduled"],
+            "string": ["manual_equipment_description", "manual_issues_concerns"]
+        }, 
+        latest_only=True
+    )
+
+def getSampleDataFrame(telemetry, keys, time):
+    return process_dataframe(
+        telemetry, keys, time, 
+        special_handling={
+            "datetime": ["manual_sample_time"],
+            "string": ["manual_sample_datapoint", "manual_sample_lab", "manual_sample_location"]
+        }
+    )
+
+
 
 def get_last_data_row(ws):
     # Start from the bottom row and work up to find the last row with data
@@ -203,16 +239,19 @@ def lambda_handler(event, context):
             )
 
         reportsheet = writer.book.worksheets[0]
-        ignore_keys = ['latitude', 'longitude', 'speed', 'a_current', 'b_current', 'c_current', 'scada_stop_cmd', 'pit_100a_pressure', 'pit_101a_pressure', 'pit_101b_pressure', 'pit_101c_pressure', 'fit_101_flow_rate', 'fi_101b_popoff', 'fcv_101a_valve', 'fcv_101b_valve', 'pit_102_pressure', 'pit_102_hi_alm', 'pit_102_hihi_alm', 'pit_102_hi_spt', 'pit_102_hihi_spt', 'p200_hand', 'p200_auto', 'xy_200_run', 'ct_200_run', 'pit_100_pressure', 'm106a_vfd_active', 'm106a_vfd_faulted', 'm106a_vfd_frequency', 'm106a_vfd_start', 'm106a_vfd_stop', 'pit_106a_pressure', 'fit_106a_flow_rate', 'm106b_vfd_active', 'm106b_vfd_faulted', 'm106b_vfd_frequency', 'm106b_vfd_start', 'm106b_vfd_stop', 'pit_106b_pressure', 'fit_106b_flow_rate', 'pit_106c_pressure', 'pit_106d_pressure', 'sdv106_open', 'sdv106_closed', 'bp_3a_auto', 'bp_3a_hand', 'bp_3a_run_cmd', 'bp_3a_run', 'bp_3a_fault', 'bp_3b_auto', 'bp_3b_hand', 'bp_3b_run_cmd', 'bp_3b_run', 'bp_3b_fault', 'pit_107a_pressure', 'fit_107a_flow_rate', 'pit_107b_pressure', 'fcv_001_valve', 'fit_107b_flow_rate', 'pit_107d_pressure', 'fcv_002_valve', 'pit_107c_pressure', 'pit_108a_pressure', 'pit_108b_pressure', 'dpi_108a_pressure', 'pit_108c_pressure', 'pit_108d_pressure', 'pdt_108b_pressure', 'pit_108e_pressure', 'pit_108f_pressure', 'pdt_108c_pressure', 'pit_108_pressure', 'pdt_108a_hi_alm', 'pdt_108a_hihi_alm', 'pdt_108b_hi_alm', 'pdt_108b_hihi_alm', 'pdt_108c_hi_alm', 'pdt_108c_hihi_alm',  'ait_102c_ph', 'ait_102d_oil_in_water', 'fit_102_flow_rate', 'lit_112a_h2o2_level', 'lit_112b_nahso3_level', 'fis_112_h2o2_popoff', 'fit_112a_h2o2_flow_rate', 'fit_112b_nahso3_flow_rate',  'at_109d_o2_in_water',  'fit_100_hi_alm', 'fit_100_hihi_alm', 'fit_100_lo_alm', 'fit_111_flow_rate', 'pit_110_pressure', 'lit_170_level', 'lit_200_level', 'lit_101_level', 'li_103D_level_alm', 'lsh_120_hihi_alm', 'pit_050_pressure', 'pit_065_pressure', 'pdi_065_pressure', 'fit_104_n2_rate', 'p100_auto', 'p100_hand', 'sales_recirculate_sw', 'fit_109a_flow_rate', 'pit_111a_n2', 'pit_111b_n2', 'pit_111c_n2', 'ct_200_current', 'sdv_101a', 'xy_100_run', 'skim_total_barrels', 'dpi_108b_pressure', 'chemical_pump_01_run_status', 'chemical_pump_01_rate_offset', 'spt_pid_h2o2_chemical_rate', 'spt_chemical_manual_rate', 'chemical_pump_auto', 'esd_exists', 'n2_purity', 'n2_outlet_flow_rate', 'n2_outlet_temp', 'n2_inlet_pressure', 'compressor_controller_temp', 'compressor_ambient_temp', 'compressor_outlet_temp', 'compressor_outlet_pressure', 'n2_outlet_pressure', 'fit_109b_water_job', 'fit_109b_water_last_month', 'fit_109b_water_month', 'fit_109b_water_lifetime', 'fit_109b_water_today', 'fit_109b_water_yesterday', 'fit_100_water_job', 'fit_100_water_last_month', 'fit_100_water_month', 'fit_100_water_lifetime', 'fit_100_water_today', 'fit_100_water_yesterday', 'h2o2_chemical_rate', 'rmt_sd_alm', 'pnl_esd_alm', 'pit_111c_hihi_alm', 'pit_111b_hihi_alm', 'pit_111a_hihi_alm', 'pit_110_hihi_alm', 'pit_108g_hihi_alm', 'pit_108c_hihi_alm', 'pit_108b_hihi_alm', 'pit_108a_hihi_alm', 'pit_107b_lolo_alm', 'pit_107a_lolo_alm', 'pit_106b_hihi_alm', 'pit_106a_hihi_alm', 'pit_101b_transmitter_alm', 'pit_101b_hihi_alm', 'pit_101a_transmitter_alm', 'pit_101a_hihi_alm', 'pit_101a_hi_alm', 'pit_100_hihi_alm', 'pit_065_hihi_alm', 'pit_050_hihi_alm', 'pdi_065_lolo_alm', 'pdi_065_lo_alm', 'pdi_065_hihi_alm', 'm106b_vfd_faulted_alm', 'm106a_vfd_faulted_alm', 'lit_200_hihi_alm', 'lit_170_hihi_alm', 'fit_107b_lolo_alm', 'fit_107a_lolo_alm', 'fit_106b_hihi_alm', 'fit_106a_hihi_alm', 'fit_004_hihi_alm', 'bp_3b_run_fail_alm', 'bp_3a_run_fail_alm', 'ait_114c_hihi_alm', 'ait_114b_hihi_alm', 'ait_114a_hihi_alm', 'ac_volt', 'bc_volt', 'ab_volt', 'psd_alm', 'ait_114a_lolo_alm', 'ait_114a_lo_alm', 'ait_114r_lolo_alm', 'ait_114r_lo_alm', 'ait_114z_lo_alm', 'ait_114z_lolo_alm', 'ait_114x_lo_alm', 'ait_114x_lolo_alm', 'ait_114c_lolo_alm', 'ait_114c_lo_alm', 'ait_114l_lolo_alm', 'ait_114l_lo_alm', 'lit_116b_hihi_alm', 'lit_116b_hi_alm', 'lit_116a_hihi_alm', 'lit_116a_hi_alm']
+        keys = ['ait_102a_turbitity','ait_102b_h2s', 'at_109a_turbidity', 'at_109b_h2s', 'at_109c_oil_in_water', 'at_109e_orp', 'fit_100_flow_rate', 'fit_109b_flow_rate', 'lit_116b_level', 'lit_116a_level', 'outlet_turbidity_temp', 'outlet_orp_temp', 'inlet_turbidity_temp', 'inlet_ph_temp', 'ait_102d_oil_in_water','outlet_ph']
-
+        manual_keys = ['manual_bag_filter_changes', 'manual_cartridge_filter_changes', 'manual_clean_water_sold_per_job', 'manual_coagulant_on_hand', 'manual_diverted_water_time', 'manual_equipment_description', 'manual_equipment_time', 'manual_h202_on_hand', 'manual_issues_concerns', 'manual_next_pigging_scheduled', 'manual_skim_oil_discharged_per_job', 'manual_standby_time', 'manual_unit_uptime', 'manual_upright_tank_issues', 'manual_vac_truck_batches', 'manual_water_events', 'manual_water_events_time', 'manual_water_to_tanks_time']
+        sample_keys = ['manual_sample_datapoint', 'manual_sample_lab', 'manual_sample_location', 'manual_sample_time', 'manual_sample_value']
         #Create a Sheet for each Device
         for device in telemetry.keys():
-            df = getDataFrame(telemetry[device], ignore_keys, time)
+            df = getDataFrame(telemetry[device], keys, time)
-            
+            dfm = getManualDataFrame(telemetry[device], manual_keys, time)
+            dfs = getSampleDataFrame(telemetry[device], sample_keys, time)
             # Write the dataframe data to XlsxWriter. Turn off the default header and
             # index and skip one row to allow us to insert a user defined header.
             df.to_excel(writer, sheet_name=device, startrow=0, header=True, index=True, float_format="%.2f")
-
+            dfm.to_excel(writer, sheet_name=device+" Manual Entry", startrow=0, header=True, index=True, float_format="%.2f")
+            dfs.to_excel(writer, sheet_name=device+" Manual Samples", startrow=0, header=True, index=True, float_format="%.2f")
             # Get the xlsxwriter workbook and worksheet objects.
             workbook = writer.book
             worksheet = writer.sheets[device]
@@ -228,22 +267,16 @@ def lambda_handler(event, context):
         #Convert to excel number
         datetime_min = to_excel(datetime_min)
         datetime_max = round(to_excel(datetime_max))
+        for chart in reportsheet._charts:
             #Change the range of the chart
-        chart = reportsheet._charts[0]
+            #chart = reportsheet._charts[0]
             chart.x_axis.scaling.min = datetime_min
             chart.x_axis.scaling.max = datetime_max
             chart.x_axis.number_format = 'hh:mm'
         reportsheet["B4"].value = dt.fromtimestamp(getTime(time)[0]/1000).strftime('%m/%d/%Y')
         """ 
+        Just a reminder of how to manipulate a single cell
         reportsheet["B5"] = "Test Well Name"
-        reportsheet["B6"] = "Test Well Lead"
-        reportsheet["B7"] = "Test COPA Lead"
-        reportsheet["B8"] = "Test Job Name"
-
-        reportsheet["B11"]= "Test Events or Spills"
-        reportsheet["B13"] = "Test Issues"
-
-        reportsheet["E5"] = "A very large summary test text to put into perspective the amount\n of work that is having to be done to this sheet\n for this to work"
         """
         # Close the Pandas Excel writer and output the Excel file.
         writer.close()
@@ -251,8 +284,9 @@ def lambda_handler(event, context):
 
         # Create an AWS SES client
         ses_client = boto3.client('ses', region_name='us-east-1')
-
+        s3 = boto3.resource('s3')
-
+        BUCKET_NAME = "thingsboard-email-reports"
+        s3.Object(BUCKET_NAME, f"Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx").put(Body=open(f"/tmp/Thunderbird_{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb'))
         # Create an email message
 
         emails = [
@@ -260,7 +294,10 @@ def lambda_handler(event, context):
             "rkamper@thunderbirdfs.com", 
             "john.griffin@acaciaes.com",
             "Bruce@enxl.us", 
-            "Joshua.Fine@fineelectricalservices2018.com"
+            "Joshua.Fine@fineelectricalservices2018.com",
+            "choice.luster@thunderbirdfs.com",
+            "rvaught@thunderbirdfs.com",
+            "sterling.smith@enxl.us"
         ]
         msg = MIMEMultipart()
         msg['Subject'] = "Thunderbird Field Services"