changes to ekko report
This commit is contained in:
Nico Melone
@@ -5,7 +5,7 @@
|
||||
],
|
||||
"customers": {
|
||||
"ec691940-52e2-11ec-a919-556e8dbef35c": {
|
||||
"name": "CrownQuest",
|
||||
"name": "OxyRock",
|
||||
"deviceTypes": [
|
||||
{
|
||||
"deviceType": "rigpump",
|
||||
@@ -17,18 +17,27 @@
|
||||
{
|
||||
"deviceType": "cqwatertanks",
|
||||
"dataPoints": [
|
||||
"fm_1_flow_rate"
|
||||
"fm_1_flow_rate",
|
||||
"tank_1_level"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"81083430-6988-11ec-a919-556e8dbef35c": {
|
||||
"name": "Henry-Petroleum",
|
||||
"deviceTypes": [
|
||||
},
|
||||
{
|
||||
"deviceType": "abbflow",
|
||||
"deviceType": "piflow",
|
||||
"dataPoints": [
|
||||
"accumulated_volume"
|
||||
"avgFrequency30Days",
|
||||
"percentRunTime30Days",
|
||||
"yesterday_totalizer_1",
|
||||
"totalizer_1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"deviceType": "advvfdipp",
|
||||
"dataPoints": [
|
||||
"flowtotalyesterday",
|
||||
"fluidlevel",
|
||||
"energytotalyesterday",
|
||||
"avgFrequency30Days",
|
||||
"percentRunTime30Days"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -36,7 +45,7 @@
|
||||
},
|
||||
"filterDevicesIn": [],
|
||||
"filterDevicesOut": [],
|
||||
"name": "CrownQuest-Daily-Report"
|
||||
"name": "OxyRock-Daily-Report"
|
||||
},
|
||||
{
|
||||
"emails": [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
1445
Report Generator/example_data.json
Normal file
1445
Report Generator/example_data.json
Normal file
File diff suppressed because it is too large
Load Diff
2
Report Generator/lambda-python3.12/build-local-test.sh
Executable file
2
Report Generator/lambda-python3.12/build-local-test.sh
Executable file
@@ -0,0 +1,2 @@
|
||||
DOCKER_HOST=unix:///Users/nico/.docker/run/docker.sock sam build --use-container
|
||||
DOCKER_HOST=unix:///Users/nico/.docker/run/docker.sock sam local invoke
|
||||
@@ -5,38 +5,70 @@
|
||||
],
|
||||
"customers": {
|
||||
"ec691940-52e2-11ec-a919-556e8dbef35c": {
|
||||
"name": "CrownQuest",
|
||||
"name": "OxyRock",
|
||||
"deviceTypes": [
|
||||
{
|
||||
"deviceType": "rigpump",
|
||||
"dataPoints": [
|
||||
"vfd_current",
|
||||
"vfd_frequency"
|
||||
]
|
||||
],
|
||||
"labels":{
|
||||
"vfd_current": "VFD Current",
|
||||
"vfd_frequency": "VFD Frequency"
|
||||
}
|
||||
},
|
||||
{
|
||||
"deviceType": "cqwatertanks",
|
||||
"dataPoints": [
|
||||
"fm_1_flow_rate"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"81083430-6988-11ec-a919-556e8dbef35c": {
|
||||
"name": "Henry-Petroleum",
|
||||
"deviceTypes": [
|
||||
"fm_1_flow_rate",
|
||||
"tank_1_level",
|
||||
"tank_2_level"
|
||||
],
|
||||
"labels":{
|
||||
"fm_1_flow_rate": "Flow Meter 1 Flow Rate",
|
||||
"tank_1_level": "Tank 1 Level",
|
||||
"tank_2_level": "Tank 2 Level"
|
||||
}
|
||||
},
|
||||
{
|
||||
"deviceType": "abbflow",
|
||||
"deviceType": "piflow",
|
||||
"dataPoints": [
|
||||
"accumulated_volume"
|
||||
]
|
||||
"avgFrequency30Days",
|
||||
"percentRunTime30Days",
|
||||
"yesterday_totalizer_1",
|
||||
"totalizer_1"
|
||||
],
|
||||
"labels":{
|
||||
"avgFrequency30Days": "Avg. Frequency 30 Days",
|
||||
"percentRunTime30Days": "Run Time 30 Days %",
|
||||
"yesterday_totalizer_1": "Yesterday's Volume",
|
||||
"totalizer_1": "Totalizer 1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"deviceType": "advvfdipp",
|
||||
"dataPoints": [
|
||||
"flowtotalyesterday",
|
||||
"fluidlevel",
|
||||
"energytotalyesterday",
|
||||
"avgFrequency30Days",
|
||||
"percentRunTime30Days"
|
||||
],
|
||||
"labels":{
|
||||
"flowtotalyesterday": "Yesterday's Volume",
|
||||
"fluidlevel": "Fluid Level",
|
||||
"energytotalyesterday": "Yesterday's Energy",
|
||||
"avgFrequency30Days": "Avg. Frequency 30 Days",
|
||||
"percentRunTime30Days": "Run Time 30 Days %"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"filterDevicesIn": [],
|
||||
"filterDevicesOut": [],
|
||||
"name": "CrownQuest-Daily-Report"
|
||||
"name": "OxyRock-Daily-Report"
|
||||
},
|
||||
{
|
||||
"emails": [
|
||||
@@ -50,7 +82,10 @@
|
||||
"deviceType": "abbflow",
|
||||
"dataPoints": [
|
||||
"accumulated_volume"
|
||||
]
|
||||
],
|
||||
"labels":{
|
||||
"accumulated_volume": "Accumulated Volume"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,79 +1,137 @@
|
||||
from tb_rest_client.rest_client_pe import *
|
||||
from tb_rest_client.rest import ApiException
|
||||
import json, xlsxwriter, boto3, os
|
||||
import json, xlsxwriter, boto3, os, time
|
||||
from threading import Lock
|
||||
from datetime import datetime as dt
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email import encoders
|
||||
from email.mime.base import MIMEBase
|
||||
|
||||
# Define a rate limiter class
|
||||
class RateLimiter:
|
||||
def __init__(self, max_calls, period):
|
||||
self.max_calls = max_calls
|
||||
self.period = period
|
||||
self.call_times = []
|
||||
self.lock = Lock()
|
||||
|
||||
def acquire(self):
|
||||
with self.lock:
|
||||
current_time = time.time()
|
||||
# Remove expired calls
|
||||
self.call_times = [t for t in self.call_times if t > current_time - self.period]
|
||||
if len(self.call_times) >= self.max_calls:
|
||||
# Wait for the oldest call to expire
|
||||
time_to_wait = self.period - (current_time - self.call_times[0])
|
||||
time.sleep(time_to_wait)
|
||||
# Register the current call
|
||||
self.call_times.append(time.time())
|
||||
|
||||
# Initialize a rate limiter
|
||||
rate_limiter = RateLimiter(max_calls=10, period=1) # Adjust `max_calls` and `period` as needed
|
||||
|
||||
def sort_dict_keys(d):
|
||||
"""Sorts the keys of all nested dictionaries in a given dictionary.
|
||||
|
||||
Args:
|
||||
d: The input dictionary.
|
||||
|
||||
Returns:
|
||||
A new dictionary with sorted keys at each level.
|
||||
"""
|
||||
sorted_d = {}
|
||||
for k, v in d.items():
|
||||
if isinstance(v, dict):
|
||||
sorted_d[k] = sort_dict_keys(v)
|
||||
else:
|
||||
sorted_d[k] = v
|
||||
return dict(sorted(sorted_d.items()))
|
||||
|
||||
def lambda_handler(event, context):
|
||||
#Creating Rest Client for ThingsBoard
|
||||
# Creating Rest Client for ThingsBoard
|
||||
with RestClientPE(base_url="https://hp.henrypump.cloud") as rest_client:
|
||||
try:
|
||||
rest_client.login(username=os.environ["username"], password=os.environ["password"])
|
||||
#Loading Config from file
|
||||
# Loading Config from file
|
||||
with open("./config.json") as f:
|
||||
config = json.load(f)
|
||||
|
||||
reportData = {}
|
||||
reportToList = {}
|
||||
#Loop through each item in config, each item represents a report
|
||||
|
||||
# Loop through each item in config, each item represents a report
|
||||
for report in config:
|
||||
reportToList[report["name"]] = report["emails"]
|
||||
#Each customer becomes it's own xlsx file later
|
||||
for customer in report["customers"].keys():
|
||||
#Get all the devices for a given customer
|
||||
devices = rest_client.get_customer_devices(customer_id=customer, page=0, page_size=100)
|
||||
#Filter devices to the desired devices
|
||||
# Apply rate limiting for API calls
|
||||
rate_limiter.acquire()
|
||||
devices = rest_client.get_customer_devices(customer_id=customer, page=0, page_size=1000)
|
||||
|
||||
if report["filterDevicesIn"]:
|
||||
devices.data = [device for device in devices.data if device.id.id in report["filterDevicesIn"]]
|
||||
if report["filterDevicesOut"]:
|
||||
devices.data = [device for device in devices.data if device.id.id not in report["filterDevicesOut"]]
|
||||
#Create the report in reportData if needed
|
||||
|
||||
if not reportData.get(report["name"], None):
|
||||
reportData[report["name"]] = {}
|
||||
#Go through all the devices and add them and their desired data to reportData
|
||||
|
||||
for device in devices.data:
|
||||
name = device.name
|
||||
keys = rest_client.get_timeseries_keys_v1(device.id)
|
||||
#Filter keys to desired keys
|
||||
for deviceType in report["customers"][customer]["deviceTypes"]:
|
||||
if device.type == deviceType["deviceType"]:
|
||||
rate_limiter.acquire()
|
||||
keys = rest_client.get_timeseries_keys_v1(device.id)
|
||||
keys = list(filter(lambda x: x in deviceType["dataPoints"], keys))
|
||||
#Create customer if needed
|
||||
#Check for report customer
|
||||
if not reportData[report["name"]].get(report["customers"][customer]["name"], None):
|
||||
reportData[report["name"]][report["customers"][customer]["name"]] = {}
|
||||
#Check to make sure the deviceType is desired in the report for the given device
|
||||
#Check for device type in config
|
||||
if device.type in list(map(lambda x: x["deviceType"], report["customers"][customer]["deviceTypes"])):
|
||||
#Create deviceType if needed
|
||||
#Check if deviceType in report
|
||||
if not reportData[report["name"]][report["customers"][customer]["name"]].get(device.type, None):
|
||||
reportData[report["name"]][report["customers"][customer]["name"]][device.type] = {}
|
||||
reportData[report["name"]][report["customers"][customer]["name"]][device.type][device.name] = rest_client.get_latest_timeseries(entity_id=device.id , keys=",".join(keys))
|
||||
if keys:
|
||||
rate_limiter.acquire()
|
||||
deviceData = rest_client.get_latest_timeseries(entity_id=device.id, keys=",".join(keys))
|
||||
for x in report["customers"][customer]["deviceTypes"]:
|
||||
if x["deviceType"] == device.type:
|
||||
labels = x["labels"]
|
||||
labelled_data = {}
|
||||
for k,v in labels.items():
|
||||
labelled_data[v] = {}
|
||||
for k,v in deviceData.items():
|
||||
labelled_data[labels[k]] = v
|
||||
reportData[report["name"]][report["customers"][customer]["name"]][device.type][device.name] = labelled_data
|
||||
else:
|
||||
reportData[report["name"]][report["customers"][customer]["name"]][device.type][device.name] = {}
|
||||
#Sort Data
|
||||
reportDataSorted = sort_dict_keys(reportData)
|
||||
#print(json.dumps(reportDataSorted,indent=4))
|
||||
except ApiException as e:
|
||||
print(e)
|
||||
print(f"API Exception: {e}")
|
||||
except Exception as e:
|
||||
print(f"Other Exception in getting data:\n{e}")
|
||||
|
||||
|
||||
# Create an AWS SES client
|
||||
ses_client = boto3.client('ses', region_name='us-east-1')
|
||||
|
||||
|
||||
s3 = boto3.resource('s3')
|
||||
BUCKET_NAME = "thingsboard-email-reports"
|
||||
# Create a workbook for each report
|
||||
for report_name, report_data in reportData.items():
|
||||
for report_name, report_data in reportDataSorted.items():
|
||||
#will generate an email lower down
|
||||
spreadsheets = []
|
||||
# Create a worksheet for each company
|
||||
for company_name, company_data in report_data.items():
|
||||
workbook = xlsxwriter.Workbook(f"/tmp/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx")
|
||||
workbook = xlsxwriter.Workbook(f"/tmp/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx",{'strings_to_numbers': True})
|
||||
bold = workbook.add_format({'bold': True})
|
||||
# Create a sheet for each device type
|
||||
for device_type, device_data in company_data.items():
|
||||
worksheet = workbook.add_worksheet(device_type)
|
||||
|
||||
# Set the header row with device types
|
||||
device_types = list(device_data.keys())
|
||||
worksheet.write_column(1, 0, device_types,bold)
|
||||
|
||||
# Set the header column with device types
|
||||
device_names = list(device_data.keys())
|
||||
worksheet.write_column(1, 0, device_names,bold)
|
||||
|
||||
# Write the data to the sheet
|
||||
for i, (telemetry_name, telemetry_data) in enumerate(device_data.items()):
|
||||
# Set the header row with telemetry names
|
||||
@@ -82,9 +140,12 @@ def lambda_handler(event, context):
|
||||
for j, (data_name, data) in enumerate(telemetry_data.items()):
|
||||
values = [d["value"] for d in data]
|
||||
worksheet.write_row(i + 1, j+ 1, values)
|
||||
worksheet.autofit()
|
||||
worksheet.autofit()
|
||||
workbook.close()
|
||||
spreadsheets.append(workbook)
|
||||
|
||||
# Store the generated report in S3.
|
||||
s3.Object(BUCKET_NAME, f'{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx').put(Body=open(f"/tmp/{report_name}-{company_name}-{dt.today().strftime('%Y-%m-%d')}.xlsx", 'rb'))
|
||||
# Create an email message
|
||||
msg = MIMEMultipart()
|
||||
msg['Subject'] = report_name
|
||||
@@ -104,7 +165,6 @@ def lambda_handler(event, context):
|
||||
attachment.add_header('Content-Disposition', 'attachment', filename=spreadsheet.filename[5:])
|
||||
|
||||
msg.attach(attachment)
|
||||
|
||||
# Send the email using AWS SES
|
||||
response = ses_client.send_raw_email(
|
||||
|
||||
|
||||
@@ -19,12 +19,22 @@ Resources:
|
||||
Variables:
|
||||
username: henry.pump.automation@gmail.com
|
||||
password: Henry Pump @ 2022
|
||||
TBREPORTBUCKET_BUCKET_NAME: !Ref TBReportBucket
|
||||
TBREPORTBUCKET_BUCKET_ARN: !GetAtt TBReportBucket.Arn
|
||||
Architectures:
|
||||
- arm64
|
||||
CodeUri: tbreport
|
||||
Runtime: python3.12
|
||||
Handler: tbreport.lambda_handler
|
||||
Policies: AmazonSESFullAccess
|
||||
Policies:
|
||||
- AmazonSESFullAccess
|
||||
- Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- s3:PutObject
|
||||
Resource:
|
||||
- !Sub arn:${AWS::Partition}:s3:::${TBReportBucket}
|
||||
- !Sub arn:${AWS::Partition}:s3:::${TBReportBucket}/*
|
||||
Layers:
|
||||
- !Ref TBReportLayer
|
||||
TBReportLayer:
|
||||
@@ -65,4 +75,33 @@ Resources:
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action: lambda:InvokeFunction
|
||||
Resource: !GetAtt TBReport.Arn
|
||||
Resource: !GetAtt TBReport.Arn
|
||||
TBReportBucket:
|
||||
Type: AWS::S3::Bucket
|
||||
Properties:
|
||||
BucketName: !Sub thingsboard-email-reports
|
||||
BucketEncryption:
|
||||
ServerSideEncryptionConfiguration:
|
||||
- ServerSideEncryptionByDefault:
|
||||
SSEAlgorithm: aws:kms
|
||||
KMSMasterKeyID: alias/aws/s3
|
||||
PublicAccessBlockConfiguration:
|
||||
IgnorePublicAcls: true
|
||||
RestrictPublicBuckets: true
|
||||
TBReportBucketBucketPolicy:
|
||||
Type: AWS::S3::BucketPolicy
|
||||
Properties:
|
||||
Bucket: !Ref TBReportBucket
|
||||
PolicyDocument:
|
||||
Id: RequireEncryptionInTransit
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Principal: '*'
|
||||
Action: '*'
|
||||
Effect: Deny
|
||||
Resource:
|
||||
- !GetAtt TBReportBucket.Arn
|
||||
- !Sub ${TBReportBucket.Arn}/*
|
||||
Condition:
|
||||
Bool:
|
||||
aws:SecureTransport: 'false'
|
||||
Reference in New Issue
Block a user