Creates generalized script for using any devicetype
This commit is contained in:
5
.gitignore
vendored
5
.gitignore
vendored
@@ -101,3 +101,8 @@ gateways.json
|
||||
gateways.xlsx
|
||||
|
||||
currentAdvVFDIPP.json
|
||||
|
||||
files/*
|
||||
|
||||
*_to.json
|
||||
*.bak
|
||||
|
||||
89
README.md
89
README.md
@@ -1,7 +1,92 @@
|
||||
# POCloud Report Generators
|
||||
|
||||
Developed by Patrick McDonagh @patrickjmcd, Henry pump
|
||||
|
||||
## Setup
|
||||
|
||||
System variables must be set up for the script to run. Add the following lines to /etc/environment
|
||||
```
|
||||
SMTP_EMAIL="<yourSMTPemailAddress>"
|
||||
SMTP_PASSWORD="<yourSMTPpassword>"
|
||||
MESHIFY_USERNAME="<yourMeshifyUsername>"
|
||||
MESHIFY_PASSWORD="<yourMeshifyPassword>"
|
||||
```
|
||||
|
||||
## Configuration Files
|
||||
|
||||
The script relies heavily on configuration files based on the Meshify devicetype. To configure a device, create a file
|
||||
named <devicetype>_channels.json file. The file should hold a JSON list.
|
||||
|
||||
### Example Configuration File
|
||||
|
||||
```
|
||||
# testdevice_channels.json
|
||||
|
||||
[
|
||||
{
|
||||
"meshify_name": "yesterday_volume",
|
||||
"vanity_name": "Yesteday Volume"
|
||||
},
|
||||
{
|
||||
"meshify_name": "volume_flow",
|
||||
"vanity_name": "Flow Rate"
|
||||
},
|
||||
...
|
||||
]
|
||||
```
|
||||
|
||||
## Recipients File
|
||||
|
||||
In order to send emails containing the reports, configure a recipients json file named <devicetype>_to.json. The
|
||||
file should hold a JSON object.
|
||||
|
||||
### Example Recipients File
|
||||
|
||||
```
|
||||
# testdevice_to.json
|
||||
|
||||
{
|
||||
"Company 1 Name": [
|
||||
"email1@company.com",
|
||||
"email2@company.com"
|
||||
],
|
||||
"Company 2 Name": [
|
||||
"email3@company2.com",
|
||||
"email4@company2.com"
|
||||
],
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
## Running the script
|
||||
|
||||
```
|
||||
usage: reports_xlsx.py [-h] [-s] deviceType
|
||||
|
||||
positional arguments:
|
||||
deviceType Meshify device type
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-s, --send Send emails to everyone in the _to.json file
|
||||
```
|
||||
|
||||
## Configuring the script to be run via crontab
|
||||
|
||||
Open the crontab file with `crontab -e`.
|
||||
|
||||
Add the following contents:
|
||||
```
|
||||
00 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py advvfdipp --send
|
||||
01 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py ipp --send
|
||||
02 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py abbflow --send
|
||||
```
|
||||
|
||||
|
||||
# POCloud-Scraper
|
||||
Scrape production data from POCloud to push to accounting servers
|
||||
|
||||
# Setup
|
||||
## Setup
|
||||
System variables must be set up for the script to run. Add the following lines to /etc/environment
|
||||
```
|
||||
HP_SQL_USER="<yourSQLusername>"
|
||||
@@ -11,7 +96,7 @@ MESHIFY_USERNAME="<yourMeshifyUsername>"
|
||||
MESHIFY_PASSWORD="<yourMeshifyPassword>"
|
||||
```
|
||||
|
||||
# Usage
|
||||
## Usage
|
||||
It is useful to run the script and store the output in a log file.
|
||||
|
||||
## Test Mode
|
||||
|
||||
30
abbflow_channels.json
Normal file
30
abbflow_channels.json
Normal file
@@ -0,0 +1,30 @@
|
||||
[
|
||||
{
|
||||
"meshify_name": "yesterday_volume",
|
||||
"vanity_name": "Yesteday Volume"
|
||||
},
|
||||
{
|
||||
"meshify_name": "volume_flow",
|
||||
"vanity_name": "Flow Rate"
|
||||
},
|
||||
{
|
||||
"meshify_name": "differential_pressure",
|
||||
"vanity_name": "Diff. Pressure"
|
||||
},
|
||||
{
|
||||
"meshify_name": "static_pressure",
|
||||
"vanity_name": "Static Pressure"
|
||||
},
|
||||
{
|
||||
"meshify_name": "temperature",
|
||||
"vanity_name": "Temperature"
|
||||
},
|
||||
{
|
||||
"meshify_name": "battery_voltage",
|
||||
"vanity_name": "Battery Voltage"
|
||||
},
|
||||
{
|
||||
"meshify_name": "last_calculation_period_volume",
|
||||
"vanity_name": "Last Calc. Period"
|
||||
}
|
||||
]
|
||||
@@ -1,182 +0,0 @@
|
||||
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
||||
import meshify
|
||||
import json
|
||||
from os import getenv
|
||||
from sys import exit, argv
|
||||
from smtplib import SMTP
|
||||
from datetime import datetime, timedelta
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email import encoders
|
||||
from email.mime.base import MIMEBase
|
||||
from time import sleep, time
|
||||
from tzlocal import get_localzone
|
||||
|
||||
VALUES_TO_INCLUDE = [
|
||||
{'meshify_name': 'yesterday_volume', 'vanity_name': 'Yesteday Volume'},
|
||||
{'meshify_name': 'volume_flow', 'vanity_name': 'Flow Rate'},
|
||||
{'meshify_name': 'differential_pressure', 'vanity_name': 'Diff. Pressure'},
|
||||
{'meshify_name': 'static_pressure', 'vanity_name': 'Static Pressure'},
|
||||
{'meshify_name': 'temperature', 'vanity_name': 'Temperature'},
|
||||
{'meshify_name': 'battery_voltage', 'vanity_name': 'Battery Voltage'},
|
||||
{'meshify_name': 'last_calculation_period_volume', 'vanity_name': 'Last Calc. Period'},
|
||||
]
|
||||
|
||||
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
||||
|
||||
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
||||
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
||||
|
||||
|
||||
def join_company_info(obj_with_companyId, company_lookup_obj):
|
||||
"""Add company information to an object with companyId property."""
|
||||
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
||||
return obj_with_companyId
|
||||
|
||||
|
||||
def filter_object_parameters(ob, list_of_parameters):
|
||||
"""Return an object of just the list of paramters."""
|
||||
new_ob = {}
|
||||
for par in list_of_parameters:
|
||||
try:
|
||||
new_ob[par] = ob[par]
|
||||
except KeyError:
|
||||
new_ob[par] = None
|
||||
return new_ob
|
||||
|
||||
|
||||
def group_by_company(devices):
|
||||
"""Group a list of devices by company."""
|
||||
grouped = {}
|
||||
for dev in devices:
|
||||
try:
|
||||
grouped[dev['company']['name']].append(dev)
|
||||
except KeyError:
|
||||
grouped[dev['company']['name']] = [dev]
|
||||
return grouped
|
||||
|
||||
|
||||
def main(device_type_name, sendEmail=False):
|
||||
"""Get the data and optionally send an email."""
|
||||
if sendEmail:
|
||||
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
||||
print("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
||||
exit()
|
||||
|
||||
devicetypes = meshify.query_meshify_api("devicetypes")
|
||||
filtered_devicetype = meshify.find_by_name(device_type_name, devicetypes)
|
||||
|
||||
companies = meshify.query_meshify_api("companies")
|
||||
company_lookup = {}
|
||||
for x in companies:
|
||||
company_lookup[x['id']] = x
|
||||
|
||||
devices = meshify.query_meshify_api("devices")
|
||||
filtered_devices = filter(lambda x: x['deviceTypeId'] == filtered_devicetype['id'], devices)
|
||||
filtered_devices = [join_company_info(x, company_lookup) for x in filtered_devices]
|
||||
|
||||
for i in range(0, len(filtered_devices)):
|
||||
filtered_devices[i]['values'] = filter_object_parameters(
|
||||
meshify.query_meshify_api("devices/{}/values".format(filtered_devices[i]['id'])), MESHIFY_NAMES)
|
||||
filtered_devices = group_by_company(filtered_devices)
|
||||
|
||||
totals = {}
|
||||
for comp in filtered_devices:
|
||||
total = {}
|
||||
average = {}
|
||||
for v in MESHIFY_NAMES:
|
||||
total[v] = 0.0
|
||||
average[v] = 0.0
|
||||
for dev in filtered_devices[comp]:
|
||||
for v in MESHIFY_NAMES:
|
||||
try:
|
||||
total[v] += float(dev['values'][v]['value'])
|
||||
except ValueError:
|
||||
# print("Can't make a total for {}".format(v))
|
||||
pass
|
||||
totals[comp] = total
|
||||
|
||||
for comp in filtered_devices:
|
||||
local_tz = get_localzone()
|
||||
now_t = time()
|
||||
now_dt = datetime.utcfromtimestamp(now_t)
|
||||
now = local_tz.localize(now_dt)
|
||||
|
||||
total = []
|
||||
average = []
|
||||
|
||||
header = "Well Name,"
|
||||
for v in VALUES_TO_INCLUDE:
|
||||
header += "{},".format(v['vanity_name'])
|
||||
header = header[:-1] + "\n"
|
||||
|
||||
values = ""
|
||||
for dev in sorted(filtered_devices[comp], key=lambda x: x['vanityName']):
|
||||
values += dev['vanityName'] + ","
|
||||
for v in MESHIFY_NAMES:
|
||||
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
||||
dt_loc = local_tz.localize(dt_ts)
|
||||
stale = (now - dt_loc) > timedelta(hours=24)
|
||||
|
||||
try:
|
||||
v = str(round(float(dev['values'][v]['value']), 3))
|
||||
if stale:
|
||||
v += " (STALE)"
|
||||
values += '{},'.format(v)
|
||||
except ValueError:
|
||||
v = str(dev['values'][v]['value'])
|
||||
if stale:
|
||||
v += " (STALE)"
|
||||
values += '{},'.format(v)
|
||||
|
||||
# values += '{},'.format(dt)
|
||||
values = values[:-1] + "\n"
|
||||
|
||||
if sendEmail:
|
||||
|
||||
with open("{}_to.json".format(device_type_name), 'r') as to_file:
|
||||
to_lookup = json.load(to_file)
|
||||
try:
|
||||
email_to = to_lookup[comp]
|
||||
except KeyError:
|
||||
print("[{}] No recipients for that company ({})!".format(datetime.now().isoformat(), comp))
|
||||
continue
|
||||
# part1 = MIMEText(header + values, "plain")
|
||||
attachment = MIMEBase('application', 'octet-stream')
|
||||
attachment.set_payload(header + values)
|
||||
encoders.encode_base64(attachment)
|
||||
|
||||
now = datetime.now()
|
||||
datestr = now.strftime("%a %b %d, %Y")
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg['Subject'] = "{} SAMPLE Daily ABB Flowmeter Report for {}".format(comp, datestr)
|
||||
msg['From'] = "alerts@henry-pump.com"
|
||||
msg['To'] = ", ".join(email_to)
|
||||
|
||||
filename = "{} {}.csv".format(comp, datestr)
|
||||
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
||||
|
||||
# msg.attach(part1)
|
||||
# msg.attach(part2)
|
||||
msg.attach(attachment)
|
||||
|
||||
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
||||
s.starttls()
|
||||
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
||||
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
||||
print("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
||||
sleep(2)
|
||||
|
||||
with open('{}-{}.csv'.format(comp, device_type_name), 'w') as csvfile:
|
||||
csvfile.write(header + values)
|
||||
|
||||
filtered_devices["totals"] = totals
|
||||
with open("current_{}.json".format(device_type_name), 'w') as jsonfile:
|
||||
json.dump(filtered_devices, jsonfile, indent=4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(argv) > 1:
|
||||
s = argv[1] == "true"
|
||||
main("abbflow", sendEmail=s)
|
||||
else:
|
||||
main("abbflow", sendEmail=False)
|
||||
42
advvfdipp_channels.json
Normal file
42
advvfdipp_channels.json
Normal file
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"meshify_name": "wellstatus",
|
||||
"vanity_name": "Well Status"
|
||||
},
|
||||
{
|
||||
"meshify_name": "flowtotalyesterday",
|
||||
"vanity_name": "Flow Total (Yesterday)"
|
||||
},
|
||||
{
|
||||
"meshify_name": "energytotalyesterday",
|
||||
"vanity_name": "Energy Total (Yesterday)"
|
||||
},
|
||||
{
|
||||
"meshify_name": "fluidlevel",
|
||||
"vanity_name": "Fluid Level"
|
||||
},
|
||||
{
|
||||
"meshify_name": "flowrate",
|
||||
"vanity_name": "Flow Rate"
|
||||
},
|
||||
{
|
||||
"meshify_name": "pidcontrolmode",
|
||||
"vanity_name": "PID Control Mode"
|
||||
},
|
||||
{
|
||||
"meshify_name": "downholesensorstatus",
|
||||
"vanity_name": "DH Sensor Status"
|
||||
},
|
||||
{
|
||||
"meshify_name": "intakepressure",
|
||||
"vanity_name": "Intake Pressure"
|
||||
},
|
||||
{
|
||||
"meshify_name": "intaketemperature",
|
||||
"vanity_name": "Intake Temperature"
|
||||
},
|
||||
{
|
||||
"meshify_name": "tubingpressure",
|
||||
"vanity_name": "Tubing Pressure"
|
||||
}
|
||||
]
|
||||
@@ -1,186 +0,0 @@
|
||||
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
||||
import meshify
|
||||
import json
|
||||
from os import getenv
|
||||
from sys import exit, argv
|
||||
from smtplib import SMTP
|
||||
from datetime import datetime, timedelta
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email import encoders
|
||||
from email.mime.base import MIMEBase
|
||||
from time import sleep, time
|
||||
from tzlocal import get_localzone
|
||||
|
||||
VALUES_TO_INCLUDE = [
|
||||
{'meshify_name': 'wellstatus', 'vanity_name': 'Well Status'},
|
||||
{'meshify_name': 'flowtotalyesterday', 'vanity_name': 'Flow Total (Yesterday)'},
|
||||
{'meshify_name': 'energytotalyesterday', 'vanity_name': 'Energy Total (Yesterday)'},
|
||||
{'meshify_name': 'fluidlevel', 'vanity_name': 'Fluid Level'},
|
||||
{'meshify_name': 'flowrate', 'vanity_name': 'Flow Rate'},
|
||||
{'meshify_name': 'pidcontrolmode', 'vanity_name': 'PID Control Mode'},
|
||||
{'meshify_name': 'downholesensorstatus', 'vanity_name': 'DH Sensor Status'},
|
||||
{'meshify_name': 'intakepressure', 'vanity_name': 'Intake Pressure'},
|
||||
{'meshify_name': 'intaketemperature', 'vanity_name': 'Intake Temperature'},
|
||||
{'meshify_name': 'tubingpressure', 'vanity_name': 'Tubing Pressure'},
|
||||
]
|
||||
|
||||
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
||||
|
||||
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
||||
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
||||
|
||||
|
||||
def join_company_info(obj_with_companyId, company_lookup_obj):
|
||||
"""Add company information to an object with companyId property."""
|
||||
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
||||
return obj_with_companyId
|
||||
|
||||
|
||||
def filter_object_parameters(ob, list_of_parameters):
|
||||
"""Return an object of just the list of paramters."""
|
||||
new_ob = {}
|
||||
for par in list_of_parameters:
|
||||
try:
|
||||
new_ob[par] = ob[par]
|
||||
except KeyError:
|
||||
new_ob[par] = None
|
||||
return new_ob
|
||||
|
||||
|
||||
def group_by_company(devices):
|
||||
"""Group a list of devices by company."""
|
||||
grouped = {}
|
||||
for dev in devices:
|
||||
try:
|
||||
grouped[dev['company']['name']].append(dev)
|
||||
except KeyError:
|
||||
grouped[dev['company']['name']] = [dev]
|
||||
return grouped
|
||||
|
||||
|
||||
def main(sendEmail=False):
|
||||
"""Get the data and optionally send an email."""
|
||||
if sendEmail:
|
||||
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
||||
print("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
||||
exit()
|
||||
|
||||
devicetypes = meshify.query_meshify_api("devicetypes")
|
||||
advvfdipp_devicetype = meshify.find_by_name("advvfdipp", devicetypes)
|
||||
|
||||
companies = meshify.query_meshify_api("companies")
|
||||
company_lookup = {}
|
||||
for x in companies:
|
||||
company_lookup[x['id']] = x
|
||||
|
||||
devices = meshify.query_meshify_api("devices")
|
||||
advvfdipp_devices = filter(lambda x: x['deviceTypeId'] == advvfdipp_devicetype['id'], devices)
|
||||
advvfdipp_devices = [join_company_info(x, company_lookup) for x in advvfdipp_devices]
|
||||
|
||||
for i in range(0, len(advvfdipp_devices)):
|
||||
advvfdipp_devices[i]['values'] = filter_object_parameters(
|
||||
meshify.query_meshify_api("devices/{}/values".format(advvfdipp_devices[i]['id'])), MESHIFY_NAMES)
|
||||
advvfdipp_devices = group_by_company(advvfdipp_devices)
|
||||
|
||||
totals = {}
|
||||
for comp in advvfdipp_devices:
|
||||
total = {}
|
||||
average = {}
|
||||
for v in MESHIFY_NAMES:
|
||||
total[v] = 0.0
|
||||
average[v] = 0.0
|
||||
for dev in advvfdipp_devices[comp]:
|
||||
for v in MESHIFY_NAMES:
|
||||
try:
|
||||
total[v] += float(dev['values'][v]['value'])
|
||||
except ValueError:
|
||||
# print("Can't make a total for {}".format(v))
|
||||
pass
|
||||
totals[comp] = total
|
||||
|
||||
for comp in advvfdipp_devices:
|
||||
local_tz = get_localzone()
|
||||
now_t = time()
|
||||
now_dt = datetime.utcfromtimestamp(now_t)
|
||||
now = local_tz.localize(now_dt)
|
||||
|
||||
total = []
|
||||
average = []
|
||||
|
||||
header = "Well Name,"
|
||||
for v in VALUES_TO_INCLUDE:
|
||||
header += "{},".format(v['vanity_name'])
|
||||
header = header[:-1] + "\n"
|
||||
|
||||
values = ""
|
||||
for dev in sorted(advvfdipp_devices[comp], key=lambda x: x['vanityName']):
|
||||
values += dev['vanityName'] + ","
|
||||
for v in MESHIFY_NAMES:
|
||||
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
||||
dt_loc = local_tz.localize(dt_ts)
|
||||
stale = (now - dt_loc) > timedelta(hours=24)
|
||||
|
||||
try:
|
||||
v = str(round(float(dev['values'][v]['value']), 3))
|
||||
if stale:
|
||||
v += " (STALE)"
|
||||
values += '{},'.format(v)
|
||||
except ValueError:
|
||||
v = str(dev['values'][v]['value'])
|
||||
if stale:
|
||||
v += " (STALE)"
|
||||
values += '{},'.format(v)
|
||||
|
||||
# values += '{},'.format(dt)
|
||||
values = values[:-1] + "\n"
|
||||
|
||||
if sendEmail:
|
||||
|
||||
with open("advvfdipp_to.json", 'r') as to_file:
|
||||
to_lookup = json.load(to_file)
|
||||
try:
|
||||
email_to = to_lookup[comp]
|
||||
except KeyError:
|
||||
print("[{}] No recipients for that company({})!".format(datetime.now().isoformat(), comp))
|
||||
continue
|
||||
# part1 = MIMEText(header + values, "plain")
|
||||
attachment = MIMEBase('application', 'octet-stream')
|
||||
attachment.set_payload(header + values)
|
||||
encoders.encode_base64(attachment)
|
||||
|
||||
now = datetime.now()
|
||||
datestr = now.strftime("%a %b %d, %Y")
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg['Subject'] = "{} SAMPLE Daily Adv. VFD IPP Report for {}".format(comp, datestr)
|
||||
msg['From'] = "alerts@henry-pump.com"
|
||||
msg['To'] = ", ".join(email_to)
|
||||
|
||||
filename = "{} {}.csv".format(comp, datestr)
|
||||
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
||||
|
||||
# msg.attach(part1)
|
||||
# msg.attach(part2)
|
||||
msg.attach(attachment)
|
||||
|
||||
# s = SMTP(host="secure.emailsrvr.com", port=25)
|
||||
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
||||
s.starttls()
|
||||
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
||||
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
||||
print("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
||||
sleep(2)
|
||||
|
||||
with open('{}.csv'.format(comp), 'w') as csvfile:
|
||||
csvfile.write(header + values)
|
||||
|
||||
advvfdipp_devices["totals"] = totals
|
||||
with open("currentAdvVFDIPP.json", 'w') as jsonfile:
|
||||
json.dump(advvfdipp_devices, jsonfile, indent=4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(argv) > 1:
|
||||
s = argv[1] == "true"
|
||||
main(sendEmail=s)
|
||||
else:
|
||||
main(sendEmail=False)
|
||||
@@ -1,10 +1,11 @@
|
||||
{
|
||||
"Henry Resources Water": [
|
||||
"pmcdonagh@henry-pump.com",
|
||||
"thardway@henry-pump.com"
|
||||
"pmcdonagh@henry-pump.com"
|
||||
],
|
||||
"CrownQuest": [
|
||||
"pmcdonagh@henry-pump.com",
|
||||
"thardway@henry-pump.com"
|
||||
"pmcdonagh@henry-pump.com"
|
||||
],
|
||||
"QEP Resources": [
|
||||
"pmcdonagh@henry-pump.com"
|
||||
]
|
||||
}
|
||||
|
||||
26
ipp_channels.json
Normal file
26
ipp_channels.json
Normal file
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"meshify_name": "devicestatus",
|
||||
"vanity_name": "Well Status"
|
||||
},
|
||||
{
|
||||
"meshify_name": "flowrate",
|
||||
"vanity_name": "Flow Rate"
|
||||
},
|
||||
{
|
||||
"meshify_name": "dhdownholestatusint",
|
||||
"vanity_name": "DH Sensor Status"
|
||||
},
|
||||
{
|
||||
"meshify_name": "dhintakepressure",
|
||||
"vanity_name": "Intake Pressure"
|
||||
},
|
||||
{
|
||||
"meshify_name": "dhintaketemperature",
|
||||
"vanity_name": "Intake Temperature"
|
||||
},
|
||||
{
|
||||
"meshify_name": "pressurein",
|
||||
"vanity_name": "Tubing Pressure"
|
||||
}
|
||||
]
|
||||
201
reports_xlsx.py
Normal file
201
reports_xlsx.py
Normal file
@@ -0,0 +1,201 @@
|
||||
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
||||
import meshify
|
||||
import json
|
||||
from os import getenv
|
||||
from sys import exit, stdout
|
||||
from smtplib import SMTP
|
||||
from datetime import datetime, timedelta
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email import encoders
|
||||
from email.mime.base import MIMEBase
|
||||
from time import sleep, time
|
||||
from tzlocal import get_localzone
|
||||
import xlsxwriter
|
||||
import argparse
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
|
||||
logger = ""
|
||||
DEVICE_TYPE_NAME = ""
|
||||
datestring = datetime.now().strftime("%Y%m%d")
|
||||
|
||||
VALUES_TO_INCLUDE = []
|
||||
MESHIFY_NAMES = []
|
||||
|
||||
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
||||
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
||||
|
||||
|
||||
def join_company_info(obj_with_companyId, company_lookup_obj):
|
||||
"""Add company information to an object with companyId property."""
|
||||
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
||||
return obj_with_companyId
|
||||
|
||||
|
||||
def filter_object_parameters(ob, list_of_parameters):
|
||||
"""Return an object of just the list of paramters."""
|
||||
new_ob = {}
|
||||
for par in list_of_parameters:
|
||||
try:
|
||||
new_ob[par] = ob[par]
|
||||
except KeyError:
|
||||
new_ob[par] = None
|
||||
return new_ob
|
||||
|
||||
|
||||
def group_by_company(devices):
|
||||
"""Group a list of devices by company."""
|
||||
grouped = {}
|
||||
for dev in devices:
|
||||
try:
|
||||
grouped[dev['company']['name']].append(dev)
|
||||
except KeyError:
|
||||
grouped[dev['company']['name']] = [dev]
|
||||
return grouped
|
||||
|
||||
|
||||
def main(sendEmail=False):
|
||||
"""Get the data and optionally send an email."""
|
||||
if sendEmail:
|
||||
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
||||
logger.error("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
||||
exit()
|
||||
|
||||
devicetypes = meshify.query_meshify_api("devicetypes")
|
||||
this_devicetype = meshify.find_by_name(DEVICE_TYPE_NAME, devicetypes)
|
||||
|
||||
companies = meshify.query_meshify_api("companies")
|
||||
company_lookup = {}
|
||||
for x in companies:
|
||||
company_lookup[x['id']] = x
|
||||
|
||||
devices = meshify.query_meshify_api("devices")
|
||||
this_devices = filter(lambda x: x['deviceTypeId'] == this_devicetype['id'], devices)
|
||||
this_devices = [join_company_info(x, company_lookup) for x in this_devices]
|
||||
|
||||
for i in range(0, len(this_devices)):
|
||||
this_devices[i]['values'] = filter_object_parameters(
|
||||
meshify.query_meshify_api("devices/{}/values".format(this_devices[i]['id'])), MESHIFY_NAMES)
|
||||
this_devices = group_by_company(this_devices)
|
||||
|
||||
for comp in this_devices:
|
||||
local_tz = get_localzone()
|
||||
now_t = time()
|
||||
now_dt = datetime.utcfromtimestamp(now_t)
|
||||
now = local_tz.localize(now_dt)
|
||||
|
||||
workbook = xlsxwriter.Workbook("files/{}_{}_{}.xlsx".format(DEVICE_TYPE_NAME, comp, datestring))
|
||||
worksheet = workbook.add_worksheet()
|
||||
worksheet.set_column('A:A', 20)
|
||||
|
||||
bold = workbook.add_format({'bold': True})
|
||||
red = workbook.add_format({'font_color': 'red'})
|
||||
|
||||
worksheet.write(0, 0, "Well Name", bold)
|
||||
for i in range(0, len(VALUES_TO_INCLUDE)):
|
||||
worksheet.write(0, i+1, VALUES_TO_INCLUDE[i]['vanity_name'], bold)
|
||||
|
||||
sorted_company_devices = sorted(this_devices[comp], key=lambda x: x['vanityName'])
|
||||
for j in range(0, len(sorted_company_devices)):
|
||||
dev = sorted_company_devices[j]
|
||||
worksheet.write(j+1, 0, dev['vanityName'])
|
||||
for k in range(0, len(MESHIFY_NAMES)):
|
||||
v = MESHIFY_NAMES[k]
|
||||
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
||||
dt_loc = local_tz.localize(dt_ts)
|
||||
stale = (now - dt_loc) > timedelta(hours=24)
|
||||
|
||||
try:
|
||||
v = round(float(dev['values'][v]['value']), 3)
|
||||
if stale:
|
||||
worksheet.write_number(j+1, 1+k, v, red)
|
||||
else:
|
||||
worksheet.write_number(j+1, 1+k, v)
|
||||
except ValueError:
|
||||
v = str(dev['values'][v]['value'])
|
||||
if stale:
|
||||
worksheet.write(j+1, 1+k, v, red)
|
||||
else:
|
||||
worksheet.write(j+1, 1+k, v)
|
||||
|
||||
workbook.close()
|
||||
|
||||
if sendEmail:
|
||||
|
||||
with open("{}_to.json".format(DEVICE_TYPE_NAME), 'r') as to_file:
|
||||
to_lookup = json.load(to_file)
|
||||
try:
|
||||
email_to = to_lookup[comp]
|
||||
except KeyError:
|
||||
logger.error("[{}] No recipients for that company({})!".format(datetime.now().isoformat(), comp))
|
||||
continue
|
||||
# part1 = MIMEText(header + values, "plain")
|
||||
attachment = MIMEBase('application', 'octet-stream')
|
||||
attachment.set_payload(open("files/{}_{}_{}.xlsx".format(DEVICE_TYPE_NAME, comp, datestring), "rb").read())
|
||||
encoders.encode_base64(attachment)
|
||||
|
||||
now = datetime.now()
|
||||
datestr = now.strftime("%a %b %d, %Y")
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg['Subject'] = "{} Daily Adv. VFD IPP Report for {}".format(comp, datestr)
|
||||
msg['From'] = "alerts@henry-pump.com"
|
||||
msg['To'] = ", ".join(email_to)
|
||||
|
||||
filename = "{} {}.xlsx".format(comp, datestr)
|
||||
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
||||
|
||||
# msg.attach(part1)
|
||||
# msg.attach(part2)
|
||||
msg.attach(attachment)
|
||||
|
||||
# s = SMTP(host="secure.emailsrvr.com", port=25)
|
||||
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
||||
s.starttls()
|
||||
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
||||
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
||||
logger.info("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
||||
sleep(2)
|
||||
|
||||
|
||||
def setup_logger():
|
||||
"""Setup and return the logger module."""
|
||||
log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(funcName)s(%(lineno)d) %(message)s')
|
||||
logFile = './emailreports.log'.format(DEVICE_TYPE_NAME)
|
||||
my_handler = RotatingFileHandler(logFile, mode='a', maxBytes=500*1024, backupCount=2, encoding=None, delay=0)
|
||||
my_handler.setFormatter(log_formatter)
|
||||
my_handler.setLevel(logging.INFO)
|
||||
logger = logging.getLogger("emailreports")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.addHandler(my_handler)
|
||||
|
||||
console_out = logging.StreamHandler(stdout)
|
||||
console_out.setFormatter(log_formatter)
|
||||
logger.addHandler(console_out)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('deviceType', help="Meshify device type")
|
||||
parser.add_argument('-s', '--send', action='store_true', help="Send emails to everyone in the _to.json file")
|
||||
|
||||
args = parser.parse_args()
|
||||
DEVICE_TYPE_NAME = args.deviceType
|
||||
SEND_EMAIL = args.send
|
||||
|
||||
logger = setup_logger()
|
||||
|
||||
try:
|
||||
with open("{}_channels.json".format(DEVICE_TYPE_NAME), 'r') as channel_file:
|
||||
VALUES_TO_INCLUDE = json.load(channel_file)
|
||||
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
||||
except IOError:
|
||||
logger.error("No channel file named {}_channels.json".format(DEVICE_TYPE_NAME))
|
||||
exit()
|
||||
except ValueError as e:
|
||||
logger.error("Channel file {}_channels.json is misformed: {}".format(DEVICE_TYPE_NAME, e))
|
||||
exit()
|
||||
|
||||
main(sendEmail=SEND_EMAIL)
|
||||
Reference in New Issue
Block a user