Creates generalized script for using any devicetype
This commit is contained in:
5
.gitignore
vendored
5
.gitignore
vendored
@@ -101,3 +101,8 @@ gateways.json
|
|||||||
gateways.xlsx
|
gateways.xlsx
|
||||||
|
|
||||||
currentAdvVFDIPP.json
|
currentAdvVFDIPP.json
|
||||||
|
|
||||||
|
files/*
|
||||||
|
|
||||||
|
*_to.json
|
||||||
|
*.bak
|
||||||
|
|||||||
89
README.md
89
README.md
@@ -1,7 +1,92 @@
|
|||||||
|
# POCloud Report Generators
|
||||||
|
|
||||||
|
Developed by Patrick McDonagh @patrickjmcd, Henry pump
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
System variables must be set up for the script to run. Add the following lines to /etc/environment
|
||||||
|
```
|
||||||
|
SMTP_EMAIL="<yourSMTPemailAddress>"
|
||||||
|
SMTP_PASSWORD="<yourSMTPpassword>"
|
||||||
|
MESHIFY_USERNAME="<yourMeshifyUsername>"
|
||||||
|
MESHIFY_PASSWORD="<yourMeshifyPassword>"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration Files
|
||||||
|
|
||||||
|
The script relies heavily on configuration files based on the Meshify devicetype. To configure a device, create a file
|
||||||
|
named <devicetype>_channels.json file. The file should hold a JSON list.
|
||||||
|
|
||||||
|
### Example Configuration File
|
||||||
|
|
||||||
|
```
|
||||||
|
# testdevice_channels.json
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"meshify_name": "yesterday_volume",
|
||||||
|
"vanity_name": "Yesteday Volume"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "volume_flow",
|
||||||
|
"vanity_name": "Flow Rate"
|
||||||
|
},
|
||||||
|
...
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
## Recipients File
|
||||||
|
|
||||||
|
In order to send emails containing the reports, configure a recipients json file named <devicetype>_to.json. The
|
||||||
|
file should hold a JSON object.
|
||||||
|
|
||||||
|
### Example Recipients File
|
||||||
|
|
||||||
|
```
|
||||||
|
# testdevice_to.json
|
||||||
|
|
||||||
|
{
|
||||||
|
"Company 1 Name": [
|
||||||
|
"email1@company.com",
|
||||||
|
"email2@company.com"
|
||||||
|
],
|
||||||
|
"Company 2 Name": [
|
||||||
|
"email3@company2.com",
|
||||||
|
"email4@company2.com"
|
||||||
|
],
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running the script
|
||||||
|
|
||||||
|
```
|
||||||
|
usage: reports_xlsx.py [-h] [-s] deviceType
|
||||||
|
|
||||||
|
positional arguments:
|
||||||
|
deviceType Meshify device type
|
||||||
|
|
||||||
|
optional arguments:
|
||||||
|
-h, --help show this help message and exit
|
||||||
|
-s, --send Send emails to everyone in the _to.json file
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuring the script to be run via crontab
|
||||||
|
|
||||||
|
Open the crontab file with `crontab -e`.
|
||||||
|
|
||||||
|
Add the following contents:
|
||||||
|
```
|
||||||
|
00 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py advvfdipp --send
|
||||||
|
01 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py ipp --send
|
||||||
|
02 07 * * * /usr/bin/python3 /home/ubuntu/reports_xlsx.py abbflow --send
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
# POCloud-Scraper
|
# POCloud-Scraper
|
||||||
Scrape production data from POCloud to push to accounting servers
|
Scrape production data from POCloud to push to accounting servers
|
||||||
|
|
||||||
# Setup
|
## Setup
|
||||||
System variables must be set up for the script to run. Add the following lines to /etc/environment
|
System variables must be set up for the script to run. Add the following lines to /etc/environment
|
||||||
```
|
```
|
||||||
HP_SQL_USER="<yourSQLusername>"
|
HP_SQL_USER="<yourSQLusername>"
|
||||||
@@ -11,7 +96,7 @@ MESHIFY_USERNAME="<yourMeshifyUsername>"
|
|||||||
MESHIFY_PASSWORD="<yourMeshifyPassword>"
|
MESHIFY_PASSWORD="<yourMeshifyPassword>"
|
||||||
```
|
```
|
||||||
|
|
||||||
# Usage
|
## Usage
|
||||||
It is useful to run the script and store the output in a log file.
|
It is useful to run the script and store the output in a log file.
|
||||||
|
|
||||||
## Test Mode
|
## Test Mode
|
||||||
|
|||||||
30
abbflow_channels.json
Normal file
30
abbflow_channels.json
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"meshify_name": "yesterday_volume",
|
||||||
|
"vanity_name": "Yesteday Volume"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "volume_flow",
|
||||||
|
"vanity_name": "Flow Rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "differential_pressure",
|
||||||
|
"vanity_name": "Diff. Pressure"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "static_pressure",
|
||||||
|
"vanity_name": "Static Pressure"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "temperature",
|
||||||
|
"vanity_name": "Temperature"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "battery_voltage",
|
||||||
|
"vanity_name": "Battery Voltage"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "last_calculation_period_volume",
|
||||||
|
"vanity_name": "Last Calc. Period"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
|
||||||
import meshify
|
|
||||||
import json
|
|
||||||
from os import getenv
|
|
||||||
from sys import exit, argv
|
|
||||||
from smtplib import SMTP
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from email.mime.multipart import MIMEMultipart
|
|
||||||
from email import encoders
|
|
||||||
from email.mime.base import MIMEBase
|
|
||||||
from time import sleep, time
|
|
||||||
from tzlocal import get_localzone
|
|
||||||
|
|
||||||
VALUES_TO_INCLUDE = [
|
|
||||||
{'meshify_name': 'yesterday_volume', 'vanity_name': 'Yesteday Volume'},
|
|
||||||
{'meshify_name': 'volume_flow', 'vanity_name': 'Flow Rate'},
|
|
||||||
{'meshify_name': 'differential_pressure', 'vanity_name': 'Diff. Pressure'},
|
|
||||||
{'meshify_name': 'static_pressure', 'vanity_name': 'Static Pressure'},
|
|
||||||
{'meshify_name': 'temperature', 'vanity_name': 'Temperature'},
|
|
||||||
{'meshify_name': 'battery_voltage', 'vanity_name': 'Battery Voltage'},
|
|
||||||
{'meshify_name': 'last_calculation_period_volume', 'vanity_name': 'Last Calc. Period'},
|
|
||||||
]
|
|
||||||
|
|
||||||
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
|
||||||
|
|
||||||
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
|
||||||
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
|
||||||
|
|
||||||
|
|
||||||
def join_company_info(obj_with_companyId, company_lookup_obj):
|
|
||||||
"""Add company information to an object with companyId property."""
|
|
||||||
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
|
||||||
return obj_with_companyId
|
|
||||||
|
|
||||||
|
|
||||||
def filter_object_parameters(ob, list_of_parameters):
|
|
||||||
"""Return an object of just the list of paramters."""
|
|
||||||
new_ob = {}
|
|
||||||
for par in list_of_parameters:
|
|
||||||
try:
|
|
||||||
new_ob[par] = ob[par]
|
|
||||||
except KeyError:
|
|
||||||
new_ob[par] = None
|
|
||||||
return new_ob
|
|
||||||
|
|
||||||
|
|
||||||
def group_by_company(devices):
|
|
||||||
"""Group a list of devices by company."""
|
|
||||||
grouped = {}
|
|
||||||
for dev in devices:
|
|
||||||
try:
|
|
||||||
grouped[dev['company']['name']].append(dev)
|
|
||||||
except KeyError:
|
|
||||||
grouped[dev['company']['name']] = [dev]
|
|
||||||
return grouped
|
|
||||||
|
|
||||||
|
|
||||||
def main(device_type_name, sendEmail=False):
|
|
||||||
"""Get the data and optionally send an email."""
|
|
||||||
if sendEmail:
|
|
||||||
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
|
||||||
print("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
|
||||||
exit()
|
|
||||||
|
|
||||||
devicetypes = meshify.query_meshify_api("devicetypes")
|
|
||||||
filtered_devicetype = meshify.find_by_name(device_type_name, devicetypes)
|
|
||||||
|
|
||||||
companies = meshify.query_meshify_api("companies")
|
|
||||||
company_lookup = {}
|
|
||||||
for x in companies:
|
|
||||||
company_lookup[x['id']] = x
|
|
||||||
|
|
||||||
devices = meshify.query_meshify_api("devices")
|
|
||||||
filtered_devices = filter(lambda x: x['deviceTypeId'] == filtered_devicetype['id'], devices)
|
|
||||||
filtered_devices = [join_company_info(x, company_lookup) for x in filtered_devices]
|
|
||||||
|
|
||||||
for i in range(0, len(filtered_devices)):
|
|
||||||
filtered_devices[i]['values'] = filter_object_parameters(
|
|
||||||
meshify.query_meshify_api("devices/{}/values".format(filtered_devices[i]['id'])), MESHIFY_NAMES)
|
|
||||||
filtered_devices = group_by_company(filtered_devices)
|
|
||||||
|
|
||||||
totals = {}
|
|
||||||
for comp in filtered_devices:
|
|
||||||
total = {}
|
|
||||||
average = {}
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
total[v] = 0.0
|
|
||||||
average[v] = 0.0
|
|
||||||
for dev in filtered_devices[comp]:
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
try:
|
|
||||||
total[v] += float(dev['values'][v]['value'])
|
|
||||||
except ValueError:
|
|
||||||
# print("Can't make a total for {}".format(v))
|
|
||||||
pass
|
|
||||||
totals[comp] = total
|
|
||||||
|
|
||||||
for comp in filtered_devices:
|
|
||||||
local_tz = get_localzone()
|
|
||||||
now_t = time()
|
|
||||||
now_dt = datetime.utcfromtimestamp(now_t)
|
|
||||||
now = local_tz.localize(now_dt)
|
|
||||||
|
|
||||||
total = []
|
|
||||||
average = []
|
|
||||||
|
|
||||||
header = "Well Name,"
|
|
||||||
for v in VALUES_TO_INCLUDE:
|
|
||||||
header += "{},".format(v['vanity_name'])
|
|
||||||
header = header[:-1] + "\n"
|
|
||||||
|
|
||||||
values = ""
|
|
||||||
for dev in sorted(filtered_devices[comp], key=lambda x: x['vanityName']):
|
|
||||||
values += dev['vanityName'] + ","
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
|
||||||
dt_loc = local_tz.localize(dt_ts)
|
|
||||||
stale = (now - dt_loc) > timedelta(hours=24)
|
|
||||||
|
|
||||||
try:
|
|
||||||
v = str(round(float(dev['values'][v]['value']), 3))
|
|
||||||
if stale:
|
|
||||||
v += " (STALE)"
|
|
||||||
values += '{},'.format(v)
|
|
||||||
except ValueError:
|
|
||||||
v = str(dev['values'][v]['value'])
|
|
||||||
if stale:
|
|
||||||
v += " (STALE)"
|
|
||||||
values += '{},'.format(v)
|
|
||||||
|
|
||||||
# values += '{},'.format(dt)
|
|
||||||
values = values[:-1] + "\n"
|
|
||||||
|
|
||||||
if sendEmail:
|
|
||||||
|
|
||||||
with open("{}_to.json".format(device_type_name), 'r') as to_file:
|
|
||||||
to_lookup = json.load(to_file)
|
|
||||||
try:
|
|
||||||
email_to = to_lookup[comp]
|
|
||||||
except KeyError:
|
|
||||||
print("[{}] No recipients for that company ({})!".format(datetime.now().isoformat(), comp))
|
|
||||||
continue
|
|
||||||
# part1 = MIMEText(header + values, "plain")
|
|
||||||
attachment = MIMEBase('application', 'octet-stream')
|
|
||||||
attachment.set_payload(header + values)
|
|
||||||
encoders.encode_base64(attachment)
|
|
||||||
|
|
||||||
now = datetime.now()
|
|
||||||
datestr = now.strftime("%a %b %d, %Y")
|
|
||||||
msg = MIMEMultipart('alternative')
|
|
||||||
msg['Subject'] = "{} SAMPLE Daily ABB Flowmeter Report for {}".format(comp, datestr)
|
|
||||||
msg['From'] = "alerts@henry-pump.com"
|
|
||||||
msg['To'] = ", ".join(email_to)
|
|
||||||
|
|
||||||
filename = "{} {}.csv".format(comp, datestr)
|
|
||||||
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
|
||||||
|
|
||||||
# msg.attach(part1)
|
|
||||||
# msg.attach(part2)
|
|
||||||
msg.attach(attachment)
|
|
||||||
|
|
||||||
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
|
||||||
s.starttls()
|
|
||||||
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
|
||||||
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
|
||||||
print("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
|
||||||
sleep(2)
|
|
||||||
|
|
||||||
with open('{}-{}.csv'.format(comp, device_type_name), 'w') as csvfile:
|
|
||||||
csvfile.write(header + values)
|
|
||||||
|
|
||||||
filtered_devices["totals"] = totals
|
|
||||||
with open("current_{}.json".format(device_type_name), 'w') as jsonfile:
|
|
||||||
json.dump(filtered_devices, jsonfile, indent=4)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(argv) > 1:
|
|
||||||
s = argv[1] == "true"
|
|
||||||
main("abbflow", sendEmail=s)
|
|
||||||
else:
|
|
||||||
main("abbflow", sendEmail=False)
|
|
||||||
42
advvfdipp_channels.json
Normal file
42
advvfdipp_channels.json
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"meshify_name": "wellstatus",
|
||||||
|
"vanity_name": "Well Status"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "flowtotalyesterday",
|
||||||
|
"vanity_name": "Flow Total (Yesterday)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "energytotalyesterday",
|
||||||
|
"vanity_name": "Energy Total (Yesterday)"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "fluidlevel",
|
||||||
|
"vanity_name": "Fluid Level"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "flowrate",
|
||||||
|
"vanity_name": "Flow Rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "pidcontrolmode",
|
||||||
|
"vanity_name": "PID Control Mode"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "downholesensorstatus",
|
||||||
|
"vanity_name": "DH Sensor Status"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "intakepressure",
|
||||||
|
"vanity_name": "Intake Pressure"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "intaketemperature",
|
||||||
|
"vanity_name": "Intake Temperature"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "tubingpressure",
|
||||||
|
"vanity_name": "Tubing Pressure"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,186 +0,0 @@
|
|||||||
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
|
||||||
import meshify
|
|
||||||
import json
|
|
||||||
from os import getenv
|
|
||||||
from sys import exit, argv
|
|
||||||
from smtplib import SMTP
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from email.mime.multipart import MIMEMultipart
|
|
||||||
from email import encoders
|
|
||||||
from email.mime.base import MIMEBase
|
|
||||||
from time import sleep, time
|
|
||||||
from tzlocal import get_localzone
|
|
||||||
|
|
||||||
VALUES_TO_INCLUDE = [
|
|
||||||
{'meshify_name': 'wellstatus', 'vanity_name': 'Well Status'},
|
|
||||||
{'meshify_name': 'flowtotalyesterday', 'vanity_name': 'Flow Total (Yesterday)'},
|
|
||||||
{'meshify_name': 'energytotalyesterday', 'vanity_name': 'Energy Total (Yesterday)'},
|
|
||||||
{'meshify_name': 'fluidlevel', 'vanity_name': 'Fluid Level'},
|
|
||||||
{'meshify_name': 'flowrate', 'vanity_name': 'Flow Rate'},
|
|
||||||
{'meshify_name': 'pidcontrolmode', 'vanity_name': 'PID Control Mode'},
|
|
||||||
{'meshify_name': 'downholesensorstatus', 'vanity_name': 'DH Sensor Status'},
|
|
||||||
{'meshify_name': 'intakepressure', 'vanity_name': 'Intake Pressure'},
|
|
||||||
{'meshify_name': 'intaketemperature', 'vanity_name': 'Intake Temperature'},
|
|
||||||
{'meshify_name': 'tubingpressure', 'vanity_name': 'Tubing Pressure'},
|
|
||||||
]
|
|
||||||
|
|
||||||
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
|
||||||
|
|
||||||
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
|
||||||
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
|
||||||
|
|
||||||
|
|
||||||
def join_company_info(obj_with_companyId, company_lookup_obj):
|
|
||||||
"""Add company information to an object with companyId property."""
|
|
||||||
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
|
||||||
return obj_with_companyId
|
|
||||||
|
|
||||||
|
|
||||||
def filter_object_parameters(ob, list_of_parameters):
|
|
||||||
"""Return an object of just the list of paramters."""
|
|
||||||
new_ob = {}
|
|
||||||
for par in list_of_parameters:
|
|
||||||
try:
|
|
||||||
new_ob[par] = ob[par]
|
|
||||||
except KeyError:
|
|
||||||
new_ob[par] = None
|
|
||||||
return new_ob
|
|
||||||
|
|
||||||
|
|
||||||
def group_by_company(devices):
|
|
||||||
"""Group a list of devices by company."""
|
|
||||||
grouped = {}
|
|
||||||
for dev in devices:
|
|
||||||
try:
|
|
||||||
grouped[dev['company']['name']].append(dev)
|
|
||||||
except KeyError:
|
|
||||||
grouped[dev['company']['name']] = [dev]
|
|
||||||
return grouped
|
|
||||||
|
|
||||||
|
|
||||||
def main(sendEmail=False):
|
|
||||||
"""Get the data and optionally send an email."""
|
|
||||||
if sendEmail:
|
|
||||||
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
|
||||||
print("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
|
||||||
exit()
|
|
||||||
|
|
||||||
devicetypes = meshify.query_meshify_api("devicetypes")
|
|
||||||
advvfdipp_devicetype = meshify.find_by_name("advvfdipp", devicetypes)
|
|
||||||
|
|
||||||
companies = meshify.query_meshify_api("companies")
|
|
||||||
company_lookup = {}
|
|
||||||
for x in companies:
|
|
||||||
company_lookup[x['id']] = x
|
|
||||||
|
|
||||||
devices = meshify.query_meshify_api("devices")
|
|
||||||
advvfdipp_devices = filter(lambda x: x['deviceTypeId'] == advvfdipp_devicetype['id'], devices)
|
|
||||||
advvfdipp_devices = [join_company_info(x, company_lookup) for x in advvfdipp_devices]
|
|
||||||
|
|
||||||
for i in range(0, len(advvfdipp_devices)):
|
|
||||||
advvfdipp_devices[i]['values'] = filter_object_parameters(
|
|
||||||
meshify.query_meshify_api("devices/{}/values".format(advvfdipp_devices[i]['id'])), MESHIFY_NAMES)
|
|
||||||
advvfdipp_devices = group_by_company(advvfdipp_devices)
|
|
||||||
|
|
||||||
totals = {}
|
|
||||||
for comp in advvfdipp_devices:
|
|
||||||
total = {}
|
|
||||||
average = {}
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
total[v] = 0.0
|
|
||||||
average[v] = 0.0
|
|
||||||
for dev in advvfdipp_devices[comp]:
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
try:
|
|
||||||
total[v] += float(dev['values'][v]['value'])
|
|
||||||
except ValueError:
|
|
||||||
# print("Can't make a total for {}".format(v))
|
|
||||||
pass
|
|
||||||
totals[comp] = total
|
|
||||||
|
|
||||||
for comp in advvfdipp_devices:
|
|
||||||
local_tz = get_localzone()
|
|
||||||
now_t = time()
|
|
||||||
now_dt = datetime.utcfromtimestamp(now_t)
|
|
||||||
now = local_tz.localize(now_dt)
|
|
||||||
|
|
||||||
total = []
|
|
||||||
average = []
|
|
||||||
|
|
||||||
header = "Well Name,"
|
|
||||||
for v in VALUES_TO_INCLUDE:
|
|
||||||
header += "{},".format(v['vanity_name'])
|
|
||||||
header = header[:-1] + "\n"
|
|
||||||
|
|
||||||
values = ""
|
|
||||||
for dev in sorted(advvfdipp_devices[comp], key=lambda x: x['vanityName']):
|
|
||||||
values += dev['vanityName'] + ","
|
|
||||||
for v in MESHIFY_NAMES:
|
|
||||||
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
|
||||||
dt_loc = local_tz.localize(dt_ts)
|
|
||||||
stale = (now - dt_loc) > timedelta(hours=24)
|
|
||||||
|
|
||||||
try:
|
|
||||||
v = str(round(float(dev['values'][v]['value']), 3))
|
|
||||||
if stale:
|
|
||||||
v += " (STALE)"
|
|
||||||
values += '{},'.format(v)
|
|
||||||
except ValueError:
|
|
||||||
v = str(dev['values'][v]['value'])
|
|
||||||
if stale:
|
|
||||||
v += " (STALE)"
|
|
||||||
values += '{},'.format(v)
|
|
||||||
|
|
||||||
# values += '{},'.format(dt)
|
|
||||||
values = values[:-1] + "\n"
|
|
||||||
|
|
||||||
if sendEmail:
|
|
||||||
|
|
||||||
with open("advvfdipp_to.json", 'r') as to_file:
|
|
||||||
to_lookup = json.load(to_file)
|
|
||||||
try:
|
|
||||||
email_to = to_lookup[comp]
|
|
||||||
except KeyError:
|
|
||||||
print("[{}] No recipients for that company({})!".format(datetime.now().isoformat(), comp))
|
|
||||||
continue
|
|
||||||
# part1 = MIMEText(header + values, "plain")
|
|
||||||
attachment = MIMEBase('application', 'octet-stream')
|
|
||||||
attachment.set_payload(header + values)
|
|
||||||
encoders.encode_base64(attachment)
|
|
||||||
|
|
||||||
now = datetime.now()
|
|
||||||
datestr = now.strftime("%a %b %d, %Y")
|
|
||||||
msg = MIMEMultipart('alternative')
|
|
||||||
msg['Subject'] = "{} SAMPLE Daily Adv. VFD IPP Report for {}".format(comp, datestr)
|
|
||||||
msg['From'] = "alerts@henry-pump.com"
|
|
||||||
msg['To'] = ", ".join(email_to)
|
|
||||||
|
|
||||||
filename = "{} {}.csv".format(comp, datestr)
|
|
||||||
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
|
||||||
|
|
||||||
# msg.attach(part1)
|
|
||||||
# msg.attach(part2)
|
|
||||||
msg.attach(attachment)
|
|
||||||
|
|
||||||
# s = SMTP(host="secure.emailsrvr.com", port=25)
|
|
||||||
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
|
||||||
s.starttls()
|
|
||||||
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
|
||||||
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
|
||||||
print("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
|
||||||
sleep(2)
|
|
||||||
|
|
||||||
with open('{}.csv'.format(comp), 'w') as csvfile:
|
|
||||||
csvfile.write(header + values)
|
|
||||||
|
|
||||||
advvfdipp_devices["totals"] = totals
|
|
||||||
with open("currentAdvVFDIPP.json", 'w') as jsonfile:
|
|
||||||
json.dump(advvfdipp_devices, jsonfile, indent=4)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(argv) > 1:
|
|
||||||
s = argv[1] == "true"
|
|
||||||
main(sendEmail=s)
|
|
||||||
else:
|
|
||||||
main(sendEmail=False)
|
|
||||||
@@ -1,10 +1,11 @@
|
|||||||
{
|
{
|
||||||
"Henry Resources Water": [
|
"Henry Resources Water": [
|
||||||
"pmcdonagh@henry-pump.com",
|
"pmcdonagh@henry-pump.com"
|
||||||
"thardway@henry-pump.com"
|
|
||||||
],
|
],
|
||||||
"CrownQuest": [
|
"CrownQuest": [
|
||||||
"pmcdonagh@henry-pump.com",
|
"pmcdonagh@henry-pump.com"
|
||||||
"thardway@henry-pump.com"
|
],
|
||||||
|
"QEP Resources": [
|
||||||
|
"pmcdonagh@henry-pump.com"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
26
ipp_channels.json
Normal file
26
ipp_channels.json
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"meshify_name": "devicestatus",
|
||||||
|
"vanity_name": "Well Status"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "flowrate",
|
||||||
|
"vanity_name": "Flow Rate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "dhdownholestatusint",
|
||||||
|
"vanity_name": "DH Sensor Status"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "dhintakepressure",
|
||||||
|
"vanity_name": "Intake Pressure"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "dhintaketemperature",
|
||||||
|
"vanity_name": "Intake Temperature"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"meshify_name": "pressurein",
|
||||||
|
"vanity_name": "Tubing Pressure"
|
||||||
|
}
|
||||||
|
]
|
||||||
201
reports_xlsx.py
Normal file
201
reports_xlsx.py
Normal file
@@ -0,0 +1,201 @@
|
|||||||
|
"""Prepare and send daily reports for Advanced VFD IPP devices in Meshify."""
|
||||||
|
import meshify
|
||||||
|
import json
|
||||||
|
from os import getenv
|
||||||
|
from sys import exit, stdout
|
||||||
|
from smtplib import SMTP
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from email.mime.multipart import MIMEMultipart
|
||||||
|
from email import encoders
|
||||||
|
from email.mime.base import MIMEBase
|
||||||
|
from time import sleep, time
|
||||||
|
from tzlocal import get_localzone
|
||||||
|
import xlsxwriter
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
|
||||||
|
logger = ""
|
||||||
|
DEVICE_TYPE_NAME = ""
|
||||||
|
datestring = datetime.now().strftime("%Y%m%d")
|
||||||
|
|
||||||
|
VALUES_TO_INCLUDE = []
|
||||||
|
MESHIFY_NAMES = []
|
||||||
|
|
||||||
|
SMTP_EMAIL = getenv("SMTP_EMAIL")
|
||||||
|
SMTP_PASSWORD = getenv("SMTP_PASSWORD")
|
||||||
|
|
||||||
|
|
||||||
|
def join_company_info(obj_with_companyId, company_lookup_obj):
|
||||||
|
"""Add company information to an object with companyId property."""
|
||||||
|
obj_with_companyId['company'] = company_lookup_obj[obj_with_companyId['companyId']]
|
||||||
|
return obj_with_companyId
|
||||||
|
|
||||||
|
|
||||||
|
def filter_object_parameters(ob, list_of_parameters):
|
||||||
|
"""Return an object of just the list of paramters."""
|
||||||
|
new_ob = {}
|
||||||
|
for par in list_of_parameters:
|
||||||
|
try:
|
||||||
|
new_ob[par] = ob[par]
|
||||||
|
except KeyError:
|
||||||
|
new_ob[par] = None
|
||||||
|
return new_ob
|
||||||
|
|
||||||
|
|
||||||
|
def group_by_company(devices):
|
||||||
|
"""Group a list of devices by company."""
|
||||||
|
grouped = {}
|
||||||
|
for dev in devices:
|
||||||
|
try:
|
||||||
|
grouped[dev['company']['name']].append(dev)
|
||||||
|
except KeyError:
|
||||||
|
grouped[dev['company']['name']] = [dev]
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
def main(sendEmail=False):
|
||||||
|
"""Get the data and optionally send an email."""
|
||||||
|
if sendEmail:
|
||||||
|
if not SMTP_EMAIL or not SMTP_PASSWORD:
|
||||||
|
logger.error("[{}] Be sure to set the SMTP email and password as environment variables SMTP_EMAIL and SMTP_PASSWORD".format(datetime.now().isoformat()))
|
||||||
|
exit()
|
||||||
|
|
||||||
|
devicetypes = meshify.query_meshify_api("devicetypes")
|
||||||
|
this_devicetype = meshify.find_by_name(DEVICE_TYPE_NAME, devicetypes)
|
||||||
|
|
||||||
|
companies = meshify.query_meshify_api("companies")
|
||||||
|
company_lookup = {}
|
||||||
|
for x in companies:
|
||||||
|
company_lookup[x['id']] = x
|
||||||
|
|
||||||
|
devices = meshify.query_meshify_api("devices")
|
||||||
|
this_devices = filter(lambda x: x['deviceTypeId'] == this_devicetype['id'], devices)
|
||||||
|
this_devices = [join_company_info(x, company_lookup) for x in this_devices]
|
||||||
|
|
||||||
|
for i in range(0, len(this_devices)):
|
||||||
|
this_devices[i]['values'] = filter_object_parameters(
|
||||||
|
meshify.query_meshify_api("devices/{}/values".format(this_devices[i]['id'])), MESHIFY_NAMES)
|
||||||
|
this_devices = group_by_company(this_devices)
|
||||||
|
|
||||||
|
for comp in this_devices:
|
||||||
|
local_tz = get_localzone()
|
||||||
|
now_t = time()
|
||||||
|
now_dt = datetime.utcfromtimestamp(now_t)
|
||||||
|
now = local_tz.localize(now_dt)
|
||||||
|
|
||||||
|
workbook = xlsxwriter.Workbook("files/{}_{}_{}.xlsx".format(DEVICE_TYPE_NAME, comp, datestring))
|
||||||
|
worksheet = workbook.add_worksheet()
|
||||||
|
worksheet.set_column('A:A', 20)
|
||||||
|
|
||||||
|
bold = workbook.add_format({'bold': True})
|
||||||
|
red = workbook.add_format({'font_color': 'red'})
|
||||||
|
|
||||||
|
worksheet.write(0, 0, "Well Name", bold)
|
||||||
|
for i in range(0, len(VALUES_TO_INCLUDE)):
|
||||||
|
worksheet.write(0, i+1, VALUES_TO_INCLUDE[i]['vanity_name'], bold)
|
||||||
|
|
||||||
|
sorted_company_devices = sorted(this_devices[comp], key=lambda x: x['vanityName'])
|
||||||
|
for j in range(0, len(sorted_company_devices)):
|
||||||
|
dev = sorted_company_devices[j]
|
||||||
|
worksheet.write(j+1, 0, dev['vanityName'])
|
||||||
|
for k in range(0, len(MESHIFY_NAMES)):
|
||||||
|
v = MESHIFY_NAMES[k]
|
||||||
|
dt_ts = datetime.utcfromtimestamp(dev['values'][v]['timestamp'])
|
||||||
|
dt_loc = local_tz.localize(dt_ts)
|
||||||
|
stale = (now - dt_loc) > timedelta(hours=24)
|
||||||
|
|
||||||
|
try:
|
||||||
|
v = round(float(dev['values'][v]['value']), 3)
|
||||||
|
if stale:
|
||||||
|
worksheet.write_number(j+1, 1+k, v, red)
|
||||||
|
else:
|
||||||
|
worksheet.write_number(j+1, 1+k, v)
|
||||||
|
except ValueError:
|
||||||
|
v = str(dev['values'][v]['value'])
|
||||||
|
if stale:
|
||||||
|
worksheet.write(j+1, 1+k, v, red)
|
||||||
|
else:
|
||||||
|
worksheet.write(j+1, 1+k, v)
|
||||||
|
|
||||||
|
workbook.close()
|
||||||
|
|
||||||
|
if sendEmail:
|
||||||
|
|
||||||
|
with open("{}_to.json".format(DEVICE_TYPE_NAME), 'r') as to_file:
|
||||||
|
to_lookup = json.load(to_file)
|
||||||
|
try:
|
||||||
|
email_to = to_lookup[comp]
|
||||||
|
except KeyError:
|
||||||
|
logger.error("[{}] No recipients for that company({})!".format(datetime.now().isoformat(), comp))
|
||||||
|
continue
|
||||||
|
# part1 = MIMEText(header + values, "plain")
|
||||||
|
attachment = MIMEBase('application', 'octet-stream')
|
||||||
|
attachment.set_payload(open("files/{}_{}_{}.xlsx".format(DEVICE_TYPE_NAME, comp, datestring), "rb").read())
|
||||||
|
encoders.encode_base64(attachment)
|
||||||
|
|
||||||
|
now = datetime.now()
|
||||||
|
datestr = now.strftime("%a %b %d, %Y")
|
||||||
|
msg = MIMEMultipart('alternative')
|
||||||
|
msg['Subject'] = "{} Daily Adv. VFD IPP Report for {}".format(comp, datestr)
|
||||||
|
msg['From'] = "alerts@henry-pump.com"
|
||||||
|
msg['To'] = ", ".join(email_to)
|
||||||
|
|
||||||
|
filename = "{} {}.xlsx".format(comp, datestr)
|
||||||
|
attachment.add_header('Content-Disposition', 'attachment', filename=filename)
|
||||||
|
|
||||||
|
# msg.attach(part1)
|
||||||
|
# msg.attach(part2)
|
||||||
|
msg.attach(attachment)
|
||||||
|
|
||||||
|
# s = SMTP(host="secure.emailsrvr.com", port=25)
|
||||||
|
s = SMTP(host="email-smtp.us-east-1.amazonaws.com", port=587)
|
||||||
|
s.starttls()
|
||||||
|
s.login(SMTP_EMAIL, SMTP_PASSWORD)
|
||||||
|
s.sendmail(from_addr="alerts@henry-pump.com", to_addrs=email_to, msg=msg.as_string())
|
||||||
|
logger.info("[{}] Email sent to {} for {}".format(datetime.now().isoformat(), email_to, comp))
|
||||||
|
sleep(2)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_logger():
|
||||||
|
"""Setup and return the logger module."""
|
||||||
|
log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(funcName)s(%(lineno)d) %(message)s')
|
||||||
|
logFile = './emailreports.log'.format(DEVICE_TYPE_NAME)
|
||||||
|
my_handler = RotatingFileHandler(logFile, mode='a', maxBytes=500*1024, backupCount=2, encoding=None, delay=0)
|
||||||
|
my_handler.setFormatter(log_formatter)
|
||||||
|
my_handler.setLevel(logging.INFO)
|
||||||
|
logger = logging.getLogger("emailreports")
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.addHandler(my_handler)
|
||||||
|
|
||||||
|
console_out = logging.StreamHandler(stdout)
|
||||||
|
console_out.setFormatter(log_formatter)
|
||||||
|
logger.addHandler(console_out)
|
||||||
|
|
||||||
|
return logger
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('deviceType', help="Meshify device type")
|
||||||
|
parser.add_argument('-s', '--send', action='store_true', help="Send emails to everyone in the _to.json file")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
DEVICE_TYPE_NAME = args.deviceType
|
||||||
|
SEND_EMAIL = args.send
|
||||||
|
|
||||||
|
logger = setup_logger()
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open("{}_channels.json".format(DEVICE_TYPE_NAME), 'r') as channel_file:
|
||||||
|
VALUES_TO_INCLUDE = json.load(channel_file)
|
||||||
|
MESHIFY_NAMES = [m['meshify_name'] for m in VALUES_TO_INCLUDE]
|
||||||
|
except IOError:
|
||||||
|
logger.error("No channel file named {}_channels.json".format(DEVICE_TYPE_NAME))
|
||||||
|
exit()
|
||||||
|
except ValueError as e:
|
||||||
|
logger.error("Channel file {}_channels.json is misformed: {}".format(DEVICE_TYPE_NAME, e))
|
||||||
|
exit()
|
||||||
|
|
||||||
|
main(sendEmail=SEND_EMAIL)
|
||||||
Reference in New Issue
Block a user