From 077adf5a131e06f9d0d18f133ca9b222a8fb53b1 Mon Sep 17 00:00:00 2001 From: William Bruno Date: Sat, 6 Nov 2021 14:59:22 -0400 Subject: [PATCH] updates --- coned_rtu/.envrc | 4 + coned_rtu/.flake8 | 7 ++ coned_rtu/.pre-commit-config.yaml | 22 ++++ coned_rtu/.python-version | 1 + coned_rtu/Dockerfile | 14 +++ coned_rtu/coned_rtu/__init__.py | 4 + coned_rtu/coned_rtu/coned.py | 199 ++++++++++++++++++++++++++++++ coned_rtu/coned_rtu/reading.py | 64 ++++++++++ coned_rtu/coned_rtu/store.py | 52 ++++++++ coned_rtu/dev_requirements.txt | 1 + coned_rtu/docker-compose.yml | 8 ++ coned_rtu/main.py | 25 ++++ coned_rtu/requirements.txt | 6 + coned_rtu/setup.py | 17 +++ coned_rtu/tests/coned_test.py | 28 +++++ coned_rtu/tests/reading_test.py | 109 ++++++++++++++++ coned_rtu/tests/store_test.py | 52 ++++++++ 17 files changed, 613 insertions(+) create mode 100644 coned_rtu/.envrc create mode 100644 coned_rtu/.flake8 create mode 100644 coned_rtu/.pre-commit-config.yaml create mode 100644 coned_rtu/.python-version create mode 100644 coned_rtu/Dockerfile create mode 100644 coned_rtu/coned_rtu/__init__.py create mode 100644 coned_rtu/coned_rtu/coned.py create mode 100644 coned_rtu/coned_rtu/reading.py create mode 100644 coned_rtu/coned_rtu/store.py create mode 100644 coned_rtu/dev_requirements.txt create mode 100644 coned_rtu/docker-compose.yml create mode 100644 coned_rtu/main.py create mode 100644 coned_rtu/requirements.txt create mode 100644 coned_rtu/setup.py create mode 100644 coned_rtu/tests/coned_test.py create mode 100644 coned_rtu/tests/reading_test.py create mode 100644 coned_rtu/tests/store_test.py diff --git a/coned_rtu/.envrc b/coned_rtu/.envrc new file mode 100644 index 0000000..02e473f --- /dev/null +++ b/coned_rtu/.envrc @@ -0,0 +1,4 @@ +# -*- mode: sh; -*- +pyversion=$(head .python-version) +pyenv=coned_rtu +use pyenv ${pyversion} diff --git a/coned_rtu/.flake8 b/coned_rtu/.flake8 new file mode 100644 index 0000000..6830d0a --- /dev/null +++ b/coned_rtu/.flake8 @@ -0,0 +1,7 @@ +[flake8] +exclude = + .direnv, + .git, + __pycache__, + build, + dist diff --git a/coned_rtu/.pre-commit-config.yaml b/coned_rtu/.pre-commit-config.yaml new file mode 100644 index 0000000..0bc7094 --- /dev/null +++ b/coned_rtu/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +default_stages: [commit] +repos: +- repo: https://github.com/psf/black + rev: 21.8b0 + hooks: + - id: black + name: black +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + args: [--max-line-length=120] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: end-of-file-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + - id: requirements-txt-fixer + - id: trailing-whitespace diff --git a/coned_rtu/.python-version b/coned_rtu/.python-version new file mode 100644 index 0000000..1635d0f --- /dev/null +++ b/coned_rtu/.python-version @@ -0,0 +1 @@ +3.9.6 diff --git a/coned_rtu/Dockerfile b/coned_rtu/Dockerfile new file mode 100644 index 0000000..b0ea103 --- /dev/null +++ b/coned_rtu/Dockerfile @@ -0,0 +1,14 @@ +FROM selenium/standalone-chrome + +USER root +RUN apt-get update && apt-get upgrade -yq +RUN apt-get install python3 python3-pip -y +USER seluser + +WORKDIR /home/seluser/coned-rtu + +COPY requirements.txt . +RUN pip3 install -r requirements.txt + +COPY coned_rtu . +ENTRYPOINT ["python3", "main.py"] diff --git a/coned_rtu/coned_rtu/__init__.py b/coned_rtu/coned_rtu/__init__.py new file mode 100644 index 0000000..cdc4a37 --- /dev/null +++ b/coned_rtu/coned_rtu/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa +from .coned import Coned, LoginFailedException, json_to_readings +from .reading import Reading +from .store import BucketList, OverlappingReadingError diff --git a/coned_rtu/coned_rtu/coned.py b/coned_rtu/coned_rtu/coned.py new file mode 100644 index 0000000..6d73cdc --- /dev/null +++ b/coned_rtu/coned_rtu/coned.py @@ -0,0 +1,199 @@ +import functools +import json + +from dateutil.parser import isoparse +import pyotp +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException, TimeoutException +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + +from .reading import Reading + +CONED_LOGIN_URL = "https://www.coned.com/en/login" +CONED_USAGE_URL = ( + "https://www.coned.com/en/accounts-billing/dashboard?tab1=billingandusage-1" +) + +DEFAULT_TIMEOUT = 60 + + +class LoginFailedException(Exception): + pass + + +def _screenshot_failure(f): + """Saves a screenshot if an error occurs. Only decorate Coned instance + functions, because we assume presence of self.""" + + @functools.wraps(f) + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except Exception as e: + self = args[0] + self.save_screenshot("error.png") + raise e + + return wrapper + + +def json_to_readings(usage_json): + readings = [] + usage = json.loads(usage_json) + for read in usage["reads"]: + # Opower gives readings with null value for intervals that don't have data + # yet, so skip them. + if read["value"] is None: + continue + + reading = Reading( + isoparse(read["startTime"]), + isoparse(read["endTime"]), + usage["unit"], + read["value"], + ) + readings.append(reading) + return readings + + +class Coned: + def __init__(self, user, password, totp, account_id, meter, maid=""): + self.user = user + self.password = password + self.totp = totp + self.account_id = account_id + self.meter = meter + self.maid = maid + + options = webdriver.ChromeOptions() + options.add_argument("--headless") + + # this one also solves some crashes sometimes + # https://stackoverflow.com/a/50725918 + options.add_argument("--no-sandbox") + + # this option can sometimes be necessary when running in docker, but + # can have a negative impact on performance + # https://stackoverflow.com/a/53970825 + options.add_argument("--disable-dev-shm-usage") + + self.driver = webdriver.Chrome(chrome_options=options) + + def opower_usage_url(self): + return f"https://cned.opower.com/ei/edge/apis/cws-real-time-ami-v1/cws/cned/accounts/{self.account_id}/meters/{self.meter}/usage" # noqa + + def save_screenshot(self, filename): + """ + Saves a 1080p screenshot of the page with the given filename in + the screenshots folder. Doesn't reset the window size. + """ + path = f"screenshots/{filename}.png" + self.driver.set_window_size(1920, 1080) + self.driver.save_screenshot(path) + + @_screenshot_failure + def login(self): + # Try to load the Billing and Usage page. If we find ourselves at the + # login page, then we need to login. If not, we have nothing to do. + self.driver.get(CONED_USAGE_URL) + if not self.at_login_page(): + return + + # Submit the login form + self.driver.find_element_by_id("form-login-email").send_keys(self.user) + self.driver.find_element_by_id("form-login-password").send_keys(self.password) + self.driver.find_element_by_id("form-login-password").submit() + + # Wait for login form to get to 2FA step. + try: + tfa_field = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until( + EC.element_to_be_clickable((By.ID, "form-login-mfa-code")) + ) + except TimeoutException as e: + # If it times out, it's probably due to bad credentials. + if self.is_bad_login(): + raise LoginFailedException + else: + raise e + + # Submit 2FA form + totp = pyotp.TOTP(self.totp) + tfa_field.send_keys(totp.now()) + tfa_field.submit() + + # Wait for dashboard to appear + WebDriverWait(self.driver, DEFAULT_TIMEOUT).until( + EC.element_to_be_clickable((By.XPATH, "//button[@data-value='overview']")) + ) + + # If prompted to select service address, use maid id to resolve + try: + must_select = WebDriverWait(self.driver, 2).until( # noqa: F841 + EC.presence_of_element_located( + ( + By.XPATH, + "//div[contains(@class, 'account-focus__accounts-container')]", + ) + ) + ) + account_button = WebDriverWait(self.driver, 5).until( + EC.element_to_be_clickable( + (By.XPATH, f"//button[@data-maid='{self.maid}']") + ) + ) + account_button.click() + except TimeoutException: + pass + + @_screenshot_failure + def get_usage(self): + self.driver.get(CONED_USAGE_URL) + + # Go to "real time usage" + rtu_button = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until( + EC.element_to_be_clickable( + (By.XPATH, "//button[@data-value='sectionRealTimeData']") + ) + ) + rtu_button.click() + + # Wait for "Download your real-time usage" in the iframe to appear so + # that it triggers authentication on the opower side + iframe = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until( + EC.presence_of_element_located( + (By.XPATH, "//*[@id='sectionRealTimeData']/iframe") + ) + ) + self.driver.switch_to.frame(iframe) + WebDriverWait(self.driver, DEFAULT_TIMEOUT).until( + EC.element_to_be_clickable((By.ID, "download-link")) + ) + + # Get the usage from opower + self.driver.get(self.opower_usage_url()) + return self.driver.find_element_by_tag_name("body").text + + @_screenshot_failure + def at_login_page(self): + """ + at_login_page returns whether the driver is at the ConEd login + page by looking for the login form. + """ + try: + self.driver.find_element_by_id("form-login-email") + return True + except NoSuchElementException: + return False + + @_screenshot_failure + def is_bad_login(self): + """ + is_bad_login returns whether there is a failed login indicator + on the page. + """ + bad_login = self.driver.find_element_by_class_name( + "login-form__container-error" + ) + return "you entered does not match our records" in bad_login.text diff --git a/coned_rtu/coned_rtu/reading.py b/coned_rtu/coned_rtu/reading.py new file mode 100644 index 0000000..b17e1c1 --- /dev/null +++ b/coned_rtu/coned_rtu/reading.py @@ -0,0 +1,64 @@ +ENERGY_UNITS = ("wh", "kwh") + + +class Reading: + """ + Reading represents an energy reading over an interval. It is essentially + the same in concept as an ESPI IntervalBlock. + """ + + def __init__(self, start_time, end_time, unit, value): + if start_time >= end_time: + raise ValueError("end_time must be after start_time.") + + self.start_time = start_time + self.end_time = end_time + + if unit.lower() not in ENERGY_UNITS: + raise ValueError("Invalid unit: use only Wh or kWh.") + + if unit.lower() == "kwh": + value = value * 1000 + + self.wh = value + + @classmethod + def combine(cls, a, b): + return cls( + a.start_time if a.start_time < b.start_time else b.start_time, + a.end_time if a.end_time > b.end_time else b.end_time, + "wh", + a.wh + b.wh, + ) + + def duration(self): + return self.end_time - self.start_time + + def overlaps(self, other): + return ( + (self.start_time < other.start_time and self.end_time > other.start_time) + or (other.start_time < self.start_time and other.end_time > self.start_time) + or (self.start_time == other.start_time or self.end_time == other.end_time) + ) + + def hash_bucket(self): + return str(self.start_time.date()) + + def __eq__(self, them): + return ( + isinstance(them, type(self)) + and self.__equality_set__() == them.__equality_set__() + ) + + def __equality_set__(self): + """Return the set of values that define my difference from others.""" + return (self.start_time, self.end_time, self.wh) + + def __hash__(self): + return hash((self.start_time, self.end_time, self.wh)) + + def __str__(self): + return f"Reading @ {self.start_time} ({self.duration()}): {self.wh} Wh" + + def __repr__(self): + return str(self) diff --git a/coned_rtu/coned_rtu/store.py b/coned_rtu/coned_rtu/store.py new file mode 100644 index 0000000..015f9ba --- /dev/null +++ b/coned_rtu/coned_rtu/store.py @@ -0,0 +1,52 @@ +from .reading import Reading + + +class OverlappingReadingError(ValueError): + pass + + +class BucketList: + """ + BucketList implements an ordered list using buckets whose keys are + determined by the Reading's hash_bucket function. When iterating, + BucketList yields in order of bucket key, then the ordered list in each + bucket. + """ + + def __init__(self, *args): + # dict to hold buckets -> ordered lists + self._dict = {} + # ordered list of bucket keys, for iteration + self._keys = [] + + def to_list(self): + out = [] + for bucket in self._keys: + for r in self._dict[bucket]: + out.append(r) + return out + + def get_bucket(self, key: str): + # create the bucket if it doesn't exist already + if key not in self._dict: + self._keys.append(key) + self._keys.sort() + self._dict[key] = [] + + return self._dict[key] + + def insert(self, reading: Reading): + bucket = self.get_bucket(reading.hash_bucket()) + + # If the new reading is identical to an existing one, deduplicate it + # and consider it successful. If the new reading overlaps with an + # existing reading, disallow it. + for r in bucket: + if reading == r: + return + if reading.overlaps(r): + raise OverlappingReadingError + + # incredibly lazy implementation, could insert sorted + bucket.append(reading) + bucket.sort(key=lambda x: x.start_time) diff --git a/coned_rtu/dev_requirements.txt b/coned_rtu/dev_requirements.txt new file mode 100644 index 0000000..95ea1e6 --- /dev/null +++ b/coned_rtu/dev_requirements.txt @@ -0,0 +1 @@ +pytest==6.2.4 diff --git a/coned_rtu/docker-compose.yml b/coned_rtu/docker-compose.yml new file mode 100644 index 0000000..f587d79 --- /dev/null +++ b/coned_rtu/docker-compose.yml @@ -0,0 +1,8 @@ +version: '3.7' + +services: + scraper: + build: . + volumes: + - ./:/home/seluser/coned-rtu + entrypoint: ["python3", "main.py", "dev.ini"] diff --git a/coned_rtu/main.py b/coned_rtu/main.py new file mode 100644 index 0000000..051c0e0 --- /dev/null +++ b/coned_rtu/main.py @@ -0,0 +1,25 @@ +import sys + +from configobj import ConfigObj + +from coned_rtu import Coned + +if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) +config = ConfigObj(sys.argv[1]) + +CONED_USER = config.get("CONED_USER") +CONED_PASS = config.get("CONED_PASS") +CONED_TOTP = config.get("CONED_TOTP") +OPOWER_ACCOUNT_ID = config.get("OPOWER_ACCOUNT_ID") +OPOWER_METER = config.get("OPOWER_METER") +CONED_MAID = config.get("CONED_MAID") + +coned = Coned( + CONED_USER, CONED_PASS, CONED_TOTP, OPOWER_ACCOUNT_ID, OPOWER_METER, CONED_MAID +) + +coned.login() +usage_json = coned.get_usage() +print(usage_json) diff --git a/coned_rtu/requirements.txt b/coned_rtu/requirements.txt new file mode 100644 index 0000000..7c4ce37 --- /dev/null +++ b/coned_rtu/requirements.txt @@ -0,0 +1,6 @@ +configobj==5.0.6 +pre-commit==2.14.1 +pyotp==2.3.0 +python-dateutil==2.8.1 +python-dotenv==0.11.0 +selenium==3.141.0 diff --git a/coned_rtu/setup.py b/coned_rtu/setup.py new file mode 100644 index 0000000..1a570ae --- /dev/null +++ b/coned_rtu/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +setup( + name="coned_rtu", + version="0.1", + description="Getting real-time usage reports from Con Edison", + packages=["coned_rtu"], + zip_safe=False, + install_requires=[ + "configobj==5.0.6", + "pre-commit==2.14.1", + "pyotp==2.3.0", + "python-dateutil==2.8.1", + "python-dotenv==0.11.0", + "selenium==3.141.0", + ], +) diff --git a/coned_rtu/tests/coned_test.py b/coned_rtu/tests/coned_test.py new file mode 100644 index 0000000..191810a --- /dev/null +++ b/coned_rtu/tests/coned_test.py @@ -0,0 +1,28 @@ +from datetime import datetime, timedelta, timezone +import unittest + +from coned_rtu import Reading, json_to_readings + +tz = timezone(timedelta(hours=-4)) + + +class TestConed(unittest.TestCase): + def test_json_to_readings(self): + with open("tests/usage.json") as f: + j = f.read() + got = json_to_readings(j) + want = [ + Reading( + datetime(2021, 8, 29, 0, 30, 0, tzinfo=tz), + datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz), + "wh", + 110.5, + ), + Reading( + datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz), + datetime(2021, 8, 29, 1, 0, 0, tzinfo=tz), + "wh", + 121.5, + ), + ] + self.assertEqual(got, want) diff --git a/coned_rtu/tests/reading_test.py b/coned_rtu/tests/reading_test.py new file mode 100644 index 0000000..887134a --- /dev/null +++ b/coned_rtu/tests/reading_test.py @@ -0,0 +1,109 @@ +from datetime import datetime, timedelta, timezone +import unittest + +from coned_rtu import Reading + + +class TestReading(unittest.TestCase): + def setUp(self): + self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc) + self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc) + + def test_start_time_must_be_before_end_time(self): + # start and end time the same + with self.assertRaises(ValueError): + Reading(self.sometime, self.sometime, "wh", 1) + + # start time after end time + with self.assertRaises(ValueError): + Reading(self.later, self.sometime, "wh", 1) + + def test_enforces_units(self): + # only wh or kwh supported + with self.assertRaises(ValueError): + Reading(self.sometime, self.later, "mwh", 1) + + # incorrect capitalization is okay + Reading(self.sometime, self.later, "Wh", 1) + + def test_duration(self): + r = Reading(self.sometime, self.later, "wh", 1) + self.assertEqual(r.duration(), timedelta(hours=24)) + + def test_overlaps(self): + t1 = self.sometime + t2 = t1 + timedelta(hours=1) + t3 = t2 + timedelta(hours=1) + t4 = t3 + timedelta(hours=1) + + # one within the other + a = Reading(t1, t4, "wh", 1) + b = Reading(t2, t3, "wh", 1) + self.assertTrue(a.overlaps(b)) + self.assertTrue(b.overlaps(a)) + + # one ends during the other + a = Reading(t1, t3, "wh", 1) + b = Reading(t2, t4, "wh", 1) + self.assertTrue(a.overlaps(b)) + self.assertTrue(b.overlaps(a)) + + # same start, one is shorter than the other + a = Reading(t1, t3, "wh", 1) + b = Reading(t1, t2, "wh", 1) + self.assertTrue(a.overlaps(b)) + self.assertTrue(b.overlaps(a)) + + # same end, one is shorter than the other + a = Reading(t1, t3, "wh", 1) + b = Reading(t2, t3, "wh", 1) + self.assertTrue(a.overlaps(b)) + self.assertTrue(b.overlaps(a)) + + # not overlapping + a = Reading(t1, t2, "wh", 1) + b = Reading(t2, t3, "wh", 1) + self.assertFalse(a.overlaps(b)) + self.assertFalse(b.overlaps(a)) + + def test_hash_bucket(self): + a = Reading(self.sometime, self.later, "wh", 1) + b = Reading(self.sometime, self.later, "wh", 2) + c = Reading( + self.sometime + timedelta(hours=24), + self.later + timedelta(hours=24), + "wh", + 1, + ) + self.assertEqual(a.hash_bucket(), b.hash_bucket()) + self.assertNotEqual(a.hash_bucket(), c.hash_bucket()) + + def test_equality(self): + r = Reading(self.sometime, self.later, "wh", 1) + + # times mismatch + bad_time = self.sometime + timedelta(seconds=10) + + other = Reading(bad_time, self.later, "wh", 1) + self.assertNotEqual(r, other) + + other = Reading(self.sometime, bad_time, "wh", 1) + self.assertNotEqual(r, other) + + # energy mismatch + other = Reading(self.sometime, self.later, "wh", 1.0000001) + self.assertNotEqual(r, other) + + # fully equal + other = Reading(self.sometime, self.later, "wh", 1) + self.assertEqual(r, other) + + def test_hashing(self): + # unequal + a = Reading(self.sometime, self.later, "wh", 1) + b = Reading(self.sometime, self.later, "wh", 2) + self.assertNotEqual(hash(a), hash(b)) + + # equal + b = Reading(self.sometime, self.later, "wh", 1) + self.assertEqual(hash(a), hash(b)) diff --git a/coned_rtu/tests/store_test.py b/coned_rtu/tests/store_test.py new file mode 100644 index 0000000..287fbd1 --- /dev/null +++ b/coned_rtu/tests/store_test.py @@ -0,0 +1,52 @@ +from datetime import datetime, timedelta, timezone +import unittest + +from coned_rtu import Reading, BucketList, OverlappingReadingError + + +class TestStore(unittest.TestCase): + def setUp(self): + self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc) + self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc) + self.hour = timedelta(hours=1) + self.day = timedelta(hours=24) + + def test(self): + bl = BucketList() + t1 = self.sometime + t2 = t1 + self.hour + t3 = t2 + self.hour + t4 = t3 + self.day + + b = Reading(t2, t3, "wh", 1) + bl.insert(b) + + a = Reading(t1, t2, "wh", 1) + bl.insert(a) + + # inserting an overlapping reading is an error + bad = Reading(t2, t2 + timedelta(minutes=30), "wh", 1) + with self.assertRaises(OverlappingReadingError): + bl.insert(bad) + + # inserting an identical reading results in no change + bl.insert(a) + + # inserted readings are sorted + got = bl.get_bucket(a.hash_bucket()) + want = [a, b] + self.assertEqual(got, want) + + # to_list returns a list in sorted order + c = Reading(t3, t4, "wh", 3) + bl.insert(c) + got = bl.to_list() + want = [a, b, c] + self.assertEqual(got, want) + + def test_get_bucket_makes_bucket_if_not_exist(self): + bl = BucketList() + day = "2021-08-30" + _ = bl.get_bucket(day) + self.assertEqual(len(bl._dict), 1) + self.assertEqual(list(bl._dict.keys())[0], day)