updates

coned_rtu/.envrc

@@ -0,0 +1,4 @@
+# -*- mode: sh; -*-
+pyversion=$(head .python-version)
+pyenv=coned_rtu
+use pyenv ${pyversion}

coned_rtu/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+        .direnv,
+        .git,
+        __pycache__,
+        build,
+        dist

coned_rtu/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+default_stages: [commit]
+repos:
+- repo: https://github.com/psf/black
+  rev: 21.8b0
+  hooks:
+    - id: black
+      name: black
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.9.2
+  hooks:
+    - id: flake8
+      args: [--max-line-length=120]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+    - id: check-added-large-files
+    - id: check-merge-conflict
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+      args: ["--fix=lf"]
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace

coned_rtu/.python-version

@@ -0,0 +1 @@
+3.9.6

coned_rtu/Dockerfile

@@ -0,0 +1,14 @@
+FROM selenium/standalone-chrome
+
+USER root
+RUN apt-get update && apt-get upgrade -yq
+RUN apt-get install python3 python3-pip -y
+USER seluser
+
+WORKDIR /home/seluser/coned-rtu
+
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt
+
+COPY coned_rtu .
+ENTRYPOINT ["python3", "main.py"]

coned_rtu/coned_rtu/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa
+from .coned import Coned, LoginFailedException, json_to_readings
+from .reading import Reading
+from .store import BucketList, OverlappingReadingError

coned_rtu/coned_rtu/coned.py

@@ -0,0 +1,199 @@
+import functools
+import json
+
+from dateutil.parser import isoparse
+import pyotp
+from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+from .reading import Reading
+
+CONED_LOGIN_URL = "https://www.coned.com/en/login"
+CONED_USAGE_URL = (
+    "https://www.coned.com/en/accounts-billing/dashboard?tab1=billingandusage-1"
+)
+
+DEFAULT_TIMEOUT = 60
+
+
+class LoginFailedException(Exception):
+    pass
+
+
+def _screenshot_failure(f):
+    """Saves a screenshot if an error occurs. Only decorate Coned instance
+    functions, because we assume presence of self."""
+
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except Exception as e:
+            self = args[0]
+            self.save_screenshot("error.png")
+            raise e
+
+    return wrapper
+
+
+def json_to_readings(usage_json):
+    readings = []
+    usage = json.loads(usage_json)
+    for read in usage["reads"]:
+        # Opower gives readings with null value for intervals that don't have data
+        # yet, so skip them.
+        if read["value"] is None:
+            continue
+
+        reading = Reading(
+            isoparse(read["startTime"]),
+            isoparse(read["endTime"]),
+            usage["unit"],
+            read["value"],
+        )
+        readings.append(reading)
+    return readings
+
+
+class Coned:
+    def __init__(self, user, password, totp, account_id, meter, maid=""):
+        self.user = user
+        self.password = password
+        self.totp = totp
+        self.account_id = account_id
+        self.meter = meter
+        self.maid = maid
+
+        options = webdriver.ChromeOptions()
+        options.add_argument("--headless")
+
+        # this one also solves some crashes sometimes
+        # https://stackoverflow.com/a/50725918
+        options.add_argument("--no-sandbox")
+
+        # this option can sometimes be necessary when running in docker, but
+        # can have a negative impact on performance
+        # https://stackoverflow.com/a/53970825
+        options.add_argument("--disable-dev-shm-usage")
+
+        self.driver = webdriver.Chrome(chrome_options=options)
+
+    def opower_usage_url(self):
+        return f"https://cned.opower.com/ei/edge/apis/cws-real-time-ami-v1/cws/cned/accounts/{self.account_id}/meters/{self.meter}/usage"  # noqa
+
+    def save_screenshot(self, filename):
+        """
+        Saves a 1080p screenshot of the page with the given filename in
+        the screenshots folder. Doesn't reset the window size.
+        """
+        path = f"screenshots/{filename}.png"
+        self.driver.set_window_size(1920, 1080)
+        self.driver.save_screenshot(path)
+
+    @_screenshot_failure
+    def login(self):
+        # Try to load the Billing and Usage page. If we find ourselves at the
+        # login page, then we need to login. If not, we have nothing to do.
+        self.driver.get(CONED_USAGE_URL)
+        if not self.at_login_page():
+            return
+
+        # Submit the login form
+        self.driver.find_element_by_id("form-login-email").send_keys(self.user)
+        self.driver.find_element_by_id("form-login-password").send_keys(self.password)
+        self.driver.find_element_by_id("form-login-password").submit()
+
+        # Wait for login form to get to 2FA step.
+        try:
+            tfa_field = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+                EC.element_to_be_clickable((By.ID, "form-login-mfa-code"))
+            )
+        except TimeoutException as e:
+            # If it times out, it's probably due to bad credentials.
+            if self.is_bad_login():
+                raise LoginFailedException
+            else:
+                raise e
+
+        # Submit 2FA form
+        totp = pyotp.TOTP(self.totp)
+        tfa_field.send_keys(totp.now())
+        tfa_field.submit()
+
+        # Wait for dashboard to appear
+        WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable((By.XPATH, "//button[@data-value='overview']"))
+        )
+
+        # If prompted to select service address, use maid id to resolve
+        try:
+            must_select = WebDriverWait(self.driver, 2).until(  # noqa: F841
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "//div[contains(@class, 'account-focus__accounts-container')]",
+                    )
+                )
+            )
+            account_button = WebDriverWait(self.driver, 5).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, f"//button[@data-maid='{self.maid}']")
+                )
+            )
+            account_button.click()
+        except TimeoutException:
+            pass
+
+    @_screenshot_failure
+    def get_usage(self):
+        self.driver.get(CONED_USAGE_URL)
+
+        # Go to "real time usage"
+        rtu_button = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable(
+                (By.XPATH, "//button[@data-value='sectionRealTimeData']")
+            )
+        )
+        rtu_button.click()
+
+        # Wait for "Download your real-time usage" in the iframe to appear so
+        # that it triggers authentication on the opower side
+        iframe = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.presence_of_element_located(
+                (By.XPATH, "//*[@id='sectionRealTimeData']/iframe")
+            )
+        )
+        self.driver.switch_to.frame(iframe)
+        WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable((By.ID, "download-link"))
+        )
+
+        # Get the usage from opower
+        self.driver.get(self.opower_usage_url())
+        return self.driver.find_element_by_tag_name("body").text
+
+    @_screenshot_failure
+    def at_login_page(self):
+        """
+        at_login_page returns whether the driver is at the ConEd login
+        page by looking for the login form.
+        """
+        try:
+            self.driver.find_element_by_id("form-login-email")
+            return True
+        except NoSuchElementException:
+            return False
+
+    @_screenshot_failure
+    def is_bad_login(self):
+        """
+        is_bad_login returns whether there is a failed login indicator
+        on the page.
+        """
+        bad_login = self.driver.find_element_by_class_name(
+            "login-form__container-error"
+        )
+        return "you entered does not match our records" in bad_login.text

coned_rtu/coned_rtu/reading.py

@@ -0,0 +1,64 @@
+ENERGY_UNITS = ("wh", "kwh")
+
+
+class Reading:
+    """
+    Reading represents an energy reading over an interval. It is essentially
+    the same in concept as an ESPI IntervalBlock.
+    """
+
+    def __init__(self, start_time, end_time, unit, value):
+        if start_time >= end_time:
+            raise ValueError("end_time must be after start_time.")
+
+        self.start_time = start_time
+        self.end_time = end_time
+
+        if unit.lower() not in ENERGY_UNITS:
+            raise ValueError("Invalid unit: use only Wh or kWh.")
+
+        if unit.lower() == "kwh":
+            value = value * 1000
+
+        self.wh = value
+
+    @classmethod
+    def combine(cls, a, b):
+        return cls(
+            a.start_time if a.start_time < b.start_time else b.start_time,
+            a.end_time if a.end_time > b.end_time else b.end_time,
+            "wh",
+            a.wh + b.wh,
+        )
+
+    def duration(self):
+        return self.end_time - self.start_time
+
+    def overlaps(self, other):
+        return (
+            (self.start_time < other.start_time and self.end_time > other.start_time)
+            or (other.start_time < self.start_time and other.end_time > self.start_time)
+            or (self.start_time == other.start_time or self.end_time == other.end_time)
+        )
+
+    def hash_bucket(self):
+        return str(self.start_time.date())
+
+    def __eq__(self, them):
+        return (
+            isinstance(them, type(self))
+            and self.__equality_set__() == them.__equality_set__()
+        )
+
+    def __equality_set__(self):
+        """Return the set of values that define my difference from others."""
+        return (self.start_time, self.end_time, self.wh)
+
+    def __hash__(self):
+        return hash((self.start_time, self.end_time, self.wh))
+
+    def __str__(self):
+        return f"Reading @ {self.start_time} ({self.duration()}): {self.wh} Wh"
+
+    def __repr__(self):
+        return str(self)

coned_rtu/coned_rtu/store.py

@@ -0,0 +1,52 @@
+from .reading import Reading
+
+
+class OverlappingReadingError(ValueError):
+    pass
+
+
+class BucketList:
+    """
+    BucketList implements an ordered list using buckets whose keys are
+    determined by the Reading's hash_bucket function. When iterating,
+    BucketList yields in order of bucket key, then the ordered list in each
+    bucket.
+    """
+
+    def __init__(self, *args):
+        # dict to hold buckets -> ordered lists
+        self._dict = {}
+        # ordered list of bucket keys, for iteration
+        self._keys = []
+
+    def to_list(self):
+        out = []
+        for bucket in self._keys:
+            for r in self._dict[bucket]:
+                out.append(r)
+        return out
+
+    def get_bucket(self, key: str):
+        # create the bucket if it doesn't exist already
+        if key not in self._dict:
+            self._keys.append(key)
+            self._keys.sort()
+            self._dict[key] = []
+
+        return self._dict[key]
+
+    def insert(self, reading: Reading):
+        bucket = self.get_bucket(reading.hash_bucket())
+
+        # If the new reading is identical to an existing one, deduplicate it
+        # and consider it successful. If the new reading overlaps with an
+        # existing reading, disallow it.
+        for r in bucket:
+            if reading == r:
+                return
+            if reading.overlaps(r):
+                raise OverlappingReadingError
+
+        # incredibly lazy implementation, could insert sorted
+        bucket.append(reading)
+        bucket.sort(key=lambda x: x.start_time)

coned_rtu/dev_requirements.txt

@@ -0,0 +1 @@
+pytest==6.2.4

coned_rtu/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3.7'
+
+services:
+  scraper:
+    build: .
+    volumes:
+      - ./:/home/seluser/coned-rtu
+    entrypoint: ["python3", "main.py", "dev.ini"]

coned_rtu/main.py

@@ -0,0 +1,25 @@
+import sys
+
+from configobj import ConfigObj
+
+from coned_rtu import Coned
+
+if len(sys.argv) < 2:
+    print(f"Usage: {sys.argv[0]} <config file>")
+    sys.exit(1)
+config = ConfigObj(sys.argv[1])
+
+CONED_USER = config.get("CONED_USER")
+CONED_PASS = config.get("CONED_PASS")
+CONED_TOTP = config.get("CONED_TOTP")
+OPOWER_ACCOUNT_ID = config.get("OPOWER_ACCOUNT_ID")
+OPOWER_METER = config.get("OPOWER_METER")
+CONED_MAID = config.get("CONED_MAID")
+
+coned = Coned(
+    CONED_USER, CONED_PASS, CONED_TOTP, OPOWER_ACCOUNT_ID, OPOWER_METER, CONED_MAID
+)
+
+coned.login()
+usage_json = coned.get_usage()
+print(usage_json)

coned_rtu/requirements.txt

@@ -0,0 +1,6 @@
+configobj==5.0.6
+pre-commit==2.14.1
+pyotp==2.3.0
+python-dateutil==2.8.1
+python-dotenv==0.11.0
+selenium==3.141.0

coned_rtu/setup.py

@@ -0,0 +1,17 @@
+from setuptools import setup
+
+setup(
+    name="coned_rtu",
+    version="0.1",
+    description="Getting real-time usage reports from Con Edison",
+    packages=["coned_rtu"],
+    zip_safe=False,
+    install_requires=[
+        "configobj==5.0.6",
+        "pre-commit==2.14.1",
+        "pyotp==2.3.0",
+        "python-dateutil==2.8.1",
+        "python-dotenv==0.11.0",
+        "selenium==3.141.0",
+    ],
+)

coned_rtu/tests/coned_test.py

@@ -0,0 +1,28 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading, json_to_readings
+
+tz = timezone(timedelta(hours=-4))
+
+
+class TestConed(unittest.TestCase):
+    def test_json_to_readings(self):
+        with open("tests/usage.json") as f:
+            j = f.read()
+        got = json_to_readings(j)
+        want = [
+            Reading(
+                datetime(2021, 8, 29, 0, 30, 0, tzinfo=tz),
+                datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz),
+                "wh",
+                110.5,
+            ),
+            Reading(
+                datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz),
+                datetime(2021, 8, 29, 1, 0, 0, tzinfo=tz),
+                "wh",
+                121.5,
+            ),
+        ]
+        self.assertEqual(got, want)

coned_rtu/tests/reading_test.py

@@ -0,0 +1,109 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading
+
+
+class TestReading(unittest.TestCase):
+    def setUp(self):
+        self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc)
+        self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc)
+
+    def test_start_time_must_be_before_end_time(self):
+        # start and end time the same
+        with self.assertRaises(ValueError):
+            Reading(self.sometime, self.sometime, "wh", 1)
+
+        # start time after end time
+        with self.assertRaises(ValueError):
+            Reading(self.later, self.sometime, "wh", 1)
+
+    def test_enforces_units(self):
+        # only wh or kwh supported
+        with self.assertRaises(ValueError):
+            Reading(self.sometime, self.later, "mwh", 1)
+
+        # incorrect capitalization is okay
+        Reading(self.sometime, self.later, "Wh", 1)
+
+    def test_duration(self):
+        r = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(r.duration(), timedelta(hours=24))
+
+    def test_overlaps(self):
+        t1 = self.sometime
+        t2 = t1 + timedelta(hours=1)
+        t3 = t2 + timedelta(hours=1)
+        t4 = t3 + timedelta(hours=1)
+
+        # one within the other
+        a = Reading(t1, t4, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # one ends during the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t2, t4, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # same start, one is shorter than the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t1, t2, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # same end, one is shorter than the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # not overlapping
+        a = Reading(t1, t2, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertFalse(a.overlaps(b))
+        self.assertFalse(b.overlaps(a))
+
+    def test_hash_bucket(self):
+        a = Reading(self.sometime, self.later, "wh", 1)
+        b = Reading(self.sometime, self.later, "wh", 2)
+        c = Reading(
+            self.sometime + timedelta(hours=24),
+            self.later + timedelta(hours=24),
+            "wh",
+            1,
+        )
+        self.assertEqual(a.hash_bucket(), b.hash_bucket())
+        self.assertNotEqual(a.hash_bucket(), c.hash_bucket())
+
+    def test_equality(self):
+        r = Reading(self.sometime, self.later, "wh", 1)
+
+        # times mismatch
+        bad_time = self.sometime + timedelta(seconds=10)
+
+        other = Reading(bad_time, self.later, "wh", 1)
+        self.assertNotEqual(r, other)
+
+        other = Reading(self.sometime, bad_time, "wh", 1)
+        self.assertNotEqual(r, other)
+
+        # energy mismatch
+        other = Reading(self.sometime, self.later, "wh", 1.0000001)
+        self.assertNotEqual(r, other)
+
+        # fully equal
+        other = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(r, other)
+
+    def test_hashing(self):
+        # unequal
+        a = Reading(self.sometime, self.later, "wh", 1)
+        b = Reading(self.sometime, self.later, "wh", 2)
+        self.assertNotEqual(hash(a), hash(b))
+
+        # equal
+        b = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(hash(a), hash(b))

coned_rtu/tests/store_test.py

@@ -0,0 +1,52 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading, BucketList, OverlappingReadingError
+
+
+class TestStore(unittest.TestCase):
+    def setUp(self):
+        self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc)
+        self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc)
+        self.hour = timedelta(hours=1)
+        self.day = timedelta(hours=24)
+
+    def test(self):
+        bl = BucketList()
+        t1 = self.sometime
+        t2 = t1 + self.hour
+        t3 = t2 + self.hour
+        t4 = t3 + self.day
+
+        b = Reading(t2, t3, "wh", 1)
+        bl.insert(b)
+
+        a = Reading(t1, t2, "wh", 1)
+        bl.insert(a)
+
+        # inserting an overlapping reading is an error
+        bad = Reading(t2, t2 + timedelta(minutes=30), "wh", 1)
+        with self.assertRaises(OverlappingReadingError):
+            bl.insert(bad)
+
+        # inserting an identical reading results in no change
+        bl.insert(a)
+
+        # inserted readings are sorted
+        got = bl.get_bucket(a.hash_bucket())
+        want = [a, b]
+        self.assertEqual(got, want)
+
+        # to_list returns a list in sorted order
+        c = Reading(t3, t4, "wh", 3)
+        bl.insert(c)
+        got = bl.to_list()
+        want = [a, b, c]
+        self.assertEqual(got, want)
+
+    def test_get_bucket_makes_bucket_if_not_exist(self):
+        bl = BucketList()
+        day = "2021-08-30"
+        _ = bl.get_bucket(day)
+        self.assertEqual(len(bl._dict), 1)
+        self.assertEqual(list(bl._dict.keys())[0], day)