Merge branch 'master' of ssh://gitlab.intrepidnet.org:2224/wbruno/home-assistant-addons

merge

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "coned-fetch"]
 	path = coned-fetch
 	url = https://github.com/jpoles1/coned-fetch
+[submodule "addon-ledfx"]
+	path = addon-ledfx
+	url = https://github.com/hassio-addons/addon-ledfx.git

coned-fetch/.devcontainer/Dockerfile

@@ -0,0 +1,33 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/typescript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version: 16, 14, 12
+FROM node:12-slim
+
+# [Optional] Uncomment this section to install additional OS packages.
+RUN apt-get update \
+    && apt-get install -y wget gnupg make git \
+    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+    && apt-get update \
+    && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
+      --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN npm i puppeteer \
+    # Add user so we don't need --no-sandbox.
+    # same layer as npm install to keep re-chowned files from using up several hundred MBs more space
+#    && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
+#    && mkdir -p /home/pptruser/Downloads \
+#    && chown -R pptruser:pptruser /home/pptruser \
+#    && chown -R pptruser:pptruser /node_modules
+
+# Run everything after as non-privileged user.
+#USER pptruser
+
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# [Optional] Uncomment if you want to install more global node packages
+# RUN su node -c "npm install -g <your-package-list -here>"

coned-fetch/.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	"build": {
+		"dockerfile": "Dockerfile",
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": {},
+	"forwardPorts": [3000, 3001],
+
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"dbaeumer.vscode-eslint"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "yarn install",
+
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	//"remoteUser": "node"
+}

coned-fetch/.dockerignore

@@ -0,0 +1 @@
+node_modules

coned-fetch/.eslintrc.js

@@ -0,0 +1,27 @@
+module.exports = {
+	env: {
+		es6: true,
+		node: true,
+	},
+	parserOptions: {
+		parser: "@typescript-eslint/parser",
+		ecmaVersion: 2018,
+		sourceType: "module",
+	},
+	plugins: ["@typescript-eslint", "prettier"],
+	extends: ["plugin:@typescript-eslint/recommended", "prettier"],
+	rules: {
+		"prettier/prettier": [
+			"warn",
+			{
+				useTabs: true,
+				printWidth: 180,
+				trailingComma: "es5",
+				htmlWhitespaceSensitivity: "strict",
+			},
+		],
+		"@typescript-eslint/no-explicit-any": "off",
+		"@typescript-eslint/no-empty-interface": "off",
+		"@typescript-eslint/no-non-null-assertion": "off",
+	},
+};

coned-fetch/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Program",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "program": "${workspaceFolder}/index.ts",
+            "preLaunchTask": "tsc: build - tsconfig.json",
+            "outFiles": [
+                "${workspaceFolder}/built/**/*.js"
+            ]
+        }
+    ]
+}

coned-fetch/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "editor.codeActionsOnSave": {
+        "source.fixAll.eslint": true
+    }
+}

coned-fetch/Dockerfile

@@ -0,0 +1,17 @@
+FROM node:12-buster
+
+RUN apt-get update && apt-get install -y chromium jq
+
+RUN ln -s /usr/bin/chromium /usr/bin/chromium-browser
+
+RUN mkdir /usr/app
+WORKDIR /usr/app
+
+COPY package*.json ./
+#ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
+RUN npm install
+
+COPY . .
+
+RUN chmod a+x run.sh
+CMD ./run.sh

coned-fetch/Makefile

@@ -0,0 +1,6 @@
+run:
+	npm start
+watch:
+	npm run watch
+docker_rebuild:
+	docker-compose up -d --build

coned-fetch/README.md

@@ -0,0 +1,41 @@
+# ConEd Realtime Data Fetch
+
+This repo contains a Node.js script which automates the retreival of realtime usage data from the ConEd opower.com API. The data is returned as a JSON file which can then be processed further as part of a data pipeline. This repo does not do much with the retreived data as of yet, and is instead intended to help others as a jumping-off point for their projects.
+
+## Quickstart
+
+In order to run the script you must have a recent version of Node.js installed. Then use the following instructions to start retreiving your data!
+
+Install the deps:
+
+```
+npm install
+```
+
+Copy `example.env` to `.env` and input your account information. See below for MFA setup:
+
+```
+EMAIL=test@test.com
+PASSWORD=correcthorsebatterystaple
+MFA_SECRET=youranswerhere
+ACCOUNT_ID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+METER_NUM=XXXXXXX
+```
+
+Run the script and use the JSON file output for whatever you need!
+
+```
+node index.js
+```
+
+## MFA Setup
+
+You can bypass 2-factor authenication by doing the following:
+
+1) Login to ConEd, and go to your profile. Reset your 2FA method, but do not complete the registration. 
+2) Clear your cookies or use incognito and re-open the ConEd website.
+3) Log back in to your account and you should be prompted to re-establish your 2FA.
+4) Select use text message for your 2FA method. 
+5) You should then be presented with an option to say you do not have texting on your phone. Select this and you should be able to use a security question instead.
+6) Create a security question and enter your answer into the .env config as detailed above!
+ 

coned-fetch/config.json

@@ -0,0 +1,45 @@
+{
+    "name": "ConEd Meter to Hass Supervisor API",
+    "version": "0.1.0d",
+    "slug": "coned",
+    "description": "ConEd Meter Reader to Hass Supervisor API",
+    "url": "https://gitlab.intrepidnet.org/home-assistant-addons/~/tree/master/coned-fetch",
+    "panel_icon": "mdi:chart-areaspline",
+    "startup": "application",
+    "homeassistant": "0.92.0b2",
+    "boot": "auto",
+    "ports": { "3000/tcp": 3000 },
+    "ports_description": { "3000/tcp": "Web Server (optional)" },
+    "host_network": "False",
+    "arch": [
+      "aarch64",
+      "amd64",
+      "armv7",
+      "i386"
+    ],
+    "hassio_api": true,
+    "auth_api": true,
+    "hassio_role": "default",
+    "homeassistant_api": true,
+    "options":
+    {
+      "email": "",
+      "password": "",
+      "mfa_type": "",
+      "mfa_secret": "",
+      "account_uuid": "",
+      "meter_number": "",
+      "site": ""
+    },
+    "schema":
+    {
+      "email": "str",
+      "password": "str",
+      "mfa_type": "str",
+      "mfa_secret": "str",
+      "account_uuid": "str",
+      "meter_number": "str",
+      "site": "str"
+     }
+  }
+  

coned-fetch/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "2"
+services:
+  coned:
+    build: .
+    ports:
+      - "3000:3000"
+    command: bash -c "npm start"
+    restart: unless-stopped

coned-fetch/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "coned-fetch",
+  "version": "1.0.0",
+  "description": "",
+  "scripts": {
+    "start": "ts-node src/index.ts",
+    "watch": "nodemon -L --watch src --ignore out/ -e ts",
+    "lint": "eslint src/. --config .eslintrc.js --ext .ts --fix",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^0.21.1",
+    "chalk": "^4.1.0",
+    "dotenv": "^8.2.0",
+    "fastify": "^3.20.0",
+    "firefox": "0.0.1",
+    "moment": "^2.29.1",
+    "puppeteer": "^5.2.1",
+    "sqlite3": "^5.0.2",
+    "trilogy": "^2.0.3",
+    "typescript": "^4.0.2"
+  },
+  "devDependencies": {
+    "@types/lowdb": "^1.0.9",
+    "@types/node": "^14.6.4",
+    "@types/puppeteer": "^3.0.2",
+    "@typescript-eslint/eslint-plugin": "^4.28.5",
+    "@typescript-eslint/parser": "^4.28.5",
+    "eslint": "^7.31.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^3.4.0",
+    "nodemon": "^2.0.4",
+    "prettier": "^2.3.2",
+    "ts-node": "^9.0.0"
+  }
+}

coned-fetch/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+CONFIG_PATH=/data/options.json
+
+export EMAIL="$(jq --raw-output '.email' $CONFIG_PATH)"
+export PASSWORD="$(jq --raw-output '.password' $CONFIG_PATH)"
+export MFA_SECRET="$(jq --raw-output '.mfa_secret' $CONFIG_PATH)"
+export ACCOUNT_UUID="$(jq --raw-output '.account_uuid' $CONFIG_PATH)"
+export METER_NUM="$(jq --raw-output '.meter_number' $CONFIG_PATH)"
+
+echo "Params:"
+echo "EMAIL =" $EMAIL
+echo "PASSWORD =" $(sed 's/^........../**********/' <<<$PASSWORD)
+echo "MFA_SECRET =" $(sed 's/^........../**********/' <<<$MFA_SECRET)
+echo "ACCOUNT_UUID =" $ACCOUNT_UUID
+echo "METER_NUM =" $METER_NUM
+
+# Start the listener and enter an endless loop
+echo "Starting server!"
+npm start

coned-fetch/src/coned.ts

@@ -0,0 +1,97 @@
+import chalk from "chalk";
+import pupeteer from "puppeteer";
+
+function sleep(ms: number) {
+	return new Promise((resolve) => {
+		setTimeout(resolve, ms);
+	});
+}
+
+export class ConEd {
+	browser: pupeteer.Browser | undefined;
+	refresh_interval_min = 15;
+	db_store: ((raw_data: any) => Promise<any>) | undefined;
+
+	public constructor(values: Partial<ConEd>) {
+		Object.assign(this, values);
+		if (!this.valid_config()) {
+			return;
+		}
+	}
+
+	valid_config(): boolean {
+		const req_config = ["EMAIL", "PASSWORD", "MFA_SECRET", "ACCOUNT_UUID", "METER_NUM"];
+		return req_config.every((config) => {
+			if (!process.env[config]) {
+				console.log(chalk.red(`${config} is not set in the .env config! Exiting...`));
+				return false;
+			}
+			return true;
+		});
+	}
+
+	async init(): Promise<void> {
+		this.browser = await pupeteer.launch({
+			defaultViewport: { width: 1920, height: 1080 },
+			args: ["--no-sandbox", "--disable-setuid-sandbox"],
+		});
+	}
+	async login(): Promise<void> {
+		const page = await this.browser!.newPage();
+		// Login to ConEd website
+		await page.goto("https://www.coned.com/en/login");
+		await page.type("#form-login-email", process.env.EMAIL!);
+		await page.type("#form-login-password", process.env.PASSWORD!);
+		await page.click("#form-login-remember-me");
+		await page.click(".submit-button");
+		// Wait for login to authenticate
+		await sleep(5000);
+		await page.screenshot({ path: "meter0.png" });
+		// Enter in 2 factor auth code (see README for details)
+		await page.type("#form-login-mfa-code", process.env.MFA_SECRET!);
+		await page.click(".js-login-new-device-form .button");
+		// Wait for authentication to complete
+		await page.waitForNavigation();
+		await sleep(5000);
+		await page.screenshot({ path: "meter1.png" });
+	}
+	async read_api(): Promise<any> {
+		return new Promise(async (resolve) => {
+			// Access the API using your newly acquired authentication cookies!
+			const api_page = await this.browser!.newPage();
+			const api_url = `https://cned.opower.com/ei/edge/apis/cws-real-time-ami-v1/cws/cned/accounts/${process.env.ACCOUNT_UUID}/meters/${process.env.METER_NUM}/usage`;
+			await api_page.goto(api_url);
+			await api_page.screenshot({ path: "meter2.png" });
+			const data_elem = await api_page.$("pre");
+			const text_data = await api_page.evaluate((el: HTMLElement) => el.textContent, data_elem);
+			const raw_data = JSON.parse(text_data!);
+			await api_page.close();
+			resolve(raw_data);
+		});
+	}
+	async fetch_once(): Promise<void> {
+		await this.init();
+		await this.login();
+		let raw_data = await this.read_api();
+		let attempt = 1;
+		while ("error" in raw_data && attempt < 5) {
+			console.log(chalk.yellow("Failed to fetch data from API:", raw_data["error"]["details"]));
+			raw_data = await this.read_api();
+			await sleep(15000);
+			attempt++;
+		}
+		if (attempt < 5) {
+			console.log(chalk.green("Successfully retrieved API data!"));
+			this.db_store!(raw_data);
+		} else {
+			console.log(chalk.red("Failed to retrieve API data:", raw_data["error"]));
+		}
+		await this.browser?.close();
+	}
+	monitor(interval_min = 15): void {
+		this.fetch_once();
+		setInterval(() => {
+			this.fetch_once();
+		}, interval_min * 60 * 1000);
+	}
+}

coned-fetch/src/index.ts

@@ -0,0 +1,144 @@
+// Import config from .env
+
+// eslint-disable-next-line
+require("dotenv").config();
+
+import { ConEd } from "./coned";
+import { connect } from "trilogy";
+import chalk from "chalk";
+import fs from "fs";
+import moment from "moment";
+import fastify from "fastify";
+import axios from "axios";
+
+const output_dir = "out";
+
+if (!fs.existsSync(output_dir)) {
+	fs.mkdirSync(output_dir);
+}
+
+async function main() {
+	const db = connect(`./${output_dir}/storage.db`);
+
+	process.on("exit", function () {
+		db.close();
+		console.log(chalk.green("Successfully closed DB connection!"));
+	});
+
+	const energySchema = {
+		id: "increments",
+		startTime: { type: Date, unique: true },
+		endTime: { type: Date, unique: true },
+		energy: { type: Number },
+	};
+	type EnergyEntry = {
+		id: number;
+		startTime: Date;
+		endTime: Date;
+		energy: number;
+	};
+	const energyModel = await db.model<EnergyEntry>("energy", energySchema);
+
+	let latest_read_startTime = new Date(0);
+	async function update_sensor(latest_read: any) {
+		if (latest_read.startTime < latest_read_startTime) return;
+		latest_read_startTime = latest_read.startTime;
+		axios.post("http://supervisor/core/api/states/sensor.coned_energy", {
+			state: latest_read.value,
+			attributes: { unit_of_measurement: "kWh", friendly_name: "ConEd Energy Usage", device_class: "energy", state_class: "measurement", last_reset: latest_read.startTime },
+		}, {
+			headers: {"Authorization": `Bearer ${process.env.SUPERVISOR_TOKEN}`, "Content-Type": "application/json"}
+		});
+		console.log(chalk.green(`Sent sensor update: ${latest_read.value} kWh @ ${moment(latest_read.startTime).format("LLL")}`));
+	}
+
+	async function db_store(raw_data: any) {
+		const filtered_data = raw_data.reads.filter((row: any) => {
+			return row.value !== null && row.value !== undefined;
+		})
+		const latest_read = filtered_data[0];
+//		update_sensor(latest_read);
+		for (const row of filtered_data) {
+			await energyModel.updateOrCreate({ startTime: new Date(row.startTime), endTime: new Date(row.endTime) }, { energy: row.value }).catch((e) => {
+				console.log(chalk.red(`Err: ${e}`));
+			});
+		}
+		console.log(chalk.green("Database Updated!"));
+		console.log(chalk.green("Database Closed!"));
+	}
+
+	const c = new ConEd({ db_store });
+	c.monitor();
+
+	const server = fastify({
+		//logger: true,
+	});
+
+	// Declare a route
+	server.get<{
+		Querystring: { h: number };
+	}>("/", (request, reply) => {
+		console.log(request.query);
+		let hour_lookback = 24;
+		if (request.query.h) {
+			if (!!Number(request.query.h)) {
+				hour_lookback = Number(request.query.h);
+			} else {
+				reply.code(400).type("text/plain").send("Err 400: Invalid Request");
+				return;
+			}
+		}
+		const d = new Date();
+		d.setHours(d.getHours() - hour_lookback);
+		energyModel.find([["startTime", ">", d]]).then((raw_data) => {
+			reply.send({ data: raw_data });
+		});
+	});
+
+	server.get<{
+		Querystring: { n: number };
+	}>("/latest", (request, reply) => {
+		console.log(request.query);
+		let n_results = 64;
+		if (request.query.n) {
+			if (!!Number(request.query.n)) {
+				n_results = Number(request.query.n);
+			} else {
+				reply.code(400).type("text/plain").send("Err 400: Invalid Request");
+				return;
+			}
+		}
+		energyModel.find({}, { limit: n_results, order: ["startTime", "desc"] }).then((raw_data) => {
+			function to_delta_string(dt: Date): string {
+				const delta_min = Math.abs(dt.getTime() - new Date().getTime()) / 6e4;
+				return delta_min < 60 ? `${delta_min.toFixed(0)}min` : `${(delta_min / 60).toPrecision(2)}hr`;
+			}
+			//Create array of energy readings starting with the most recent
+			const readings = Object.values(raw_data).map((row: EnergyEntry) => row.energy);
+			//Time of earliest read in list
+			const earliest_dt: Date = Object.values(raw_data)[raw_data.length - 1].startTime;
+			const earliest_delta = to_delta_string(earliest_dt);
+			//Time of peak (most recent if tie)
+			const peak_index = readings.indexOf(Math.max.apply(null, readings));
+			const peak = Object.values(raw_data)[peak_index].energy;
+			const peak_dt = Object.values(raw_data)[peak_index].startTime;
+			const peak_delta = to_delta_string(peak_dt);
+			//Time of latest read in list
+			const latest_dt: Date = Object.values(raw_data)[0].startTime;
+			const latest_delta = to_delta_string(latest_dt);
+			const resp_data = { earliest_delta, latest_delta, peak, peak_delta, readings };
+			reply.send(resp_data);
+		});
+	});
+
+	// Run the server!
+	server.listen(3000, "0.0.0.0", function (err, address) {
+		if (err) {
+			server.log.error(err);
+			process.exit(1);
+		}
+		server.log.info(`server listening on ${address}`);
+	});
+}
+
+main();

coned-fetch/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "es2018",
+    "module": "commonjs",
+    "outDir": "./lib",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true
+  },
+  "exclude": [
+    "test",
+    "statmed-frontend"
+  ]
+}

coned_rtu/.envrc

@@ -0,0 +1,4 @@
+# -*- mode: sh; -*-
+pyversion=$(head .python-version)
+pyenv=coned_rtu
+use pyenv ${pyversion}

coned_rtu/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+        .direnv,
+        .git,
+        __pycache__,
+        build,
+        dist

coned_rtu/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+default_stages: [commit]
+repos:
+- repo: https://github.com/psf/black
+  rev: 21.8b0
+  hooks:
+    - id: black
+      name: black
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.9.2
+  hooks:
+    - id: flake8
+      args: [--max-line-length=120]
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.0.1
+  hooks:
+    - id: check-added-large-files
+    - id: check-merge-conflict
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+      args: ["--fix=lf"]
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace

coned_rtu/.python-version

@@ -0,0 +1 @@
+3.9.6

coned_rtu/Dockerfile

@@ -0,0 +1,14 @@
+FROM selenium/standalone-chrome
+
+USER root
+RUN apt-get update && apt-get upgrade -yq
+RUN apt-get install python3 python3-pip -y
+USER seluser
+
+WORKDIR /home/seluser/coned-rtu
+
+COPY requirements.txt .
+RUN pip3 install -r requirements.txt
+
+COPY coned_rtu .
+ENTRYPOINT ["python3", "main.py"]

coned_rtu/coned_rtu/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa
+from .coned import Coned, LoginFailedException, json_to_readings
+from .reading import Reading
+from .store import BucketList, OverlappingReadingError

coned_rtu/coned_rtu/coned.py

@@ -0,0 +1,199 @@
+import functools
+import json
+
+from dateutil.parser import isoparse
+import pyotp
+from selenium import webdriver
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+from .reading import Reading
+
+CONED_LOGIN_URL = "https://www.coned.com/en/login"
+CONED_USAGE_URL = (
+    "https://www.coned.com/en/accounts-billing/dashboard?tab1=billingandusage-1"
+)
+
+DEFAULT_TIMEOUT = 60
+
+
+class LoginFailedException(Exception):
+    pass
+
+
+def _screenshot_failure(f):
+    """Saves a screenshot if an error occurs. Only decorate Coned instance
+    functions, because we assume presence of self."""
+
+    @functools.wraps(f)
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except Exception as e:
+            self = args[0]
+            self.save_screenshot("error.png")
+            raise e
+
+    return wrapper
+
+
+def json_to_readings(usage_json):
+    readings = []
+    usage = json.loads(usage_json)
+    for read in usage["reads"]:
+        # Opower gives readings with null value for intervals that don't have data
+        # yet, so skip them.
+        if read["value"] is None:
+            continue
+
+        reading = Reading(
+            isoparse(read["startTime"]),
+            isoparse(read["endTime"]),
+            usage["unit"],
+            read["value"],
+        )
+        readings.append(reading)
+    return readings
+
+
+class Coned:
+    def __init__(self, user, password, totp, account_id, meter, maid=""):
+        self.user = user
+        self.password = password
+        self.totp = totp
+        self.account_id = account_id
+        self.meter = meter
+        self.maid = maid
+
+        options = webdriver.ChromeOptions()
+        options.add_argument("--headless")
+
+        # this one also solves some crashes sometimes
+        # https://stackoverflow.com/a/50725918
+        options.add_argument("--no-sandbox")
+
+        # this option can sometimes be necessary when running in docker, but
+        # can have a negative impact on performance
+        # https://stackoverflow.com/a/53970825
+        options.add_argument("--disable-dev-shm-usage")
+
+        self.driver = webdriver.Chrome(chrome_options=options)
+
+    def opower_usage_url(self):
+        return f"https://cned.opower.com/ei/edge/apis/cws-real-time-ami-v1/cws/cned/accounts/{self.account_id}/meters/{self.meter}/usage"  # noqa
+
+    def save_screenshot(self, filename):
+        """
+        Saves a 1080p screenshot of the page with the given filename in
+        the screenshots folder. Doesn't reset the window size.
+        """
+        path = f"screenshots/{filename}.png"
+        self.driver.set_window_size(1920, 1080)
+        self.driver.save_screenshot(path)
+
+    @_screenshot_failure
+    def login(self):
+        # Try to load the Billing and Usage page. If we find ourselves at the
+        # login page, then we need to login. If not, we have nothing to do.
+        self.driver.get(CONED_USAGE_URL)
+        if not self.at_login_page():
+            return
+
+        # Submit the login form
+        self.driver.find_element_by_id("form-login-email").send_keys(self.user)
+        self.driver.find_element_by_id("form-login-password").send_keys(self.password)
+        self.driver.find_element_by_id("form-login-password").submit()
+
+        # Wait for login form to get to 2FA step.
+        try:
+            tfa_field = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+                EC.element_to_be_clickable((By.ID, "form-login-mfa-code"))
+            )
+        except TimeoutException as e:
+            # If it times out, it's probably due to bad credentials.
+            if self.is_bad_login():
+                raise LoginFailedException
+            else:
+                raise e
+
+        # Submit 2FA form
+        totp = pyotp.TOTP(self.totp)
+        tfa_field.send_keys(totp.now())
+        tfa_field.submit()
+
+        # Wait for dashboard to appear
+        WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable((By.XPATH, "//button[@data-value='overview']"))
+        )
+
+        # If prompted to select service address, use maid id to resolve
+        try:
+            must_select = WebDriverWait(self.driver, 2).until(  # noqa: F841
+                EC.presence_of_element_located(
+                    (
+                        By.XPATH,
+                        "//div[contains(@class, 'account-focus__accounts-container')]",
+                    )
+                )
+            )
+            account_button = WebDriverWait(self.driver, 5).until(
+                EC.element_to_be_clickable(
+                    (By.XPATH, f"//button[@data-maid='{self.maid}']")
+                )
+            )
+            account_button.click()
+        except TimeoutException:
+            pass
+
+    @_screenshot_failure
+    def get_usage(self):
+        self.driver.get(CONED_USAGE_URL)
+
+        # Go to "real time usage"
+        rtu_button = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable(
+                (By.XPATH, "//button[@data-value='sectionRealTimeData']")
+            )
+        )
+        rtu_button.click()
+
+        # Wait for "Download your real-time usage" in the iframe to appear so
+        # that it triggers authentication on the opower side
+        iframe = WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.presence_of_element_located(
+                (By.XPATH, "//*[@id='sectionRealTimeData']/iframe")
+            )
+        )
+        self.driver.switch_to.frame(iframe)
+        WebDriverWait(self.driver, DEFAULT_TIMEOUT).until(
+            EC.element_to_be_clickable((By.ID, "download-link"))
+        )
+
+        # Get the usage from opower
+        self.driver.get(self.opower_usage_url())
+        return self.driver.find_element_by_tag_name("body").text
+
+    @_screenshot_failure
+    def at_login_page(self):
+        """
+        at_login_page returns whether the driver is at the ConEd login
+        page by looking for the login form.
+        """
+        try:
+            self.driver.find_element_by_id("form-login-email")
+            return True
+        except NoSuchElementException:
+            return False
+
+    @_screenshot_failure
+    def is_bad_login(self):
+        """
+        is_bad_login returns whether there is a failed login indicator
+        on the page.
+        """
+        bad_login = self.driver.find_element_by_class_name(
+            "login-form__container-error"
+        )
+        return "you entered does not match our records" in bad_login.text

coned_rtu/coned_rtu/reading.py

@@ -0,0 +1,64 @@
+ENERGY_UNITS = ("wh", "kwh")
+
+
+class Reading:
+    """
+    Reading represents an energy reading over an interval. It is essentially
+    the same in concept as an ESPI IntervalBlock.
+    """
+
+    def __init__(self, start_time, end_time, unit, value):
+        if start_time >= end_time:
+            raise ValueError("end_time must be after start_time.")
+
+        self.start_time = start_time
+        self.end_time = end_time
+
+        if unit.lower() not in ENERGY_UNITS:
+            raise ValueError("Invalid unit: use only Wh or kWh.")
+
+        if unit.lower() == "kwh":
+            value = value * 1000
+
+        self.wh = value
+
+    @classmethod
+    def combine(cls, a, b):
+        return cls(
+            a.start_time if a.start_time < b.start_time else b.start_time,
+            a.end_time if a.end_time > b.end_time else b.end_time,
+            "wh",
+            a.wh + b.wh,
+        )
+
+    def duration(self):
+        return self.end_time - self.start_time
+
+    def overlaps(self, other):
+        return (
+            (self.start_time < other.start_time and self.end_time > other.start_time)
+            or (other.start_time < self.start_time and other.end_time > self.start_time)
+            or (self.start_time == other.start_time or self.end_time == other.end_time)
+        )
+
+    def hash_bucket(self):
+        return str(self.start_time.date())
+
+    def __eq__(self, them):
+        return (
+            isinstance(them, type(self))
+            and self.__equality_set__() == them.__equality_set__()
+        )
+
+    def __equality_set__(self):
+        """Return the set of values that define my difference from others."""
+        return (self.start_time, self.end_time, self.wh)
+
+    def __hash__(self):
+        return hash((self.start_time, self.end_time, self.wh))
+
+    def __str__(self):
+        return f"Reading @ {self.start_time} ({self.duration()}): {self.wh} Wh"
+
+    def __repr__(self):
+        return str(self)

coned_rtu/coned_rtu/store.py

@@ -0,0 +1,52 @@
+from .reading import Reading
+
+
+class OverlappingReadingError(ValueError):
+    pass
+
+
+class BucketList:
+    """
+    BucketList implements an ordered list using buckets whose keys are
+    determined by the Reading's hash_bucket function. When iterating,
+    BucketList yields in order of bucket key, then the ordered list in each
+    bucket.
+    """
+
+    def __init__(self, *args):
+        # dict to hold buckets -> ordered lists
+        self._dict = {}
+        # ordered list of bucket keys, for iteration
+        self._keys = []
+
+    def to_list(self):
+        out = []
+        for bucket in self._keys:
+            for r in self._dict[bucket]:
+                out.append(r)
+        return out
+
+    def get_bucket(self, key: str):
+        # create the bucket if it doesn't exist already
+        if key not in self._dict:
+            self._keys.append(key)
+            self._keys.sort()
+            self._dict[key] = []
+
+        return self._dict[key]
+
+    def insert(self, reading: Reading):
+        bucket = self.get_bucket(reading.hash_bucket())
+
+        # If the new reading is identical to an existing one, deduplicate it
+        # and consider it successful. If the new reading overlaps with an
+        # existing reading, disallow it.
+        for r in bucket:
+            if reading == r:
+                return
+            if reading.overlaps(r):
+                raise OverlappingReadingError
+
+        # incredibly lazy implementation, could insert sorted
+        bucket.append(reading)
+        bucket.sort(key=lambda x: x.start_time)

coned_rtu/dev_requirements.txt

@@ -0,0 +1 @@
+pytest==6.2.4

coned_rtu/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '3.7'
+
+services:
+  scraper:
+    build: .
+    volumes:
+      - ./:/home/seluser/coned-rtu
+    entrypoint: ["python3", "main.py", "dev.ini"]

coned_rtu/main.py

@@ -0,0 +1,25 @@
+import sys
+
+from configobj import ConfigObj
+
+from coned_rtu import Coned
+
+if len(sys.argv) < 2:
+    print(f"Usage: {sys.argv[0]} <config file>")
+    sys.exit(1)
+config = ConfigObj(sys.argv[1])
+
+CONED_USER = config.get("CONED_USER")
+CONED_PASS = config.get("CONED_PASS")
+CONED_TOTP = config.get("CONED_TOTP")
+OPOWER_ACCOUNT_ID = config.get("OPOWER_ACCOUNT_ID")
+OPOWER_METER = config.get("OPOWER_METER")
+CONED_MAID = config.get("CONED_MAID")
+
+coned = Coned(
+    CONED_USER, CONED_PASS, CONED_TOTP, OPOWER_ACCOUNT_ID, OPOWER_METER, CONED_MAID
+)
+
+coned.login()
+usage_json = coned.get_usage()
+print(usage_json)

coned_rtu/requirements.txt

@@ -0,0 +1,6 @@
+configobj==5.0.6
+pre-commit==2.14.1
+pyotp==2.3.0
+python-dateutil==2.8.1
+python-dotenv==0.11.0
+selenium==3.141.0

coned_rtu/setup.py

@@ -0,0 +1,17 @@
+from setuptools import setup
+
+setup(
+    name="coned_rtu",
+    version="0.1",
+    description="Getting real-time usage reports from Con Edison",
+    packages=["coned_rtu"],
+    zip_safe=False,
+    install_requires=[
+        "configobj==5.0.6",
+        "pre-commit==2.14.1",
+        "pyotp==2.3.0",
+        "python-dateutil==2.8.1",
+        "python-dotenv==0.11.0",
+        "selenium==3.141.0",
+    ],
+)

coned_rtu/tests/coned_test.py

@@ -0,0 +1,28 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading, json_to_readings
+
+tz = timezone(timedelta(hours=-4))
+
+
+class TestConed(unittest.TestCase):
+    def test_json_to_readings(self):
+        with open("tests/usage.json") as f:
+            j = f.read()
+        got = json_to_readings(j)
+        want = [
+            Reading(
+                datetime(2021, 8, 29, 0, 30, 0, tzinfo=tz),
+                datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz),
+                "wh",
+                110.5,
+            ),
+            Reading(
+                datetime(2021, 8, 29, 0, 45, 0, tzinfo=tz),
+                datetime(2021, 8, 29, 1, 0, 0, tzinfo=tz),
+                "wh",
+                121.5,
+            ),
+        ]
+        self.assertEqual(got, want)

coned_rtu/tests/reading_test.py

@@ -0,0 +1,109 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading
+
+
+class TestReading(unittest.TestCase):
+    def setUp(self):
+        self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc)
+        self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc)
+
+    def test_start_time_must_be_before_end_time(self):
+        # start and end time the same
+        with self.assertRaises(ValueError):
+            Reading(self.sometime, self.sometime, "wh", 1)
+
+        # start time after end time
+        with self.assertRaises(ValueError):
+            Reading(self.later, self.sometime, "wh", 1)
+
+    def test_enforces_units(self):
+        # only wh or kwh supported
+        with self.assertRaises(ValueError):
+            Reading(self.sometime, self.later, "mwh", 1)
+
+        # incorrect capitalization is okay
+        Reading(self.sometime, self.later, "Wh", 1)
+
+    def test_duration(self):
+        r = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(r.duration(), timedelta(hours=24))
+
+    def test_overlaps(self):
+        t1 = self.sometime
+        t2 = t1 + timedelta(hours=1)
+        t3 = t2 + timedelta(hours=1)
+        t4 = t3 + timedelta(hours=1)
+
+        # one within the other
+        a = Reading(t1, t4, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # one ends during the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t2, t4, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # same start, one is shorter than the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t1, t2, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # same end, one is shorter than the other
+        a = Reading(t1, t3, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertTrue(a.overlaps(b))
+        self.assertTrue(b.overlaps(a))
+
+        # not overlapping
+        a = Reading(t1, t2, "wh", 1)
+        b = Reading(t2, t3, "wh", 1)
+        self.assertFalse(a.overlaps(b))
+        self.assertFalse(b.overlaps(a))
+
+    def test_hash_bucket(self):
+        a = Reading(self.sometime, self.later, "wh", 1)
+        b = Reading(self.sometime, self.later, "wh", 2)
+        c = Reading(
+            self.sometime + timedelta(hours=24),
+            self.later + timedelta(hours=24),
+            "wh",
+            1,
+        )
+        self.assertEqual(a.hash_bucket(), b.hash_bucket())
+        self.assertNotEqual(a.hash_bucket(), c.hash_bucket())
+
+    def test_equality(self):
+        r = Reading(self.sometime, self.later, "wh", 1)
+
+        # times mismatch
+        bad_time = self.sometime + timedelta(seconds=10)
+
+        other = Reading(bad_time, self.later, "wh", 1)
+        self.assertNotEqual(r, other)
+
+        other = Reading(self.sometime, bad_time, "wh", 1)
+        self.assertNotEqual(r, other)
+
+        # energy mismatch
+        other = Reading(self.sometime, self.later, "wh", 1.0000001)
+        self.assertNotEqual(r, other)
+
+        # fully equal
+        other = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(r, other)
+
+    def test_hashing(self):
+        # unequal
+        a = Reading(self.sometime, self.later, "wh", 1)
+        b = Reading(self.sometime, self.later, "wh", 2)
+        self.assertNotEqual(hash(a), hash(b))
+
+        # equal
+        b = Reading(self.sometime, self.later, "wh", 1)
+        self.assertEqual(hash(a), hash(b))

coned_rtu/tests/store_test.py

@@ -0,0 +1,52 @@
+from datetime import datetime, timedelta, timezone
+import unittest
+
+from coned_rtu import Reading, BucketList, OverlappingReadingError
+
+
+class TestStore(unittest.TestCase):
+    def setUp(self):
+        self.sometime = datetime(2010, 12, 25, 14, 30, 0, tzinfo=timezone.utc)
+        self.later = datetime(2010, 12, 26, 14, 30, 0, tzinfo=timezone.utc)
+        self.hour = timedelta(hours=1)
+        self.day = timedelta(hours=24)
+
+    def test(self):
+        bl = BucketList()
+        t1 = self.sometime
+        t2 = t1 + self.hour
+        t3 = t2 + self.hour
+        t4 = t3 + self.day
+
+        b = Reading(t2, t3, "wh", 1)
+        bl.insert(b)
+
+        a = Reading(t1, t2, "wh", 1)
+        bl.insert(a)
+
+        # inserting an overlapping reading is an error
+        bad = Reading(t2, t2 + timedelta(minutes=30), "wh", 1)
+        with self.assertRaises(OverlappingReadingError):
+            bl.insert(bad)
+
+        # inserting an identical reading results in no change
+        bl.insert(a)
+
+        # inserted readings are sorted
+        got = bl.get_bucket(a.hash_bucket())
+        want = [a, b]
+        self.assertEqual(got, want)
+
+        # to_list returns a list in sorted order
+        c = Reading(t3, t4, "wh", 3)
+        bl.insert(c)
+        got = bl.to_list()
+        want = [a, b, c]
+        self.assertEqual(got, want)
+
+    def test_get_bucket_makes_bucket_if_not_exist(self):
+        bl = BucketList()
+        day = "2021-08-30"
+        _ = bl.get_bucket(day)
+        self.assertEqual(len(bl._dict), 1)
+        self.assertEqual(list(bl._dict.keys())[0], day)

oru/config.json

@@ -1,6 +1,6 @@
 {
   "name": "Orange and Rockland Utility to MQTT Bridge",
-  "version": "0.2.57",
+  "version": "0.2.57a",
   "slug": "oru",
   "description": "Orange and Rockland Utility meter reading to MQTT",
   "url": "https://github.com/bvlaicu/home-assistant-addons/tree/master/oru",
@@ -9,8 +9,9 @@
   "homeassistant": "0.92.0b2",
   "boot": "auto",
   "map": ["share:rw", "ssl"],
-  "host_dbus": "true",
   "devices": ["/dev/bus/usb:/dev/bus/usb:rwm"],
+  "privileged": ["SYS_RESOURCE"],
+  "host_dbus": true,
   "host_network": "False",
   "arch": [
     "aarch64",

repository.json

@@ -1,5 +1,5 @@
 {
-  "name": "bvlaicu add-ons for Hassio",
-  "url": "https://github.com/bvlaicu/home-assistant-addons",
-  "maintainer": "bvlaicu"
+  "name": "Add-ons for Hassio",
+  "url": "http://gitlab.intrepidnet.org/wbruno/home-assistant-addons",
+  "maintainer": "wbruno"
 }