coned-fetch
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/typescript-node/.devcontainer/base.Dockerfile
|
||||
|
||||
# [Choice] Node.js version: 16, 14, 12
|
||||
FROM node:12-slim
|
||||
|
||||
# [Optional] Uncomment this section to install additional OS packages.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y wget gnupg make git \
|
||||
&& wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
|
||||
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
|
||||
--no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN npm i puppeteer \
|
||||
# Add user so we don't need --no-sandbox.
|
||||
# same layer as npm install to keep re-chowned files from using up several hundred MBs more space
|
||||
# && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
|
||||
# && mkdir -p /home/pptruser/Downloads \
|
||||
# && chown -R pptruser:pptruser /home/pptruser \
|
||||
# && chown -R pptruser:pptruser /node_modules
|
||||
|
||||
# Run everything after as non-privileged user.
|
||||
#USER pptruser
|
||||
|
||||
|
||||
# [Optional] Uncomment if you want to install an additional version of node using nvm
|
||||
# ARG EXTRA_NODE_VERSION=10
|
||||
# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
|
||||
|
||||
# [Optional] Uncomment if you want to install more global node packages
|
||||
# RUN su node -c "npm install -g <your-package-list -here>"
|
||||
@@ -0,0 +1,27 @@
|
||||
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
|
||||
// https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/typescript-node
|
||||
{
|
||||
"name": "Node.js & TypeScript",
|
||||
"build": {
|
||||
"dockerfile": "Dockerfile",
|
||||
},
|
||||
|
||||
// Set *default* container specific settings.json values on container create.
|
||||
"settings": {},
|
||||
"forwardPorts": [3000, 3001],
|
||||
|
||||
|
||||
// Add the IDs of extensions you want installed when the container is created.
|
||||
"extensions": [
|
||||
"dbaeumer.vscode-eslint"
|
||||
],
|
||||
|
||||
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
||||
// "forwardPorts": [],
|
||||
|
||||
// Use 'postCreateCommand' to run commands after the container is created.
|
||||
// "postCreateCommand": "yarn install",
|
||||
|
||||
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
|
||||
//"remoteUser": "node"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
node_modules
|
||||
@@ -0,0 +1,27 @@
|
||||
module.exports = {
|
||||
env: {
|
||||
es6: true,
|
||||
node: true,
|
||||
},
|
||||
parserOptions: {
|
||||
parser: "@typescript-eslint/parser",
|
||||
ecmaVersion: 2018,
|
||||
sourceType: "module",
|
||||
},
|
||||
plugins: ["@typescript-eslint", "prettier"],
|
||||
extends: ["plugin:@typescript-eslint/recommended", "prettier"],
|
||||
rules: {
|
||||
"prettier/prettier": [
|
||||
"warn",
|
||||
{
|
||||
useTabs: true,
|
||||
printWidth: 180,
|
||||
trailingComma: "es5",
|
||||
htmlWhitespaceSensitivity: "strict",
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-empty-interface": "off",
|
||||
"@typescript-eslint/no-non-null-assertion": "off",
|
||||
},
|
||||
};
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"name": "Launch Program",
|
||||
"skipFiles": [
|
||||
"<node_internals>/**"
|
||||
],
|
||||
"program": "${workspaceFolder}/index.ts",
|
||||
"preLaunchTask": "tsc: build - tsconfig.json",
|
||||
"outFiles": [
|
||||
"${workspaceFolder}/built/**/*.js"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.fixAll.eslint": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
FROM node:12-buster
|
||||
|
||||
RUN apt-get update && apt-get install -y chromium jq
|
||||
|
||||
RUN ln -s /usr/bin/chromium /usr/bin/chromium-browser
|
||||
|
||||
RUN mkdir /usr/app
|
||||
WORKDIR /usr/app
|
||||
|
||||
COPY package*.json ./
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
|
||||
RUN npm install
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN chmod a+x run.sh
|
||||
CMD ./run.sh
|
||||
@@ -0,0 +1,6 @@
|
||||
run:
|
||||
npm start
|
||||
watch:
|
||||
npm run watch
|
||||
docker_rebuild:
|
||||
docker-compose up -d --build
|
||||
@@ -0,0 +1,41 @@
|
||||
# ConEd Realtime Data Fetch
|
||||
|
||||
This repo contains a Node.js script which automates the retreival of realtime usage data from the ConEd opower.com API. The data is returned as a JSON file which can then be processed further as part of a data pipeline. This repo does not do much with the retreived data as of yet, and is instead intended to help others as a jumping-off point for their projects.
|
||||
|
||||
## Quickstart
|
||||
|
||||
In order to run the script you must have a recent version of Node.js installed. Then use the following instructions to start retreiving your data!
|
||||
|
||||
Install the deps:
|
||||
|
||||
```
|
||||
npm install
|
||||
```
|
||||
|
||||
Copy `example.env` to `.env` and input your account information. See below for MFA setup:
|
||||
|
||||
```
|
||||
EMAIL=test@test.com
|
||||
PASSWORD=correcthorsebatterystaple
|
||||
MFA_SECRET=youranswerhere
|
||||
ACCOUNT_ID=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
|
||||
METER_NUM=XXXXXXX
|
||||
```
|
||||
|
||||
Run the script and use the JSON file output for whatever you need!
|
||||
|
||||
```
|
||||
node index.js
|
||||
```
|
||||
|
||||
## MFA Setup
|
||||
|
||||
You can bypass 2-factor authenication by doing the following:
|
||||
|
||||
1) Login to ConEd, and go to your profile. Reset your 2FA method, but do not complete the registration.
|
||||
2) Clear your cookies or use incognito and re-open the ConEd website.
|
||||
3) Log back in to your account and you should be prompted to re-establish your 2FA.
|
||||
4) Select use text message for your 2FA method.
|
||||
5) You should then be presented with an option to say you do not have texting on your phone. Select this and you should be able to use a security question instead.
|
||||
6) Create a security question and enter your answer into the .env config as detailed above!
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"name": "ConEd Meter to Hass Supervisor APUI",
|
||||
"version": "0.1.00",
|
||||
"slug": "coned",
|
||||
"description": "ConEd Meter Reader to Hass Supervisor API",
|
||||
"url": "https://gitlab.intrepidnet.org/home-assistant-addons/tree/master/coned-fetch",
|
||||
"panel_icon": "mdi:chart-areaspline",
|
||||
"startup": "application",
|
||||
"homeassistant": "0.92.0b2",
|
||||
"boot": "auto",
|
||||
"map": ["share:rw", "ssl"],
|
||||
"devices": ["/dev/bus/usb:/dev/bus/usb:rwm"],
|
||||
"privileged": ["SYS_RESOURCE"],
|
||||
"host_dbus": true,
|
||||
"host_network": "False",
|
||||
"arch": [
|
||||
"aarch64",
|
||||
"amd64",
|
||||
"armv7",
|
||||
"i386"
|
||||
],
|
||||
"hassio_api": true,
|
||||
"auth_api": true,
|
||||
"hassio_role": "default",
|
||||
"homeassistant_api": false,
|
||||
"options":
|
||||
{
|
||||
"email": "",
|
||||
"password": "",
|
||||
"mfa_secret": "",
|
||||
"account_uuid": "",
|
||||
"meter_number": ""
|
||||
},
|
||||
"schema":
|
||||
{
|
||||
"email": "str",
|
||||
"password": "str",
|
||||
"mfa_secret": "str",
|
||||
"account_uuid": "str",
|
||||
"meter_number": "str",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
version: "2"
|
||||
services:
|
||||
coned:
|
||||
build: .
|
||||
ports:
|
||||
- "3000:3000"
|
||||
command: bash -c "npm start"
|
||||
restart: unless-stopped
|
||||
Generated
+4925
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "coned-fetch",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"scripts": {
|
||||
"start": "ts-node src/index.ts",
|
||||
"watch": "nodemon -L --watch src --ignore out/ -e ts",
|
||||
"lint": "eslint src/. --config .eslintrc.js --ext .ts --fix",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"axios": "^0.21.1",
|
||||
"chalk": "^4.1.0",
|
||||
"dotenv": "^8.2.0",
|
||||
"fastify": "^3.20.0",
|
||||
"moment": "^2.29.1",
|
||||
"puppeteer": "^5.2.1",
|
||||
"sqlite3": "^5.0.2",
|
||||
"trilogy": "^2.0.3",
|
||||
"typescript": "^4.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/lowdb": "^1.0.9",
|
||||
"@types/node": "^14.6.4",
|
||||
"@types/puppeteer": "^3.0.2",
|
||||
"@typescript-eslint/eslint-plugin": "^4.28.5",
|
||||
"@typescript-eslint/parser": "^4.28.5",
|
||||
"eslint": "^7.31.0",
|
||||
"eslint-config-prettier": "^8.3.0",
|
||||
"eslint-plugin-prettier": "^3.4.0",
|
||||
"nodemon": "^2.0.4",
|
||||
"prettier": "^2.3.2",
|
||||
"ts-node": "^9.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
|
||||
CONFIG_PATH=/data/options.json
|
||||
|
||||
export EMAIL="$(jq --raw-output '.email' $CONFIG_PATH)"
|
||||
export PASSWORD="$(jq --raw-output '.password' $CONFIG_PATH)"
|
||||
export MFA_SECRET="$(jq --raw-output '.mfa_secret' $CONFIG_PATH)"
|
||||
export ACCOUNT_UUID="$(jq --raw-output '.account_uuid' $CONFIG_PATH)"
|
||||
export METER_NUM="$(jq --raw-output '.meter_number' $CONFIG_PATH)"
|
||||
|
||||
echo "Params:"
|
||||
echo "EMAIL =" $EMAIL
|
||||
echo "PASSWORD =" $(sed 's/^........../**********/' <<<$PASSWORD)
|
||||
echo "MFA_SECRET =" $(sed 's/^........../**********/' <<<$MFA_SECRET)
|
||||
echo "ACCOUNT_UUID =" $ACCOUNT_UUID
|
||||
echo "METER_NUM =" $METER_NUM
|
||||
|
||||
# Start the listener and enter an endless loop
|
||||
echo "Starting server!"
|
||||
npm start
|
||||
@@ -0,0 +1,97 @@
|
||||
import chalk from "chalk";
|
||||
import pupeteer from "puppeteer";
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
}
|
||||
|
||||
export class ConEd {
|
||||
browser: pupeteer.Browser | undefined;
|
||||
refresh_interval_min = 15;
|
||||
db_store: ((raw_data: any) => Promise<any>) | undefined;
|
||||
|
||||
public constructor(values: Partial<ConEd>) {
|
||||
Object.assign(this, values);
|
||||
if (!this.valid_config()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
valid_config(): boolean {
|
||||
const req_config = ["EMAIL", "PASSWORD", "MFA_SECRET", "ACCOUNT_UUID", "METER_NUM"];
|
||||
return req_config.every((config) => {
|
||||
if (!process.env[config]) {
|
||||
console.log(chalk.red(`${config} is not set in the .env config! Exiting...`));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
async init(): Promise<void> {
|
||||
this.browser = await pupeteer.launch({
|
||||
defaultViewport: { width: 1920, height: 1080 },
|
||||
args: ["--no-sandbox", "--disable-setuid-sandbox"],
|
||||
});
|
||||
}
|
||||
async login(): Promise<void> {
|
||||
const page = await this.browser!.newPage();
|
||||
// Login to ConEd website
|
||||
await page.goto("https://www.coned.com/en/login");
|
||||
await page.type("#form-login-email", process.env.EMAIL!);
|
||||
await page.type("#form-login-password", process.env.PASSWORD!);
|
||||
await page.click("#form-login-remember-me");
|
||||
await page.click(".submit-button");
|
||||
// Wait for login to authenticate
|
||||
await sleep(5000);
|
||||
await page.screenshot({ path: "meter0.png" });
|
||||
// Enter in 2 factor auth code (see README for details)
|
||||
await page.type("#form-login-mfa-code", process.env.MFA_SECRET!);
|
||||
await page.click(".js-login-new-device-form .button");
|
||||
// Wait for authentication to complete
|
||||
await page.waitForNavigation();
|
||||
await sleep(5000);
|
||||
await page.screenshot({ path: "meter1.png" });
|
||||
}
|
||||
async read_api(): Promise<any> {
|
||||
return new Promise(async (resolve) => {
|
||||
// Access the API using your newly acquired authentication cookies!
|
||||
const api_page = await this.browser!.newPage();
|
||||
const api_url = `https://cned.opower.com/ei/edge/apis/cws-real-time-ami-v1/cws/cned/accounts/${process.env.ACCOUNT_UUID}/meters/${process.env.METER_NUM}/usage`;
|
||||
await api_page.goto(api_url);
|
||||
await api_page.screenshot({ path: "meter2.png" });
|
||||
const data_elem = await api_page.$("pre");
|
||||
const text_data = await api_page.evaluate((el: HTMLElement) => el.textContent, data_elem);
|
||||
const raw_data = JSON.parse(text_data!);
|
||||
await api_page.close();
|
||||
resolve(raw_data);
|
||||
});
|
||||
}
|
||||
async fetch_once(): Promise<void> {
|
||||
await this.init();
|
||||
await this.login();
|
||||
let raw_data = await this.read_api();
|
||||
let attempt = 1;
|
||||
while ("error" in raw_data && attempt < 5) {
|
||||
console.log(chalk.yellow("Failed to fetch data from API:", raw_data["error"]["details"]));
|
||||
raw_data = await this.read_api();
|
||||
await sleep(15000);
|
||||
attempt++;
|
||||
}
|
||||
if (attempt < 5) {
|
||||
console.log(chalk.green("Successfully retrieved API data!"));
|
||||
this.db_store!(raw_data);
|
||||
} else {
|
||||
console.log(chalk.red("Failed to retrieve API data:", raw_data["error"]));
|
||||
}
|
||||
await this.browser?.close();
|
||||
}
|
||||
monitor(interval_min = 15): void {
|
||||
this.fetch_once();
|
||||
setInterval(() => {
|
||||
this.fetch_once();
|
||||
}, interval_min * 60 * 1000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
// Import config from .env
|
||||
|
||||
// eslint-disable-next-line
|
||||
require("dotenv").config();
|
||||
|
||||
import { ConEd } from "./coned";
|
||||
import { connect } from "trilogy";
|
||||
import chalk from "chalk";
|
||||
import fs from "fs";
|
||||
import moment from "moment";
|
||||
import fastify from "fastify";
|
||||
import axios from "axios";
|
||||
|
||||
const output_dir = "out";
|
||||
|
||||
if (!fs.existsSync(output_dir)) {
|
||||
fs.mkdirSync(output_dir);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const db = connect(`./${output_dir}/storage.db`);
|
||||
|
||||
process.on("exit", function () {
|
||||
db.close();
|
||||
console.log(chalk.green("Successfully closed DB connection!"));
|
||||
});
|
||||
|
||||
const energySchema = {
|
||||
id: "increments",
|
||||
startTime: { type: Date, unique: true },
|
||||
endTime: { type: Date, unique: true },
|
||||
energy: { type: Number },
|
||||
};
|
||||
type EnergyEntry = {
|
||||
id: number;
|
||||
startTime: Date;
|
||||
endTime: Date;
|
||||
energy: number;
|
||||
};
|
||||
const energyModel = await db.model<EnergyEntry>("energy", energySchema);
|
||||
|
||||
let latest_read_startTime = new Date(0);
|
||||
async function update_sensor(latest_read: any) {
|
||||
if (latest_read.startTime < latest_read_startTime) return;
|
||||
latest_read_startTime = latest_read.startTime;
|
||||
axios.post("http://supervisor/core/api/states/sensor.coned_energy", {
|
||||
state: latest_read.value,
|
||||
attributes: { unit_of_measurement: "kWh", friendly_name: "ConEd Energy Usage", device_class: "energy", state_class: "measurement", last_reset: latest_read.startTime },
|
||||
}, {
|
||||
headers: {"Authorization": `Bearer ${process.env.SUPERVISOR_TOKEN}`, "Content-Type": "application/json"}
|
||||
});
|
||||
console.log(chalk.green(`Sent sensor update: ${latest_read.value} kWh @ ${moment(latest_read.startTime).format("LLL")}`));
|
||||
}
|
||||
|
||||
async function db_store(raw_data: any) {
|
||||
const filtered_data = raw_data.reads.filter((row: any) => {
|
||||
return row.value !== null && row.value !== undefined;
|
||||
})
|
||||
const latest_read = filtered_data[0];
|
||||
update_sensor(latest_read);
|
||||
for (const row of filtered_data) {
|
||||
await energyModel.updateOrCreate({ startTime: new Date(row.startTime), endTime: new Date(row.endTime) }, { energy: row.value }).catch((e) => {
|
||||
console.log(chalk.red(`Err: ${e}`));
|
||||
});
|
||||
}
|
||||
console.log(chalk.green("Database Updated!"));
|
||||
console.log(chalk.green("Database Closed!"));
|
||||
}
|
||||
|
||||
const c = new ConEd({ db_store });
|
||||
c.monitor();
|
||||
|
||||
const server = fastify({
|
||||
//logger: true,
|
||||
});
|
||||
|
||||
// Declare a route
|
||||
server.get<{
|
||||
Querystring: { h: number };
|
||||
}>("/", (request, reply) => {
|
||||
console.log(request.query);
|
||||
let hour_lookback = 24;
|
||||
if (request.query.h) {
|
||||
if (!!Number(request.query.h)) {
|
||||
hour_lookback = Number(request.query.h);
|
||||
} else {
|
||||
reply.code(400).type("text/plain").send("Err 400: Invalid Request");
|
||||
return;
|
||||
}
|
||||
}
|
||||
const d = new Date();
|
||||
d.setHours(d.getHours() - hour_lookback);
|
||||
energyModel.find([["startTime", ">", d]]).then((raw_data) => {
|
||||
reply.send({ data: raw_data });
|
||||
});
|
||||
});
|
||||
|
||||
server.get<{
|
||||
Querystring: { n: number };
|
||||
}>("/latest", (request, reply) => {
|
||||
console.log(request.query);
|
||||
let n_results = 64;
|
||||
if (request.query.n) {
|
||||
if (!!Number(request.query.n)) {
|
||||
n_results = Number(request.query.n);
|
||||
} else {
|
||||
reply.code(400).type("text/plain").send("Err 400: Invalid Request");
|
||||
return;
|
||||
}
|
||||
}
|
||||
energyModel.find({}, { limit: n_results, order: ["startTime", "desc"] }).then((raw_data) => {
|
||||
function to_delta_string(dt: Date): string {
|
||||
const delta_min = Math.abs(dt.getTime() - new Date().getTime()) / 6e4;
|
||||
return delta_min < 60 ? `${delta_min.toFixed(0)}min` : `${(delta_min / 60).toPrecision(2)}hr`;
|
||||
}
|
||||
//Create array of energy readings starting with the most recent
|
||||
const readings = Object.values(raw_data).map((row: EnergyEntry) => row.energy);
|
||||
//Time of earliest read in list
|
||||
const earliest_dt: Date = Object.values(raw_data)[raw_data.length - 1].startTime;
|
||||
const earliest_delta = to_delta_string(earliest_dt);
|
||||
//Time of peak (most recent if tie)
|
||||
const peak_index = readings.indexOf(Math.max.apply(null, readings));
|
||||
const peak = Object.values(raw_data)[peak_index].energy;
|
||||
const peak_dt = Object.values(raw_data)[peak_index].startTime;
|
||||
const peak_delta = to_delta_string(peak_dt);
|
||||
//Time of latest read in list
|
||||
const latest_dt: Date = Object.values(raw_data)[0].startTime;
|
||||
const latest_delta = to_delta_string(latest_dt);
|
||||
const resp_data = { earliest_delta, latest_delta, peak, peak_delta, readings };
|
||||
reply.send(resp_data);
|
||||
});
|
||||
});
|
||||
|
||||
// Run the server!
|
||||
server.listen(3000, "0.0.0.0", function (err, address) {
|
||||
if (err) {
|
||||
server.log.error(err);
|
||||
process.exit(1);
|
||||
}
|
||||
server.log.info(`server listening on ${address}`);
|
||||
});
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2018",
|
||||
"module": "commonjs",
|
||||
"outDir": "./lib",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true
|
||||
},
|
||||
"exclude": [
|
||||
"test",
|
||||
"statmed-frontend"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user