From 2c58f9cf64fe818b2a093a0789523f3174fb42c5 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 00:15:05 -0400 Subject: [PATCH 01/70] chore: remove unused express routes code --- api/server.js | 9 ++------- api/src/app.js | 24 +----------------------- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/api/server.js b/api/server.js index 9bdc2be6..8398d9f9 100644 --- a/api/server.js +++ b/api/server.js @@ -1,8 +1,7 @@ -/* eslint-disable no-irregular-whitespace */ const http = require('http'); const { version } = require('./package.json'); const mqtt = require('./src/util/mqtt.util'); -const { app /* , routes */ } = require('./src/app'); +const { app } = require('./src/app'); const logger = require('./src/util/logger.util'); const storage = require('./src/util/storage.util'); const database = require('./src/util/db.util'); @@ -18,11 +17,7 @@ module.exports.start = async () => { await database.init(); - http.Server(app).listen(SERVER.PORT, () => { - // logger.log(`listening on 0.0.0.0:${PORT}`); - // logger.log('registered routes:'); - // logger.log(routes); - }); + http.Server(app).listen(SERVER.PORT); mqtt.connect(); storage.purge(); diff --git a/api/src/app.js b/api/src/app.js index 4b90c78e..79e91302 100644 --- a/api/src/app.js +++ b/api/src/app.js @@ -12,26 +12,4 @@ app.use('/api/storage/train', express.static(`${STORAGE.PATH}/train`)); app.use('/api/tmp', express.static(`/tmp`)); app.use('/', express.static(`./frontend`)); -let routes = []; -// eslint-disable-next-line no-underscore-dangle -app._router.stack.forEach((middleware) => { - if (middleware.route) { - routes.push(middleware.route); - } else if (middleware.name === 'router') { - middleware.handle.stack.forEach((handler) => { - if (handler.route) routes.push(handler.route); - }); - } -}); -routes = routes.map((route) => { - const methods = []; - for (const [key, value] of Object.entries(route.methods)) { - if (value) methods.push(key); - } - return { - path: route.path, - methods: methods.join(', '), - }; -}); - -module.exports = { app, routes }; +module.exports = { app }; From eb862cd609927e30005899db3faf89bc098d8bd1 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 00:45:05 -0400 Subject: [PATCH 02/70] chore(logger): use winston as logger --- api/package-lock.json | 142 ++++++++++++++++++++ api/package.json | 3 +- api/server.js | 7 +- api/src/controllers/recognize.controller.js | 12 +- api/src/controllers/train.controller.js | 12 +- api/src/util/db.util.js | 7 +- api/src/util/detectors/factory.js | 4 +- api/src/util/frigate.util.js | 3 +- api/src/util/fs.util.js | 14 +- api/src/util/logger.util.js | 53 +++++--- api/src/util/notify/actions/index.js | 5 +- api/src/util/notify/factory.js | 4 +- api/src/util/process.util.js | 14 +- api/src/util/respond.util.js | 3 +- api/src/util/storage.util.js | 6 +- api/src/util/train.util.js | 16 +-- api/src/util/validators.util.js | 3 +- 17 files changed, 218 insertions(+), 90 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index 9e74ac5e..fb626e4d 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -4,6 +4,16 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@dabh/diagnostics": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", + "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", + "requires": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "@mapbox/node-pre-gyp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz", @@ -81,6 +91,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, + "async": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -264,6 +279,31 @@ "simple-swizzle": "^0.2.2" } }, + "colors": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", + "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" + }, + "colorspace": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", + "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", + "requires": { + "color": "3.0.x", + "text-hex": "1.0.x" + }, + "dependencies": { + "color": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", + "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", + "requires": { + "color-convert": "^1.9.1", + "color-string": "^1.5.2" + } + } + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -442,6 +482,11 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, + "enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -539,6 +584,16 @@ "validator": "^13.5.2" } }, + "fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + }, + "fecha": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.1.tgz", + "integrity": "sha512-MMMQ0ludy/nBs1/o0zVOiKTpG7qMbonKUzjJgQFEuvq6INZ1OraKPRAWkBq5vlKLOUMpmNYG1JoN3oDPUQ9m3Q==" + }, "file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -573,6 +628,11 @@ } } }, + "fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, "follow-redirects": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.1.tgz", @@ -798,6 +858,11 @@ "define-properties": "^1.1.3" } }, + "is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -811,6 +876,11 @@ "argparse": "^2.0.1" } }, + "kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, "leven": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-2.1.0.tgz", @@ -821,6 +891,18 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "logform": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", + "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", + "requires": { + "colors": "^1.2.1", + "fast-safe-stringify": "^2.0.4", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "triple-beam": "^1.3.0" + } + }, "long-timeout": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", @@ -1073,6 +1155,14 @@ "wrappy": "1" } }, + "one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "requires": { + "fn.name": "1.x.x" + } + }, "parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1359,6 +1449,11 @@ } } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1455,11 +1550,21 @@ } } }, + "text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "triple-beam": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", + "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -1520,6 +1625,43 @@ "string-width": "^1.0.2 || 2" } }, + "winston": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", + "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", + "requires": { + "@dabh/diagnostics": "^2.0.2", + "async": "^3.1.0", + "is-stream": "^2.0.0", + "logform": "^2.2.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "winston-transport": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", + "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", + "requires": { + "readable-stream": "^2.3.7", + "triple-beam": "^1.2.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/api/package.json b/api/package.json index e26b8c56..b72ebabd 100644 --- a/api/package.json +++ b/api/package.json @@ -32,6 +32,7 @@ "mqtt": "^4.2.8", "node-schedule": "^2.0.0", "sharp": "^0.28.3", - "uuid": "^8.3.2" + "uuid": "^8.3.2", + "winston": "^3.3.3" } } diff --git a/api/server.js b/api/server.js index 8398d9f9..a81f13a6 100644 --- a/api/server.js +++ b/api/server.js @@ -2,18 +2,19 @@ const http = require('http'); const { version } = require('./package.json'); const mqtt = require('./src/util/mqtt.util'); const { app } = require('./src/app'); -const logger = require('./src/util/logger.util'); const storage = require('./src/util/storage.util'); const database = require('./src/util/db.util'); const config = require('./src/constants/config'); const shutdown = require('./src/util/shutdown.util'); const { SERVER } = require('./src/constants'); +require('./src/util/logger.util').init(); + module.exports.start = async () => { storage.setup(); - logger.log(`Double Take v${version}`); + console.log(`Double Take v${version}`); - logger.log(config()); + console.log(config()); await database.init(); diff --git a/api/src/controllers/recognize.controller.js b/api/src/controllers/recognize.controller.js index 14b2d168..ec08cefb 100644 --- a/api/src/controllers/recognize.controller.js +++ b/api/src/controllers/recognize.controller.js @@ -3,7 +3,6 @@ const { v4: uuidv4 } = require('uuid'); const process = require('../util/process.util'); const actions = require('../util/detectors/actions'); const notify = require('../util/notify/actions'); -const logger = require('../util/logger.util'); const time = require('../util/time.util'); const recognize = require('../util/recognize.util'); const frigate = require('../util/frigate.util'); @@ -85,7 +84,7 @@ module.exports.start = async (req, res) => { } } - logger.log(`${time.current()}\nprocessing ${camera}: ${id}`, { dashes: true }); + console.log(`processing ${camera}: ${id}`); perf.start('request'); PROCESSING = true; @@ -153,11 +152,8 @@ module.exports.start = async (req, res) => { if (resultsOutput === 'all') output.results = results; - logger.log('response:'); - logger.log(output); - logger.log(`${time.current()}\ndone processing ${camera}: ${id} in ${duration} sec`, { - dashes: true, - }); + console.log(`done processing ${camera}: ${id} in ${duration} sec`); + console.log(output); PROCESSING = false; @@ -171,7 +167,7 @@ module.exports.start = async (req, res) => { IDS.push(id); } } catch (error) { - logger.log(error.message); + console.error(error.message); PROCESSING = false; respond(error, res); } diff --git a/api/src/controllers/train.controller.js b/api/src/controllers/train.controller.js index 3cd4dcd4..a85e80ce 100644 --- a/api/src/controllers/train.controller.js +++ b/api/src/controllers/train.controller.js @@ -3,8 +3,6 @@ const fs = require('fs'); const sharp = require('sharp'); const database = require('../util/db.util'); const train = require('../util/train.util'); -const logger = require('../util/logger.util'); -const time = require('../util/time.util'); const filesystem = require('../util/fs.util'); const { respond, HTTPSuccess } = require('../util/respond.util'); const { OK } = require('../constants/http-status'); @@ -80,15 +78,13 @@ module.exports.delete = async (req, res) => { } const results = [...(await Promise.all(promises))]; - logger.log( - `${time.current()}\ndone untraining for ${name} in ${parseFloat( - (perf.stop().time / 1000).toFixed(2) - )} sec` + console.log( + `done untraining for ${name} in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec` ); respond(HTTPSuccess(OK, results), res); } catch (error) { - logger.log(`train delete error: ${error.message}`); + console.error(`train delete error: ${error.message}`); respond(error, res); } }; @@ -109,7 +105,7 @@ module.exports.add = async (req, res) => { await train.queue(images); } catch (error) { - logger.log(`train init error: ${error.message}`); + console.error(`train init error: ${error.message}`); respond(error, res); } }; diff --git a/api/src/util/db.util.js b/api/src/util/db.util.js index 51e2e7d5..c3d7bad7 100644 --- a/api/src/util/db.util.js +++ b/api/src/util/db.util.js @@ -1,7 +1,6 @@ const Database = require('better-sqlite3'); const time = require('./time.util'); const filesystem = require('./fs.util'); -const logger = require('./logger.util'); const { STORAGE, SAVE } = require('../constants'); @@ -57,7 +56,7 @@ module.exports.init = async () => { const files = await filesystem.files().train(); database.insert('init', files); } catch (error) { - logger.log(`db init error: ${error.message}`); + console.error(`db init error: ${error.message}`); } }; @@ -101,7 +100,7 @@ module.exports.migrations = () => { ); } } catch (error) { - logger.log(`db migrations error: ${error.message}`); + console.error(`db migrations error: ${error.message}`); } }; @@ -134,7 +133,7 @@ module.exports.files = (status, data) => { return files; } catch (error) { - logger.log(`files error: ${error.message}`); + console.error(`files error: ${error.message}`); } }; diff --git a/api/src/util/detectors/factory.js b/api/src/util/detectors/factory.js index c97104a7..232e86c5 100644 --- a/api/src/util/detectors/factory.js +++ b/api/src/util/detectors/factory.js @@ -1,5 +1,3 @@ -const logger = require('../logger.util'); - const detectors = require('.'); /** @@ -9,6 +7,6 @@ module.exports.get = (detector) => { try { return detectors[detector]; } catch (error) { - logger.log(`${detector} factory error: ${error.message}`); + console.error(`${detector} factory error: ${error.message}`); } }; diff --git a/api/src/util/frigate.util.js b/api/src/util/frigate.util.js index e553280d..78c81efa 100644 --- a/api/src/util/frigate.util.js +++ b/api/src/util/frigate.util.js @@ -1,6 +1,5 @@ const axios = require('axios'); const sleep = require('./sleep.util'); -const logger = require('./logger.util'); const { FRIGATE } = require('../constants'); @@ -96,7 +95,7 @@ module.exports.snapshotReady = async (id) => { await sleep(0.05); } if (!ready) { - logger.log('frigate snapshot ready error'); + console.error('frigate snapshot ready error'); } return ready; }; diff --git a/api/src/util/fs.util.js b/api/src/util/fs.util.js index 631fd69c..b16d3f7b 100644 --- a/api/src/util/fs.util.js +++ b/api/src/util/fs.util.js @@ -1,6 +1,4 @@ const fs = require('fs'); -// const { v4: uuidv4 } = require('uuid'); -const logger = require('./logger.util'); const { STORAGE } = require('../constants'); module.exports.folders = () => { @@ -69,7 +67,7 @@ module.exports.writerStream = async (stream, file) => { resolve(); }) .on('error', (error) => { - logger.log(`writer error: ${error.message}`); + console.error(`writer error: ${error.message}`); }); }); }; @@ -81,18 +79,18 @@ module.exports.writeMatches = (name, source, destination) => { } fs.copyFile(source, destination, (error) => { if (error) { - logger.log(`write match error: ${error.message}`); + console.error(`write match error: ${error.message}`); } }); } catch (error) { - logger.log(`create match folder error: ${error.message}`); + console.error(`create match folder error: ${error.message}`); } }; module.exports.copy = (source, destination) => { fs.copyFile(source, destination, (error) => { if (error) { - logger.log(`copy file error: ${error.message}`); + console.error(`copy file error: ${error.message}`); } }); }; @@ -103,7 +101,7 @@ module.exports.delete = (destination) => { fs.unlinkSync(destination); } } catch (error) { - logger.log(`delete error: ${error.message}`); + console.error(`delete error: ${error.message}`); } }; @@ -113,6 +111,6 @@ module.exports.move = (source, destination) => { fs.renameSync(source, destination); } } catch (error) { - logger.log(`move error: ${error.message}`); + console.error(`move error: ${error.message}`); } }; diff --git a/api/src/util/logger.util.js b/api/src/util/logger.util.js index 92edd3cf..d0a21970 100644 --- a/api/src/util/logger.util.js +++ b/api/src/util/logger.util.js @@ -1,28 +1,37 @@ -const os = require('os'); -const fs = require('fs'); +const { createLogger, format, transports } = require('winston'); +const util = require('util'); const { STORAGE } = require('../constants'); -let logStream = false; - -module.exports.log = (message) => { - try { - if (!logStream) - logStream = fs.createWriteStream(`${STORAGE.PATH}/messages.log`, { flags: 'a' }); +const combineMessageAndSplat = () => { + return { + transform: (info /* , opts */) => { + info.message = util.format(info.message, ...(info[Symbol.for('splat')] || [])); + return info; + }, + }; +}; - const logMessage = - typeof message === 'string' - ? message.replace(/\n/g, os.EOL) - : JSON.stringify(message, null, '\t'); - logStream.write(logMessage + os.EOL); +module.exports.init = () => { + const logFormat = format.combine(combineMessageAndSplat(), format.simple()); - if (typeof message === 'object') console.dir(message, { depth: null }); - else console.log(message); - } catch (error) { - console.log('logger error'); - console.log(error); - } -}; + const logger = createLogger({ + transports: [ + new transports.Console({ + format: format.combine(format.colorize(), logFormat), + }), + new transports.File({ + filename: `${STORAGE.PATH}/messages.log`, + format: format.combine( + logFormat, + format.timestamp({ format: 'YY-MM-DD HH:mm:ss' }), + format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`) + ), + }), + ], + }); -module.exports.dashes = (message) => { - this.log('-'.repeat(message.length)); + console.log = (...args) => logger.info(...args); + console.info = (...args) => logger.info(...args); + console.warn = (...args) => logger.warn(...args); + console.error = (...args) => logger.error(...args); }; diff --git a/api/src/util/notify/actions/index.js b/api/src/util/notify/actions/index.js index 64ecdffd..6443ee82 100644 --- a/api/src/util/notify/actions/index.js +++ b/api/src/util/notify/actions/index.js @@ -1,6 +1,5 @@ const factory = require('../factory'); const { lowercaseKeys } = require('../../helpers.util'); -const logger = require('../../logger.util'); const { NOTIFY } = require('../../../constants'); module.exports.send = (service, output) => factory.get(service).send(output); @@ -14,10 +13,10 @@ module.exports.publish = (output, camera, zones) => { const check = this.checks(service, { camera, zones }); if (check === true) { this.send(service, output).catch((error) => { - logger.log(`${service} send error: ${error.message}`); + console.error(`${service} send error: ${error.message}`); }); } else { - logger.log(`${service}: ${check}`); + console.error(`${service}: ${check}`); } } }; diff --git a/api/src/util/notify/factory.js b/api/src/util/notify/factory.js index aaafd0bf..697abaf2 100644 --- a/api/src/util/notify/factory.js +++ b/api/src/util/notify/factory.js @@ -1,11 +1,9 @@ -const logger = require('../logger.util'); - const services = require('.'); module.exports.get = (service) => { try { return services[service]; } catch (error) { - logger.log(`${service} factory error: ${error.message}`); + console.error(`${service} factory error: ${error.message}`); } }; diff --git a/api/src/util/process.util.js b/api/src/util/process.util.js index 10054c90..57873df1 100644 --- a/api/src/util/process.util.js +++ b/api/src/util/process.util.js @@ -2,7 +2,6 @@ const axios = require('axios'); const fs = require('fs'); const perf = require('execution-time')(); const { v4: uuidv4 } = require('uuid'); -const logger = require('./logger.util'); const sleep = require('./sleep.util'); const filesystem = require('./fs.util'); const database = require('./db.util'); @@ -93,7 +92,7 @@ module.exports.save = async (event, results, filename, tmp) => { }); await filesystem.writerStream(fs.createReadStream(tmp), `${STORAGE.PATH}/matches/${filename}`); } catch (error) { - logger.log(`save results error: ${error.message}`); + console.error(`save results error: ${error.message}`); } }; @@ -106,9 +105,9 @@ module.exports.process = async ({ detector, tmp }) => { return { duration, results: normalize({ detector, data }) }; } catch (error) { if (error.response && error.response.data.error) { - logger.log(`${detector} process error: ${error.response.data.error}`); + console.error(`${detector} process error: ${error.response.data.error}`); } else { - logger.log(`${detector} process error: ${error.message}`); + console.error(`${detector} process error: ${error.message}`); } } }; @@ -123,12 +122,11 @@ module.exports.isValidURL = async ({ type, url }) => { const { headers } = request; const isValid = validOptions.includes(headers['content-type']); if (!isValid) { - logger.log(`url validation failed for ${type}: ${url}`); - logger.log(`content type: ${headers['content-type']}`); + console.error(`url validation failed for ${type}: ${url}`); } return isValid; } catch (error) { - logger.log(`url validation error: ${error.message}`); + console.error(`url validation error: ${error.message}`); return false; } }; @@ -142,7 +140,7 @@ module.exports.stream = async (url) => { }); return request.data; } catch (error) { - logger.log(`stream error: ${error.message}`); + console.error(`stream error: ${error.message}`); } }; diff --git a/api/src/util/respond.util.js b/api/src/util/respond.util.js index 11911810..51b04f3a 100644 --- a/api/src/util/respond.util.js +++ b/api/src/util/respond.util.js @@ -1,4 +1,3 @@ -const logger = require('./logger.util'); const { BAD_REQUEST } = require('../constants/http-status'); module.exports.respond = (err, res) => { @@ -17,7 +16,7 @@ module.exports.respond = (err, res) => { return res.status(status).json(message); } catch (error) { - logger.log(err); + console.error(err.toString()); } }; diff --git a/api/src/util/storage.util.js b/api/src/util/storage.util.js index 0db43e53..76bd2177 100644 --- a/api/src/util/storage.util.js +++ b/api/src/util/storage.util.js @@ -1,7 +1,5 @@ const fs = require('fs'); const schedule = require('node-schedule'); -const logger = require('./logger.util'); -const time = require('./time.util'); const database = require('./db.util'); const { STORAGE, PURGE } = require('../constants'); @@ -42,9 +40,9 @@ module.exports.purge = async () => { const ids = files.map(({ id }) => id); db.prepare(`DELETE FROM match WHERE id IN (${ids.join(',')})`).run(); - if (files.length > 0) logger.log(`${time.current()}\npurged ${files.length} file(s)`); + if (files.length > 0) console.log(`purged ${files.length} file(s)`); } catch (error) { - logger.log(`purge error: ${error.message}`); + console.error(`purge error: ${error.message}`); } }); }; diff --git a/api/src/util/train.util.js b/api/src/util/train.util.js index 12992329..315a7d6e 100644 --- a/api/src/util/train.util.js +++ b/api/src/util/train.util.js @@ -1,6 +1,4 @@ const perf = require('execution-time')(); -const time = require('./time.util'); -const logger = require('./logger.util'); const database = require('./db.util'); const { train, remove } = require('./detectors/actions'); @@ -9,7 +7,7 @@ const { DETECTORS } = require('../constants'); module.exports.queue = async (files) => { try { perf.start(); - logger.log(`${time.current()}\nqueuing ${files.length} file(s) for training`); + console.log(`queuing ${files.length} file(s) for training`); const inserts = []; const outputs = []; @@ -17,7 +15,7 @@ module.exports.queue = async (files) => { const output = []; const promises = []; const { id, name, filename, key } = files[i]; - logger.log(`file ${i + 1}: ${name} - ${filename}`); + console.log(`file ${i + 1}: ${name} - ${filename}`); const detectors = Object.fromEntries( Object.entries(DETECTORS).map(([k, v]) => [k.toLowerCase(), v]) @@ -40,10 +38,10 @@ module.exports.queue = async (files) => { outputs.push(output); } database.insert('train', inserts); - logger.log(`training complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec`); + console.log(`training complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec`); return outputs; } catch (error) { - logger.log(`queue error: ${error.message}`); + console.error(`queue error: ${error.message}`); } }; @@ -60,11 +58,11 @@ module.exports.process = async ({ name, key, detector }) => { const message = typeof data === 'string' ? { data } : { ...data }; if (data.message) { - logger.log(`${detector} training error: ${data.message}`); + console.error(`${detector} training error: ${data.message}`); } else if (data.error) { - logger.log(`${detector} training error: ${data.error}`); + console.error(`${detector} training error: ${data.error}`); } else { - logger.log(`${detector} training error: ${error.message}`); + console.error(`${detector} training error: ${error.message}`); } return { diff --git a/api/src/util/validators.util.js b/api/src/util/validators.util.js index 9b99df21..59268afe 100644 --- a/api/src/util/validators.util.js +++ b/api/src/util/validators.util.js @@ -1,5 +1,4 @@ const axios = require('axios'); -const logger = require('./logger.util'); const { expressValidator } = require('./validate.util'); const { query, param /* , body */ } = expressValidator; @@ -90,7 +89,7 @@ module.exports.doesUrlResolve = async (url) => { }); return data; } catch (error) { - logger.log(`url resolve error: ${error.message}`); + console.error(`url resolve error: ${error.message}`); return false; } }; From d105843d5f3c8ddc6da090af28a37b2d163fd916 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 02:18:00 -0400 Subject: [PATCH 03/70] feat: support for home assistant mqtt auto discovery #38 also includes mqtt util refactor --- api/src/constants/defaults.js | 1 + api/src/controllers/recognize.controller.js | 2 +- api/src/util/mqtt.util.js | 340 ++++++++++++-------- 3 files changed, 215 insertions(+), 128 deletions(-) diff --git a/api/src/constants/defaults.js b/api/src/constants/defaults.js index 0c3e6661..3e06ba67 100644 --- a/api/src/constants/defaults.js +++ b/api/src/constants/defaults.js @@ -25,6 +25,7 @@ module.exports = { frigate: 'frigate/events', matches: 'double-take/matches', cameras: 'double-take/cameras', + homeassistant: 'homeassistant', }, }, notify: { diff --git a/api/src/controllers/recognize.controller.js b/api/src/controllers/recognize.controller.js index ec08cefb..3b001923 100644 --- a/api/src/controllers/recognize.controller.js +++ b/api/src/controllers/recognize.controller.js @@ -159,7 +159,7 @@ module.exports.start = async (req, res) => { respond(HTTPSuccess(OK, output), res); - mqtt.publish(output); + mqtt.recognize(output); notify.publish(output, camera, results); diff --git a/api/src/util/mqtt.util.js b/api/src/util/mqtt.util.js index 4efe709d..dfa46d86 100644 --- a/api/src/util/mqtt.util.js +++ b/api/src/util/mqtt.util.js @@ -2,14 +2,13 @@ const { v4: uuidv4 } = require('uuid'); const axios = require('axios'); const mqtt = require('mqtt'); const fs = require('fs'); -const logger = require('./logger.util'); const { contains } = require('./helpers.util'); const { SERVER, MQTT, FRIGATE, CAMERAS } = require('../constants'); -let previousMqttLengths = []; -let justSubscribed = false; -let client = false; -const personResetTimeout = {}; +let PREVIOUS_MQTT_LENGTHS = []; +let JUST_SUBSCRIBED = false; +let CLIENT = false; +const PERSON_RESET_TIMEOUT = {}; const cameraTopics = () => { return CAMERAS @@ -21,174 +20,261 @@ const cameraTopics = () => { : []; }; +const processMessage = ({ topic, message }) => { + const init = async () => { + if ((topic.includes('/snapshot') || cameraTopics().includes(topic)) && !JUST_SUBSCRIBED) + await processMessage({ topic, message }).snapshot(); + if (topic.includes('/events')) await processMessage({ topic, message }).frigate(); + }; + + const snapshot = async () => { + const camera = topic.split('/')[1]; + const filename = `${uuidv4()}.jpg`; + const buffer = Buffer.from(message); + + if (PREVIOUS_MQTT_LENGTHS.includes(buffer.length)) { + return; + } + PREVIOUS_MQTT_LENGTHS.unshift(buffer.length); + + fs.writeFileSync(`/tmp/${filename}`, buffer); + await axios({ + method: 'get', + url: `http://0.0.0.0:${SERVER.PORT}/api/recognize`, + params: { + url: `http://0.0.0.0:${SERVER.PORT}/api/tmp/${filename}`, + type: 'mqtt', + camera, + }, + validateStatus() { + return true; + }, + }); + // only store last 10 mqtt lengths + PREVIOUS_MQTT_LENGTHS = PREVIOUS_MQTT_LENGTHS.slice(0, 10); + }; + + const frigate = async () => { + await axios({ + method: 'post', + url: `http://0.0.0.0:${SERVER.PORT}/api/recognize`, + data: { + ...JSON.parse(message.toString()), + }, + validateStatus() { + return true; + }, + }); + }; + + return { init, snapshot, frigate }; +}; + module.exports.connect = () => { if (!MQTT.HOST) { return; } - client = mqtt.connect(`mqtt://${MQTT.HOST}`, { + CLIENT = mqtt.connect(`mqtt://${MQTT.HOST}`, { reconnectPeriod: 10000, username: MQTT.USERNAME, password: MQTT.PASSWORD, }); - client - .on('connect', () => { - logger.log('MQTT: connected'); - this.available('online'); - this.subscribe(); - }) - .on('error', (err) => { - logger.log(`MQTT: ${err.code}`); - }) - .on('offline', () => { - logger.log('MQTT: offline'); - }) - .on('disconnect', () => { - logger.log('MQTT: disconnected'); - }) - .on('reconnect', () => { - logger.log('MQTT: attempting to reconnect'); - }) - .on('message', async (topic, message) => { - if ((topic.includes('/snapshot') || cameraTopics().includes(topic)) && !justSubscribed) { - const camera = topic.split('/')[1]; - const filename = `${uuidv4()}.jpg`; - const buffer = Buffer.from(message); - - if (previousMqttLengths.includes(buffer.length)) { - return; - } - previousMqttLengths.unshift(buffer.length); - - fs.writeFileSync(`/tmp/${filename}`, buffer); - await axios({ - method: 'get', - url: `http://0.0.0.0:${SERVER.PORT}/api/recognize`, - params: { - url: `http://0.0.0.0:${SERVER.PORT}/api/tmp/${filename}`, - type: 'mqtt', - camera, - }, - validateStatus() { - return true; - }, - }); - // only store last 10 mqtt lengths - previousMqttLengths = previousMqttLengths.slice(0, 10); - } else if (topic.includes('/events')) { - await axios({ - method: 'post', - url: `http://0.0.0.0:${SERVER.PORT}/api/recognize`, - data: JSON.parse(message.toString()), - validateStatus() { - return true; - }, - }); - } - }); + CLIENT.on('connect', () => { + console.log('MQTT: connected'); + this.available('online'); + this.subscribe(); + }) + .on('error', (err) => console.error(`MQTT: ${err.code}`)) + .on('offline', () => console.error('MQTT: offline')) + .on('disconnect', () => console.error('MQTT: disconnected')) + .on('reconnect', () => console.warn('MQTT: attempting to reconnect')) + .on('message', async (topic, message) => processMessage({ topic, message }).init()); }; module.exports.available = async (state) => { - if (client) await client.publish('double-take/available', state, { retain: true }); + if (CLIENT) this.publish({ topic: 'double-take/available', message: state }); }; -module.exports.publish = (data) => { +module.exports.subscribe = () => { + const topics = []; + + topics.push(...cameraTopics()); + + if (FRIGATE.URL && MQTT.TOPICS.FRIGATE) { + const isArray = Array.isArray(MQTT.TOPICS.FRIGATE); + + const frigateTopics = isArray ? MQTT.TOPICS.FRIGATE : [MQTT.TOPICS.FRIGATE]; + topics.push(...frigateTopics); + frigateTopics.forEach((topic) => { + const [prefix] = topic.split('/'); + topics.push( + ...(FRIGATE.CAMERAS + ? FRIGATE.CAMERAS.map((camera) => `${prefix}/${camera}/person/snapshot`) + : [`${prefix}/+/person/snapshot`]) + ); + }); + } + + if (topics.length) { + CLIENT.subscribe(topics, (err) => { + if (err) { + console.error(`MQTT: error subscribing to ${topics.join(', ')}`); + return; + } + console.log(`MQTT: subscribed to ${topics.join(', ')}`); + JUST_SUBSCRIBED = true; + setTimeout(() => (JUST_SUBSCRIBED = false), 5000); + }); + } +}; + +module.exports.recognize = (data) => { try { - if (!MQTT || !MQTT.HOST) { - return; - } + if (!MQTT || !MQTT.HOST) return; const { matches, unknown, camera } = data; + const hasUnkown = unknown && Object.keys(unknown).length; const configData = JSON.parse(JSON.stringify(data)); delete configData.matches; delete configData.unknown; delete configData.results; - if (unknown && Object.keys(unknown).length) { - client.publish( - `${MQTT.TOPICS.MATCHES}/unknown`, - JSON.stringify({ + const messages = []; + + let personCount = matches.length ? matches.length : hasUnkown ? 1 : 0; + // check to see if unknown bounding box is contained within or contains any of the match bounding boxes + // if false, then add 1 to the person count + if (matches.length && hasUnkown) { + let unknownContained = false; + matches.forEach((match) => { + if (contains(match.box, unknown.box) || contains(unknown.box, match.box)) + unknownContained = true; + }); + if (!unknownContained) personCount += 1; + } + + messages.push({ + topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`, + message: personCount.toString(), + }); + + if (hasUnkown) { + messages.push({ + topic: `${MQTT.TOPICS.MATCHES}/unknown`, + message: JSON.stringify({ ...configData, unknown, }), - { retain: true } - ); + }); + + if (MQTT.TOPICS.HOMEASSISTANT) { + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/unknown/config`, + message: JSON.stringify({ + name: 'unknown', + icon: 'mdi:account', + value_template: '{{ value_json.camera }}', + state_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/unknown/state`, + json_attributes_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/unknown/state`, + availability_topic: 'double-take/available', + }), + }); + + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/unknown/state`, + message: JSON.stringify({ + ...configData, + unknown, + }), + }); + } } matches.forEach((match) => { - client.publish( - `${MQTT.TOPICS.MATCHES}/${match.name}`, - JSON.stringify({ + messages.push({ + topic: `${MQTT.TOPICS.MATCHES}/${match.name}`, + message: JSON.stringify({ ...configData, match, }), - { retain: true } - ); + }); + + if (MQTT.TOPICS.HOMEASSISTANT) { + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${match.name}/config`, + message: JSON.stringify({ + name: match.name, + icon: 'mdi:account', + value_template: '{{ value_json.camera }}', + state_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${match.name}/state`, + json_attributes_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${match.name}/state`, + availability_topic: 'double-take/available', + }), + }); + + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${match.name}/state`, + message: JSON.stringify({ + ...configData, + match, + }), + }); + } }); - if (matches.length || (unknown && Object.keys(unknown).length)) { - client.publish( - `${MQTT.TOPICS.CAMERAS}/${camera}`, - JSON.stringify({ + if (matches.length || hasUnkown) { + messages.push({ + topic: `${MQTT.TOPICS.CAMERAS}/${camera}`, + message: JSON.stringify({ ...configData, matches, unknown, }), - { retain: true } - ); - } - - let personCount = matches.length; - // check to see if unknown bounding box is contained within or contains any of the match bounding boxes - // if false, then add 1 to the person count - if (matches.length && unknown && Object.keys(unknown).length) { - let unknownContained = false; - matches.forEach((match) => { - if (contains(match.box, unknown.box) || contains(unknown.box, match.box)) - unknownContained = true; }); - if (!unknownContained) personCount += 1; + + if (MQTT.TOPICS.HOMEASSISTANT) { + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${camera}/config`, + message: JSON.stringify({ + name: camera, + icon: 'mdi:camera', + value_template: '{{ value_json.personCount }}', + state_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${camera}/state`, + json_attributes_topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${camera}/state`, + availability_topic: 'double-take/available', + }), + }); + + messages.push({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${camera}/state`, + message: JSON.stringify({ + ...configData, + matches, + unknown, + personCount, + }), + }); + } } - client.publish(`${MQTT.TOPICS.CAMERAS}/${camera}/person`, personCount.toString(), { - retain: true, - }); + this.publish(messages); - clearTimeout(personResetTimeout[camera]); - personResetTimeout[camera] = setTimeout(() => { - client.publish(`${MQTT.TOPICS.CAMERAS}/${camera}/person`, '0', { - retain: true, - }); + clearTimeout(PERSON_RESET_TIMEOUT[camera]); + PERSON_RESET_TIMEOUT[camera] = setTimeout(() => { + this.publish({ topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`, message: '0' }); }, 30000); } catch (error) { - logger.log(`MQTT: publish error: ${error.message}`); + console.error(`MQTT: recognize error: ${error.message}`); } }; -module.exports.subscribe = () => { - const topics = []; - - topics.push(...cameraTopics()); +module.exports.publish = (data) => { + const multiple = Array.isArray(data); + const single = data && !multiple && typeof data === 'object'; - if (FRIGATE.URL && MQTT.TOPICS.FRIGATE) { - const [prefix] = MQTT.TOPICS.FRIGATE.split('/'); - topics.push(MQTT.TOPICS.FRIGATE); - topics.push( - ...(FRIGATE.CAMERAS - ? FRIGATE.CAMERAS.map((camera) => `${prefix}/${camera}/person/snapshot`) - : [`${prefix}/+/person/snapshot`]) - ); - } + if (!single && !multiple) console.error('MQTT: publish error'); - if (topics.length) { - client.subscribe(topics, (err) => { - if (err) { - logger.log(`MQTT: error subscribing to ${topics.join(', ')}`); - return; - } - logger.log(`MQTT: subscribed to ${topics.join(', ')}`); - justSubscribed = true; - setTimeout(() => (justSubscribed = false), 5000); - }); - } + const messages = single ? [{ ...data }] : data; + messages.forEach((message) => CLIENT.publish(message.topic, message.message, { retain: true })); }; From f2184fa5f12eaa2139e607eda1bd9446d4dc6763 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 02:21:15 -0400 Subject: [PATCH 04/70] refactor: read config file location from constants path --- api/src/controllers/config.controller.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/controllers/config.controller.js b/api/src/controllers/config.controller.js index 13fd854a..9af02825 100644 --- a/api/src/controllers/config.controller.js +++ b/api/src/controllers/config.controller.js @@ -1,5 +1,6 @@ const fs = require('fs'); const config = require('../constants/config'); +const { STORAGE } = require('../constants'); const { respond, HTTPSuccess /* , HTTPError */ } = require('../util/respond.util'); const { OK } = require('../constants/http-status'); @@ -7,7 +8,7 @@ module.exports.get = async (req, res) => { try { const { format } = req.query; const output = - format === 'yaml' ? fs.readFileSync(`${__dirname}/../../../config.yml`, 'utf8') : config(); + format === 'yaml' ? fs.readFileSync(`${STORAGE.CONFIG.PATH}/config.yml`, 'utf8') : config(); res.status(OK).json(output); } catch (error) { respond(error, res); @@ -17,7 +18,7 @@ module.exports.get = async (req, res) => { module.exports.patch = async (req, res) => { try { const { code } = req.body; - fs.writeFileSync(`${__dirname}/../../../config.yml`, code); + fs.writeFileSync(`${STORAGE.CONFIG.PATH}/config.yml`, code); respond(HTTPSuccess(OK, req.body), res); } catch (error) { respond(error, res); From a10024419ee9a36d009f76a6cbf4415235d29d49 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 02:21:31 -0400 Subject: [PATCH 05/70] chore: change default camera name --- api/src/util/validators.util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/util/validators.util.js b/api/src/util/validators.util.js index 59268afe..ba474d2d 100644 --- a/api/src/util/validators.util.js +++ b/api/src/util/validators.util.js @@ -11,7 +11,7 @@ module.exports.recognize = ({ get }) => { ]; if (get) { validations = validations.concat([ - query('camera').default('double-take'), + query('camera').default('manual'), query('url').isLength({ min: 1 }), query('attempts').default(1).isInt().withMessage('not a valid number'), ]); From 2e4f58c26a6f1b9d88ecf78cb7ce42456c1d546e Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 12:08:42 -0400 Subject: [PATCH 06/70] refactor(api:config): refactored config with support for override file --- api/src/constants/config.js | 66 ++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/api/src/constants/config.js b/api/src/constants/config.js index be59de10..b7d0713c 100644 --- a/api/src/constants/config.js +++ b/api/src/constants/config.js @@ -1,10 +1,12 @@ const yaml = require('js-yaml'); const fs = require('fs'); const _ = require('lodash'); -const defaults = require('./defaults'); +const DEFAULTS = require('./defaults'); +const { core: SYSTEM_CORE, local: SYSTEM_LOCAL } = require('./system'); const { version } = require('../../package.json'); -let config = false; +let CONFIG = false; +let OVERRIDE = false; const customizer = (objValue, srcValue) => { if (_.isNull(srcValue)) { @@ -12,21 +14,53 @@ const customizer = (objValue, srcValue) => { } }; +const loadYaml = (file) => { + try { + return yaml.load(fs.readFileSync(file, 'utf8')); + } catch (error) { + return error; + } +}; + +const setup = (file, message) => { + if (!fs.existsSync(SYSTEM_CORE.storage.config.path)) + fs.mkdirSync(SYSTEM_CORE.storage.config.path, { recursive: true }); + fs.writeFileSync(`${SYSTEM_CORE.storage.config.path}/${file}`, message); +}; + module.exports = () => { - if (config) return config; + if (CONFIG) return CONFIG; - config = {}; - try { - config = { ...yaml.load(fs.readFileSync(`${__dirname}/../../../config.yml`, 'utf8')) }; - // eslint-disable-next-line no-empty - } catch (error) {} + CONFIG = {}; + OVERRIDE = {}; - if (!config.notify || !config.notify.gotify) { - delete defaults.notify.gotify; - } - config = _.isEmpty(config) ? defaults : _.mergeWith(defaults, config, customizer); - config.storage = { path: './.storage', ...config.storage }; - config.version = version; - config = _(config).toPairs().sortBy(0).fromPairs().value(); - return config; + const isLegacyPath = fs.existsSync('./config.yml'); + if (isLegacyPath) + console.warn( + 'config.yml file loaded from legacy path, this will be removed in a future update' + ); + + const configData = loadYaml( + isLegacyPath ? './config.yml' : `${SYSTEM_CORE.storage.config.path}/config.yml` + ); + if (configData && configData.code === 'ENOENT') setup('config.yml', '# Double Take\n'); + else CONFIG = { ...configData }; + + const overrideData = + process.env.NODE_ENV === 'local' + ? loadYaml(`${SYSTEM_CORE.storage.config.path}/config.override.yml`) + : false; + if (overrideData && (overrideData.code === 'ENOENT' || _.isEmpty(overrideData))) { + OVERRIDE = { ...SYSTEM_LOCAL }; + setup('config.override.yml', `# Double Take\n\n${yaml.dump(SYSTEM_LOCAL)}`); + } else OVERRIDE = { ...overrideData }; + + if (!CONFIG.notify || !CONFIG.notify.gotify) delete DEFAULTS.notify.gotify; + + CONFIG = _.isEmpty(CONFIG) ? DEFAULTS : _.mergeWith(DEFAULTS, CONFIG, customizer); + CONFIG = _.mergeWith(CONFIG, SYSTEM_CORE); + if (process.env.NODE_ENV === 'local') CONFIG = _.mergeWith(CONFIG, OVERRIDE, customizer); + CONFIG.version = version; + CONFIG = _(CONFIG).toPairs().sortBy(0).fromPairs().value(); + return CONFIG; }; From 1f3691eddbafdd5c612e1345e8e045f5544aa11a Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 12:09:07 -0400 Subject: [PATCH 07/70] chore: support for legacy config bind mount --- api/src/controllers/config.controller.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/api/src/controllers/config.controller.js b/api/src/controllers/config.controller.js index 9af02825..600f9518 100644 --- a/api/src/controllers/config.controller.js +++ b/api/src/controllers/config.controller.js @@ -7,8 +7,14 @@ const { OK } = require('../constants/http-status'); module.exports.get = async (req, res) => { try { const { format } = req.query; + const isLegacyPath = fs.existsSync('./config.yml'); const output = - format === 'yaml' ? fs.readFileSync(`${STORAGE.CONFIG.PATH}/config.yml`, 'utf8') : config(); + format === 'yaml' + ? fs.readFileSync( + isLegacyPath ? './config.yml' : `${STORAGE.CONFIG.PATH}/config.yml`, + 'utf8' + ) + : config(); res.status(OK).json(output); } catch (error) { respond(error, res); @@ -17,8 +23,9 @@ module.exports.get = async (req, res) => { module.exports.patch = async (req, res) => { try { + const isLegacyPath = fs.existsSync('./config.yml'); const { code } = req.body; - fs.writeFileSync(`${STORAGE.CONFIG.PATH}/config.yml`, code); + fs.writeFileSync(isLegacyPath ? './config.yml' : `${STORAGE.CONFIG.PATH}/config.yml`, code); respond(HTTPSuccess(OK, req.body), res); } catch (error) { respond(error, res); From 6d4ea3e37942e3bf11a0e36cb84dea53c257e7a2 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 8 Jul 2021 12:12:07 -0400 Subject: [PATCH 08/70] refactor: don't let logger use config --- api/src/util/logger.util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/util/logger.util.js b/api/src/util/logger.util.js index d0a21970..373b3428 100644 --- a/api/src/util/logger.util.js +++ b/api/src/util/logger.util.js @@ -1,6 +1,6 @@ const { createLogger, format, transports } = require('winston'); const util = require('util'); -const { STORAGE } = require('../constants'); +const { core: SYSTEM_CORE } = require('../constants/system'); const combineMessageAndSplat = () => { return { @@ -20,7 +20,7 @@ module.exports.init = () => { format: format.combine(format.colorize(), logFormat), }), new transports.File({ - filename: `${STORAGE.PATH}/messages.log`, + filename: `${SYSTEM_CORE.storage.path}/messages.log`, format: format.combine( logFormat, format.timestamp({ format: 'YY-MM-DD HH:mm:ss' }), From 654b32e95fc45ef97e46333986f776d87d79ed5a Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 00:24:44 -0400 Subject: [PATCH 09/70] chore: require logger first --- api/server.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/server.js b/api/server.js index a81f13a6..ee8801a5 100644 --- a/api/server.js +++ b/api/server.js @@ -1,3 +1,4 @@ +require('./src/util/logger.util').init(); const http = require('http'); const { version } = require('./package.json'); const mqtt = require('./src/util/mqtt.util'); @@ -8,8 +9,6 @@ const config = require('./src/constants/config'); const shutdown = require('./src/util/shutdown.util'); const { SERVER } = require('./src/constants'); -require('./src/util/logger.util').init(); - module.exports.start = async () => { storage.setup(); console.log(`Double Take v${version}`); From de37e3ac058d2627c13f4f980b08e055e8e56d51 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 00:25:34 -0400 Subject: [PATCH 10/70] revert: remove config.override.yml logic for now --- api/src/constants/config.js | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/api/src/constants/config.js b/api/src/constants/config.js index b7d0713c..7acf984d 100644 --- a/api/src/constants/config.js +++ b/api/src/constants/config.js @@ -2,11 +2,10 @@ const yaml = require('js-yaml'); const fs = require('fs'); const _ = require('lodash'); const DEFAULTS = require('./defaults'); -const { core: SYSTEM_CORE, local: SYSTEM_LOCAL } = require('./system'); +const { core: SYSTEM_CORE } = require('./system'); const { version } = require('../../package.json'); let CONFIG = false; -let OVERRIDE = false; const customizer = (objValue, srcValue) => { if (_.isNull(srcValue)) { @@ -32,7 +31,6 @@ module.exports = () => { if (CONFIG) return CONFIG; CONFIG = {}; - OVERRIDE = {}; const isLegacyPath = fs.existsSync('./config.yml'); if (isLegacyPath) @@ -43,23 +41,13 @@ module.exports = () => { const configData = loadYaml( isLegacyPath ? './config.yml' : `${SYSTEM_CORE.storage.config.path}/config.yml` ); - if (configData && configData.code === 'ENOENT') setup('config.yml', '# Double Take\n'); + if (configData && configData.code === 'ENOENT') setup('config.yml', '# Double Take'); else CONFIG = { ...configData }; - const overrideData = - process.env.NODE_ENV === 'local' - ? loadYaml(`${SYSTEM_CORE.storage.config.path}/config.override.yml`) - : false; - if (overrideData && (overrideData.code === 'ENOENT' || _.isEmpty(overrideData))) { - OVERRIDE = { ...SYSTEM_LOCAL }; - setup('config.override.yml', `# Double Take\n\n${yaml.dump(SYSTEM_LOCAL)}`); - } else OVERRIDE = { ...overrideData }; - if (!CONFIG.notify || !CONFIG.notify.gotify) delete DEFAULTS.notify.gotify; CONFIG = _.isEmpty(CONFIG) ? DEFAULTS : _.mergeWith(DEFAULTS, CONFIG, customizer); CONFIG = _.mergeWith(CONFIG, SYSTEM_CORE); - if (process.env.NODE_ENV === 'local') CONFIG = _.mergeWith(CONFIG, OVERRIDE, customizer); CONFIG.version = version; CONFIG = _(CONFIG).toPairs().sortBy(0).fromPairs().value(); return CONFIG; From a4353ac0117a17e71fb2cd0552a1c1696b2abf2c Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 00:27:23 -0400 Subject: [PATCH 11/70] refactor: store system config constants in new file --- api/src/constants/system.js | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 api/src/constants/system.js diff --git a/api/src/constants/system.js b/api/src/constants/system.js new file mode 100644 index 00000000..721a2693 --- /dev/null +++ b/api/src/constants/system.js @@ -0,0 +1,7 @@ +module.exports.core = { + storage: { path: './.storage', config: { path: './.storage/config' } }, +}; + +module.exports.dev = { + mqtt: { host: 'double-take-mqtt' }, +}; From 11353c814477fa09250dc4b3caf9739d1e523e3d Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:05:54 -0400 Subject: [PATCH 12/70] build: update entry command to write config.yml file before nodemon starts --- Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4d9eedd..e08259be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,21 +4,21 @@ RUN apk add libimagequant-dev --repository=http://dl-cdn.alpinelinux.org/alpine/ RUN apk add vips-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community WORKDIR /double-take/api -COPY ./api/package.json . +COPY /api/package.json . RUN npm install --production WORKDIR /double-take/frontend -COPY ./frontend/package.json . +COPY /frontend/package.json . RUN npm install WORKDIR /double-take/api -COPY ./api/server.js . -COPY ./api/src ./src +COPY /api/server.js . +COPY /api/src ./src WORKDIR /double-take/frontend -COPY ./frontend/src ./src -COPY ./frontend/public ./public -COPY ./frontend/.env.production ./frontend/vue.config.js ./ +COPY /frontend/src ./src +COPY /frontend/public ./public +COPY /frontend/.env.production ./frontend/vue.config.js ./ RUN npm run build RUN mv dist /tmp/dist && rm -r * && mv /tmp/dist/* . @@ -28,8 +28,9 @@ RUN mkdir /.storage RUN ln -s /.storage /double-take/.storage WORKDIR /double-take -RUN echo "# Double Take\n\n\n" >> config.yml RUN npm install nodemon -g -CMD nodemon --watch config.yml api/server.js \ No newline at end of file +CMD mkdir -p ./.storage/config && \ + echo $'# Double Take' > ./.storage/config/config.yml && \ + nodemon -e yml --watch ./.storage/config -q api/server.js \ No newline at end of file From a5ef5915ca2487b7ec51090f1b7b9f37b3155376 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:08:43 -0400 Subject: [PATCH 13/70] build: package update --- frontend/package-lock.json | 61 +++++++++++++++++++++++++------------- frontend/package.json | 6 ++-- package-lock.json | 48 ++++++++++++++++++++---------- package.json | 4 +-- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8ae1ccbc..a83d22c6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1223,6 +1223,23 @@ "@hapi/hoek": "^8.3.0" } }, + "@humanwhocodes/config-array": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", + "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "dev": true, + "requires": { + "@humanwhocodes/object-schema": "^1.2.0", + "debug": "^4.1.1", + "minimatch": "^3.0.4" + } + }, + "@humanwhocodes/object-schema": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.0.tgz", + "integrity": "sha512-wdppn25U8z/2yiaT6YGquE6X8sSv7hNMWSXYSSU1jGv/yd6XqjXgTDJ8KP4NgjTXfJ3GbRjeeb8RTV7a/VpM+w==", + "dev": true + }, "@intervolga/optimize-cssnano-plugin": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@intervolga/optimize-cssnano-plugin/-/optimize-cssnano-plugin-1.0.6.tgz", @@ -5096,13 +5113,14 @@ "dev": true }, "eslint": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.29.0.tgz", - "integrity": "sha512-82G/JToB9qIy/ArBzIWG9xvvwL3R86AlCjtGw+A29OMZDqhTybz/MByORSukGxeI+YPCR4coYyITKk8BFH9nDA==", + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz", + "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==", "dev": true, "requires": { "@babel/code-frame": "7.12.11", "@eslint/eslintrc": "^0.4.2", + "@humanwhocodes/config-array": "^0.5.0", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -5642,15 +5660,15 @@ } }, "eslint-plugin-vue": { - "version": "7.12.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-7.12.1.tgz", - "integrity": "sha512-xHf/wCt88qmzqQerjaSteUFGASj7fPreglKD4ijnvoKRkoSJ3/H3kuJE8QFFtc+2wjw6hRDs834HH7vpuTJQzg==", + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-7.13.0.tgz", + "integrity": "sha512-u0+jL8h2MshRuMTCLslktxRsPTjlENNcNufhgHu01N982DmHVdeFniyMPoVLLRjACQOwdz3FdlsgYGBMBG+AKg==", "dev": true, "requires": { "eslint-utils": "^2.1.0", "natural-compare": "^1.4.0", "semver": "^7.3.2", - "vue-eslint-parser": "^7.6.0" + "vue-eslint-parser": "^7.8.0" }, "dependencies": { "lru-cache": { @@ -6273,9 +6291,9 @@ } }, "flatted": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", - "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.0.tgz", + "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", "dev": true }, "flush-write-stream": { @@ -10337,9 +10355,9 @@ "integrity": "sha512-QLXhsXse21Gz22pFJWVScy443FhnIAOK1OF07VlkaqypylftcpeVF6g9pXCSpQfkXpUc0LsH6kqwB9IlO3Lcgw==" }, "prismjs": { - "version": "1.24.0", - "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.0.tgz", - "integrity": "sha512-SqV5GRsNqnzCL8k5dfAjCNhUrF3pR0A9lTDSCUZeh/LIshheXJEaP0hwLz2t4XHivd2J/v2HR+gRnigzeKe3cQ==" + "version": "1.24.1", + "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.24.1.tgz", + "integrity": "sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==" }, "process": { "version": "0.11.10", @@ -12186,9 +12204,9 @@ }, "dependencies": { "ajv": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.0.tgz", - "integrity": "sha512-cnUG4NSBiM4YFBxgZIj/In3/6KX+rQ2l2YPRVcvAMQGWEPKuXoPIhxzwqh31jA3IPbI4qEOp/5ILI4ynioXsGQ==", + "version": "8.6.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.1.tgz", + "integrity": "sha512-42VLtQUOLefAvKFAQIxIZDaThq6om/PrfP0CYk3/vn+y4BMNkKnbli8ON2QCiHov4KkzOSJ/xSoBJdayiiYvVQ==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -12964,17 +12982,18 @@ "integrity": "sha512-8Dgws6dwt+1LoKEU4HDzIPfaRgWHl0xLDLW6kxrAk/z3GAJ2thkajcXC4qZgmqj0cZRx/Z/dzooKi5HWtUQwxA==" }, "vue-eslint-parser": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-7.6.0.tgz", - "integrity": "sha512-QXxqH8ZevBrtiZMZK0LpwaMfevQi9UL7lY6Kcp+ogWHC88AuwUPwwCIzkOUc1LR4XsYAt/F9yHXAB/QoD17QXA==", + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-7.8.0.tgz", + "integrity": "sha512-ehmmrLZNYLUoKayvVW8l8HyPQIfuYZHiJoQLRP3dapDlTU7bGs4tqIKVGdAEpMuXS/b4R/PImCt7Tkj4UhX1SQ==", "dev": true, "requires": { "debug": "^4.1.1", - "eslint-scope": "^5.0.0", + "eslint-scope": "^5.1.1", "eslint-visitor-keys": "^1.1.0", "espree": "^6.2.1", "esquery": "^1.4.0", - "lodash": "^4.17.15" + "lodash": "^4.17.21", + "semver": "^6.3.0" }, "dependencies": { "eslint-scope": { diff --git a/frontend/package.json b/frontend/package.json index e4ae8c49..57f93427 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,7 +25,7 @@ "primeflex": "^2.0.0", "primeicons": "^4.1.0", "primevue": "^3.5.1", - "prismjs": "^1.24.0", + "prismjs": "^1.24.1", "vue": "^3.1.4", "vue-axios": "^3.2.4", "vue-router": "^4.0.10", @@ -39,8 +39,8 @@ "@vue/eslint-config-airbnb": "^5.3.0", "@vue/eslint-config-prettier": "^6.0.0", "babel-eslint": "^10.1.0", - "eslint": "^7.29.0", - "eslint-plugin-vue": "^7.12.1", + "eslint": "^7.30.0", + "eslint-plugin-vue": "^7.13.0", "node-sass": "^6.0.1", "sass-loader": "^10.2.0" }, diff --git a/package-lock.json b/package-lock.json index 5176dbdb..53401775 100644 --- a/package-lock.json +++ b/package-lock.json @@ -304,6 +304,23 @@ "strip-json-comments": "^3.1.1" } }, + "@humanwhocodes/config-array": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", + "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "dev": true, + "requires": { + "@humanwhocodes/object-schema": "^1.2.0", + "debug": "^4.1.1", + "minimatch": "^3.0.4" + } + }, + "@humanwhocodes/object-schema": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-1.2.0.tgz", + "integrity": "sha512-wdppn25U8z/2yiaT6YGquE6X8sSv7hNMWSXYSSU1jGv/yd6XqjXgTDJ8KP4NgjTXfJ3GbRjeeb8RTV7a/VpM+w==", + "dev": true + }, "@sindresorhus/is": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", @@ -832,9 +849,9 @@ "dev": true }, "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", "dev": true, "requires": { "ms": "2.1.2" @@ -1011,13 +1028,14 @@ "dev": true }, "eslint": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.29.0.tgz", - "integrity": "sha512-82G/JToB9qIy/ArBzIWG9xvvwL3R86AlCjtGw+A29OMZDqhTybz/MByORSukGxeI+YPCR4coYyITKk8BFH9nDA==", + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz", + "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==", "dev": true, "requires": { "@babel/code-frame": "7.12.11", "@eslint/eslintrc": "^0.4.2", + "@humanwhocodes/config-array": "^0.5.0", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -1467,9 +1485,9 @@ } }, "flatted": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.1.1.tgz", - "integrity": "sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.0.tgz", + "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", "dev": true }, "fs-extra": { @@ -2230,9 +2248,9 @@ "dev": true }, "nodemon": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.9.tgz", - "integrity": "sha512-6O4k7C8f2HQArGpaPBOqGGddjzDLQAqCYmq3tKMeNIbz7Is/hOphMHy2dcY10sSq5wl3cqyn9Iz+Ep2j51JOLg==", + "version": "2.0.10", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.10.tgz", + "integrity": "sha512-369KB2EC1fLzz7hIuKSRSIVhh9PXqFAwh1stxlNX8DMyat9y/maswuRxRMttyelnduLDa04r4wgVZ4fgRwZWuQ==", "dev": true, "requires": { "chokidar": "^3.2.2", @@ -3087,9 +3105,9 @@ }, "dependencies": { "ajv": { - "version": "8.6.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.0.tgz", - "integrity": "sha512-cnUG4NSBiM4YFBxgZIj/In3/6KX+rQ2l2YPRVcvAMQGWEPKuXoPIhxzwqh31jA3IPbI4qEOp/5ILI4ynioXsGQ==", + "version": "8.6.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.1.tgz", + "integrity": "sha512-42VLtQUOLefAvKFAQIxIZDaThq6om/PrfP0CYk3/vn+y4BMNkKnbli8ON2QCiHov4KkzOSJ/xSoBJdayiiYvVQ==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", diff --git a/package.json b/package.json index 061a2f5e..53cea980 100644 --- a/package.json +++ b/package.json @@ -24,13 +24,13 @@ "devDependencies": { "@commitlint/cli": "^12.1.4", "@commitlint/config-conventional": "^12.1.4", - "eslint": "^7.29.0", + "eslint": "^7.30.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-config-prettier": "^8.1.0", "eslint-plugin-import": "^2.23.4", "eslint-plugin-prettier": "^3.3.1", "husky": "^6.0.0", - "nodemon": "^2.0.9", + "nodemon": "^2.0.10", "prettier": "^2.3.2" } } From c3f92e5c81b90c685ac443feaa1e23873c8b749b Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:13:20 -0400 Subject: [PATCH 14/70] build: local development with docker-compose --- .develop/Dockerfile.dev | 33 +++++++++++++++++++++++++++++++++ .develop/mosquitto.conf | 2 ++ docker-compose.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 .develop/Dockerfile.dev create mode 100644 .develop/mosquitto.conf create mode 100644 docker-compose.yml diff --git a/.develop/Dockerfile.dev b/.develop/Dockerfile.dev new file mode 100644 index 00000000..1afc4a71 --- /dev/null +++ b/.develop/Dockerfile.dev @@ -0,0 +1,33 @@ +FROM node:14-alpine +RUN apk --update add --no-cache bash python make g++ libpng libpng-dev jpeg-dev pango-dev cairo-dev giflib-dev libheif-dev +RUN apk add libimagequant-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/main +RUN apk add vips-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community + +WORKDIR /double-take/api +COPY /api/package.json . +RUN npm install --production + +WORKDIR /double-take/frontend +COPY /frontend/package.json . +RUN npm install + +WORKDIR /double-take/api +COPY /api/server.js . +COPY /api/src ./src + +WORKDIR /double-take/frontend +COPY /frontend/src ./src +COPY /frontend/public ./public +COPY /frontend/.env.production ./frontend/vue.config.js ./ + +WORKDIR / +RUN mkdir /.storage +RUN ln -s /.storage /double-take/.storage + +WORKDIR /double-take + +RUN npm install nodemon -g + +CMD mkdir -p ./.storage/config && \ + echo $'# Double Take' > ./.storage/config/config.yml && \ + npm run api \ No newline at end of file diff --git a/.develop/mosquitto.conf b/.develop/mosquitto.conf new file mode 100644 index 00000000..daa41378 --- /dev/null +++ b/.develop/mosquitto.conf @@ -0,0 +1,2 @@ +listener 1883 +allow_anonymous true \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4502079e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +version: '3.7' + +services: + frontend: + container_name: double-take-frontend + command: npm run frontend + depends_on: + - api + build: + context: . + dockerfile: ./.develop/Dockerfile.dev + environment: + VUE_APP_API_URL: http://localhost:3000/api + ports: + - 8080:8080 + volumes: + - .:/double-take + - /double-take/frontend/node_modules + + api: + container_name: double-take-api + depends_on: + - mqtt + build: + context: . + dockerfile: ./.develop/Dockerfile.dev + ports: + - 3000:3000 + volumes: + - .:/double-take + - /double-take/api/node_modules + + mqtt: + container_name: double-take-mqtt + restart: unless-stopped + image: eclipse-mosquitto + volumes: + - ${PWD}/.develop/mosquitto.conf:/mosquitto/config/mosquitto.conf + ports: + - 1883:1883 From 9fe9e0c61d24080eefdd932c3d486bc2e447438b Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:13:45 -0400 Subject: [PATCH 15/70] chore: remove docker-compose.yml --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index c312b082..6989c930 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .DS_Store node_modules -docker-compose.yml # frontend frontend/dist From 4ef44370a9888a8ef6b42b9484b2c2ef673db81c Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:14:13 -0400 Subject: [PATCH 16/70] chore: update api script for local container --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 53cea980..7a8f2379 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "update": "npm update && cd ./api && npm update && cd ../frontend && npm update", "audit": "npm audit fix && cd ./api && npm audit fix && cd ../frontend && npm audit fix", "frontend": "cd ./frontend && npm run serve", - "api": "cd ./api && nodemon --watch ./ --watch ../config.yml -q server.js || true", + "api": "nodemon -e yml,js --watch ./.storage/config --watch ./api -q api/server.js || true", "bump": "npm version $VERSION --no-git-tag-version --allow-same-version && cd ./api && npm version $VERSION --no-git-tag-version --allow-same-version && cd ../frontend && npm version $VERSION --no-git-tag-version --allow-same-version" }, "repository": { From 61ab916e295ace6d7655c7a88536e7492206e558 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:18:22 -0400 Subject: [PATCH 17/70] chore: version bump --- api/package-lock.json | 2 +- api/package.json | 2 +- frontend/package-lock.json | 2 +- frontend/package.json | 2 +- package-lock.json | 2 +- package.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index fb626e4d..ea3440be 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1,6 +1,6 @@ { "name": "double-take-api", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/api/package.json b/api/package.json index b72ebabd..aea4c3ec 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "double-take-api", - "version": "0.8.0", + "version": "0.9.0", "description": "Unified UI and API for processing and training images for facial recognition", "scripts": { "start": "node server.js", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a83d22c6..d6820af1 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,6 +1,6 @@ { "name": "double-take-frontend", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 57f93427..a07c638a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "double-take-frontend", - "version": "0.8.0", + "version": "0.9.0", "description": "Unified UI and API for processing and training images for facial recognition", "scripts": { "serve": "vue-cli-service serve", diff --git a/package-lock.json b/package-lock.json index 53401775..7820e544 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "double-take", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7a8f2379..e2883563 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "double-take", - "version": "0.8.0", + "version": "0.9.0", "description": "Unified UI and API for processing and training images for facial recognition", "scripts": { "prepare": "husky install", From 175437177b068c61ebfe314748eb80cdb4d5dd49 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:33:48 -0400 Subject: [PATCH 18/70] fix: watch legacy config location for restart --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e08259be..6726d306 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,4 +33,4 @@ RUN npm install nodemon -g CMD mkdir -p ./.storage/config && \ echo $'# Double Take' > ./.storage/config/config.yml && \ - nodemon -e yml --watch ./.storage/config -q api/server.js \ No newline at end of file + nodemon -e yml --watch ./.storage/config --watch ./config.yml -q api/server.js \ No newline at end of file From e1285e64ae1322f4faf03a8adf28bc8c970ff818 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 01:45:16 -0400 Subject: [PATCH 19/70] fix: reset home assistant camera topic person count after 30sec --- api/src/util/mqtt.util.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/api/src/util/mqtt.util.js b/api/src/util/mqtt.util.js index dfa46d86..2c370954 100644 --- a/api/src/util/mqtt.util.js +++ b/api/src/util/mqtt.util.js @@ -263,6 +263,17 @@ module.exports.recognize = (data) => { clearTimeout(PERSON_RESET_TIMEOUT[camera]); PERSON_RESET_TIMEOUT[camera] = setTimeout(() => { this.publish({ topic: `${MQTT.TOPICS.CAMERAS}/${camera}/person`, message: '0' }); + if (MQTT.TOPICS.HOMEASSISTANT) { + this.publish({ + topic: `${MQTT.TOPICS.HOMEASSISTANT}/sensor/${camera}/state`, + message: JSON.stringify({ + ...configData, + matches, + unknown, + personCount: 0, + }), + }); + } }, 30000); } catch (error) { console.error(`MQTT: recognize error: ${error.message}`); From ccbc6ecfcba2b758e825ea6caa952ebfde1655ef Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 02:03:16 -0400 Subject: [PATCH 20/70] fix: check to see if config.yml exists before creating it --- .develop/Dockerfile.dev | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.develop/Dockerfile.dev b/.develop/Dockerfile.dev index 1afc4a71..59be5fe6 100644 --- a/.develop/Dockerfile.dev +++ b/.develop/Dockerfile.dev @@ -29,5 +29,5 @@ WORKDIR /double-take RUN npm install nodemon -g CMD mkdir -p ./.storage/config && \ - echo $'# Double Take' > ./.storage/config/config.yml && \ + [ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ npm run api \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6726d306..0027ac98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,5 +32,5 @@ WORKDIR /double-take RUN npm install nodemon -g CMD mkdir -p ./.storage/config && \ - echo $'# Double Take' > ./.storage/config/config.yml && \ + [ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ nodemon -e yml --watch ./.storage/config --watch ./config.yml -q api/server.js \ No newline at end of file From 18f6288a35adc6ba1f7fd6640c9aefedb3ce1ad9 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 02:21:54 -0400 Subject: [PATCH 21/70] refactor: use entrypoint.sh instead of command --- Dockerfile | 6 +++--- entrypoint.sh | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 0027ac98..31006f73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,6 @@ WORKDIR /double-take RUN npm install nodemon -g -CMD mkdir -p ./.storage/config && \ - [ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ - nodemon -e yml --watch ./.storage/config --watch ./config.yml -q api/server.js \ No newline at end of file +COPY /entrypoint.sh . + +ENTRYPOINT ["/bin/bash", "./entrypoint.sh"] \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 00000000..6af51715 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,3 @@ +mkdir -p ./.storage/config +[ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ +nodemon -e yml --watch ./.storage/config --watch ./config.yml -q api/server.js \ No newline at end of file From 2455d3edc38de58ac030deec9221db659a349dbf Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 9 Jul 2021 02:22:15 -0400 Subject: [PATCH 22/70] chore: allow entrypoint.sh --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index e947097b..46002a6f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * +!entrypoint.sh !frontend/* !api/* From e7a66c898e7dae999ec950a74f111b86a1c1a2f2 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Mon, 12 Jul 2021 18:19:22 -0400 Subject: [PATCH 23/70] fix(frigate): check for camera name in zones during name check --- api/src/util/frigate.util.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api/src/util/frigate.util.js b/api/src/util/frigate.util.js index 78c81efa..eead47b7 100644 --- a/api/src/util/frigate.util.js +++ b/api/src/util/frigate.util.js @@ -19,13 +19,17 @@ module.exports.checks = async ({ return `Frigate URL not configured`; } - if (FRIGATE.CAMERAS && !FRIGATE.CAMERAS.includes(camera)) { + const cameraMatch = FRIGATE.ZONES + ? FRIGATE.ZONES.filter(({ CAMERA }) => camera === CAMERA).length + ? FRIGATE.ZONES.filter(({ CAMERA }) => camera === CAMERA)[0] + : false + : false; + + if (FRIGATE.CAMERAS && !FRIGATE.CAMERAS.includes(camera) && !cameraMatch) { return `${id} - ${camera} not on approved list`; } if (FRIGATE.ZONES) { - const [cameraMatch] = FRIGATE.ZONES.filter(({ CAMERA }) => camera === CAMERA); - if (cameraMatch) { const [match] = FRIGATE.ZONES.filter( ({ CAMERA, ZONE }) => camera === CAMERA && zones.includes(ZONE) From 384d74c8674868ac0360a4dd1ce37d406199de9e Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 14 Jul 2021 02:29:45 -0400 Subject: [PATCH 24/70] chore: entrypoint.dev.sh for local container development --- .develop/Dockerfile.dev | 6 +----- .develop/entrypoint.dev.sh | 5 +++++ docker-compose.yml | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100755 .develop/entrypoint.dev.sh diff --git a/.develop/Dockerfile.dev b/.develop/Dockerfile.dev index 59be5fe6..e27b654b 100644 --- a/.develop/Dockerfile.dev +++ b/.develop/Dockerfile.dev @@ -26,8 +26,4 @@ RUN ln -s /.storage /double-take/.storage WORKDIR /double-take -RUN npm install nodemon -g - -CMD mkdir -p ./.storage/config && \ - [ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ - npm run api \ No newline at end of file +RUN npm install nodemon -g \ No newline at end of file diff --git a/.develop/entrypoint.dev.sh b/.develop/entrypoint.dev.sh new file mode 100755 index 00000000..440f3680 --- /dev/null +++ b/.develop/entrypoint.dev.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +mkdir -p ./.storage/config +[ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml +npm run api \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4502079e..383e80bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: api: container_name: double-take-api + entrypoint: ./.develop/entrypoint.dev.sh depends_on: - mqtt build: From f26ad0c59bc86ac847c4d7e29891a289b0dc27d1 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 14 Jul 2021 02:30:49 -0400 Subject: [PATCH 25/70] style: single command to multiple for readability --- entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 6af51715..daf03290 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,3 +1,5 @@ +#!/bin/bash + mkdir -p ./.storage/config -[ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml && \ +[ -f ./.storage/config/config.yml ] || echo $'# Double Take' > ./.storage/config/config.yml nodemon -e yml --watch ./.storage/config --watch ./config.yml -q api/server.js \ No newline at end of file From 65894af1fc15fded9afa60a61bea859cedc36bc2 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 14 Jul 2021 02:32:19 -0400 Subject: [PATCH 26/70] chore: require constants first --- api/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server.js b/api/server.js index ee8801a5..62308e77 100644 --- a/api/server.js +++ b/api/server.js @@ -1,5 +1,6 @@ require('./src/util/logger.util').init(); const http = require('http'); +const { SERVER } = require('./src/constants'); const { version } = require('./package.json'); const mqtt = require('./src/util/mqtt.util'); const { app } = require('./src/app'); @@ -7,7 +8,6 @@ const storage = require('./src/util/storage.util'); const database = require('./src/util/db.util'); const config = require('./src/constants/config'); const shutdown = require('./src/util/shutdown.util'); -const { SERVER } = require('./src/constants'); module.exports.start = async () => { storage.setup(); From 13c7625aae7e6d0eb07e71eb2f4c6d9ccbf36157 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 14 Jul 2021 02:35:48 -0400 Subject: [PATCH 27/70] refactor(storage): load training images from controller --- api/src/app.js | 2 -- api/src/controllers/storage.controller.js | 17 +++++++++++++++++ api/src/routes/index.js | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/src/app.js b/api/src/app.js index 79e91302..9b593230 100644 --- a/api/src/app.js +++ b/api/src/app.js @@ -1,14 +1,12 @@ const express = require('express'); const cors = require('cors'); const router = require('./routes'); -const { STORAGE } = require('./constants'); const app = express(); app.use('*', cors()); app.use(express.json({ limit: '10mb' })); app.use(express.urlencoded({ extended: false })); app.use('/api', router); -app.use('/api/storage/train', express.static(`${STORAGE.PATH}/train`)); app.use('/api/tmp', express.static(`/tmp`)); app.use('/', express.static(`./frontend`)); diff --git a/api/src/controllers/storage.controller.js b/api/src/controllers/storage.controller.js index 83d2e718..0705f149 100644 --- a/api/src/controllers/storage.controller.js +++ b/api/src/controllers/storage.controller.js @@ -88,3 +88,20 @@ module.exports.matches = async (req, res) => { respond(error, res); } }; +module.exports.train = async (req, res) => { + try { + const { name, filename } = req.params; + const source = `${PATH}/train/${name}/${filename}`; + + if (!fs.existsSync(source)) { + throw HTTPError(BAD_REQUEST, `${source} does not exist`); + } + + const buffer = fs.readFileSync(source); + res.set('Content-Type', 'image/jpeg'); + return res.end(buffer); + } catch (error) { + respond(error, res); + } +}; + diff --git a/api/src/routes/index.js b/api/src/routes/index.js index 3cc5803f..6731a0cd 100644 --- a/api/src/routes/index.js +++ b/api/src/routes/index.js @@ -30,5 +30,6 @@ router.get('/train/add/:name', train.add); router.get('/train/remove/:name', train.delete); router.get('/storage/matches/:filename', validate(validators.storage().matches()), storage.matches); +router.get('/storage/train/:name/:filename', storage.train); module.exports = router; From 8b17248f0de31ac4acce608875c2878d1d921b74 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 14 Jul 2021 23:19:02 -0400 Subject: [PATCH 28/70] chore: update tile on ui routes --- frontend/src/router/index.js | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 763c5535..5e7d0d80 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -1,29 +1,48 @@ import { /* createWebHistory, */ createWebHashHistory, createRouter } from 'vue-router'; import Match from '@/views/Match.vue'; import Config from '@/views/Config.vue'; -import Files from '@/views/Files.vue'; +import Train from '@/views/Train.vue'; const routes = [ { path: '/', - name: 'Match', + meta: { + title: 'Matches', + }, component: Match, }, { path: '/config', - name: 'Config', + meta: { + title: 'Config', + }, component: Config, }, { - path: '/files', - name: 'Files', - component: Files, + path: '/train', + meta: { + title: 'Train', + }, + component: Train, + }, + { + path: '/:catchAll(.*)', + redirect: '/', }, ]; const router = createRouter({ history: createWebHashHistory(), routes, + scrollBehavior() { + // Scroll to the top of the page on route navigation + return { x: 0, y: 0 }; + }, +}); + +router.beforeEach((to, from, next) => { + document.title = to.meta.title ? `${to.meta.title} | Double Take` : 'Double Take'; + next(); }); export default router; From d40cb8ee070d1e2d633df1f3e83cdeea055cdcc5 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 15 Jul 2021 00:30:41 -0400 Subject: [PATCH 29/70] fix: toggle all when a match was disabled --- frontend/src/views/Match.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index 7e64a202..8b47c2f9 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -60,7 +60,9 @@ export default { return JSON.parse(JSON.stringify(this.matches.source)).filter((obj) => obj); }, areAllSelected() { - return this.filtered.length > 0 && this.matches.selected.length === this.filtered.length; + return ( + this.filtered.length > 0 && this.matches.selected.length + this.matches.disabled.length === this.filtered.length + ); }, filtered() { const files = JSON.parse(JSON.stringify(this.matches.source)).filter((obj) => obj); From dd48b50cef94b0a47e9312b1f1ecca8dc2b9b3ea Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 15 Jul 2021 01:11:11 -0400 Subject: [PATCH 30/70] feat: allow uploading, retraining, and untraining from UI --- frontend/src/views/Train.vue | 280 +++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 frontend/src/views/Train.vue diff --git a/frontend/src/views/Train.vue b/frontend/src/views/Train.vue new file mode 100644 index 00000000..1ff2d137 --- /dev/null +++ b/frontend/src/views/Train.vue @@ -0,0 +1,280 @@ + + + + + From cda8238790dcc4a7e36e7b568d5fa8a734f2baec Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 15 Jul 2021 01:12:15 -0400 Subject: [PATCH 31/70] chore: move components out of match folder to parent folder --- frontend/src/components/match/Asset.vue | 291 ----------------- frontend/src/components/match/AssetFile.vue | 146 --------- frontend/src/components/match/Grid.vue | 43 --- frontend/src/components/match/Header.vue | 336 -------------------- frontend/src/views/Files.vue | 51 --- 5 files changed, 867 deletions(-) delete mode 100644 frontend/src/components/match/Asset.vue delete mode 100644 frontend/src/components/match/AssetFile.vue delete mode 100644 frontend/src/components/match/Grid.vue delete mode 100644 frontend/src/components/match/Header.vue delete mode 100644 frontend/src/views/Files.vue diff --git a/frontend/src/components/match/Asset.vue b/frontend/src/components/match/Asset.vue deleted file mode 100644 index 4bfc08bc..00000000 --- a/frontend/src/components/match/Asset.vue +++ /dev/null @@ -1,291 +0,0 @@ - - - - - diff --git a/frontend/src/components/match/AssetFile.vue b/frontend/src/components/match/AssetFile.vue deleted file mode 100644 index f13f72e5..00000000 --- a/frontend/src/components/match/AssetFile.vue +++ /dev/null @@ -1,146 +0,0 @@ - - - - - diff --git a/frontend/src/components/match/Grid.vue b/frontend/src/components/match/Grid.vue deleted file mode 100644 index 5ea06fab..00000000 --- a/frontend/src/components/match/Grid.vue +++ /dev/null @@ -1,43 +0,0 @@ - - - - - diff --git a/frontend/src/components/match/Header.vue b/frontend/src/components/match/Header.vue deleted file mode 100644 index 4f578d3f..00000000 --- a/frontend/src/components/match/Header.vue +++ /dev/null @@ -1,336 +0,0 @@ - - - - - diff --git a/frontend/src/views/Files.vue b/frontend/src/views/Files.vue deleted file mode 100644 index 61395756..00000000 --- a/frontend/src/views/Files.vue +++ /dev/null @@ -1,51 +0,0 @@ - - - - - From d0e334f617f6d514f1d617215bfa3e43dc8cb2e5 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Thu, 15 Jul 2021 01:13:05 -0400 Subject: [PATCH 32/70] refactor: move folder get/post calls to header component --- frontend/src/views/Match.vue | 41 +++--------------------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index 8b47c2f9..4324a3c5 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -1,6 +1,7 @@ -
untrained
+
+
untrained
+ +
diff --git a/frontend/src/components/Grid.vue b/frontend/src/components/Grid.vue index 10007fb7..b375b2d5 100644 --- a/frontend/src/components/Grid.vue +++ b/frontend/src/components/Grid.vue @@ -6,6 +6,7 @@ diff --git a/frontend/src/views/Train.vue b/frontend/src/views/Train.vue index 1ff2d137..85bd538e 100644 --- a/frontend/src/views/Train.vue +++ b/frontend/src/views/Train.vue @@ -7,6 +7,7 @@ :matches="matches" :areAllSelected="areAllSelected" @trainingFolder="trainingFolder = $event" + @folders="folders = $event" />
@@ -20,10 +21,12 @@
@@ -47,6 +50,7 @@ export default { files: false, status: false, }, + folders: [], status: [], matches: { source: [], From e80368698954cf2a8276f35e863d5bd9cc5d3f2d Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 16 Jul 2021 02:49:30 -0400 Subject: [PATCH 45/70] refactor: when adding new training images, only process those and not untrained ones --- api/src/controllers/train.controller.js | 9 +++++---- api/src/util/train.util.js | 5 ++++- frontend/src/views/Match.vue | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/api/src/controllers/train.controller.js b/api/src/controllers/train.controller.js index 04f38c19..6b50a0f5 100644 --- a/api/src/controllers/train.controller.js +++ b/api/src/controllers/train.controller.js @@ -83,7 +83,7 @@ module.exports.add = async (req, res) => { try { const { name } = req.params; respond(HTTPSuccess(OK, { message: `training queued for ${name}` }), res); - await train.add(name); + await train.add(name, { files }); } catch (error) { console.error(`train add error: ${error.message}`); respond(error, res); @@ -119,18 +119,19 @@ module.exports.patch = async (req, res) => { module.exports.upload = async (req, res) => { try { const { name } = req.params; - const { files } = req; + const files = []; await Promise.all( - files.map(async (obj) => { + req.files.map(async (obj) => { const { originalname, buffer } = obj; const ext = `.${originalname.split('.').pop()}`; const filename = `${originalname.replace(ext, '')}-${time.unix()}${ext}`; await filesystem.writer(`${STORAGE.PATH}/train/${name}/${filename}`, buffer); + files.push(filename); }) ); - train.add(name); + train.add(name, { files }); respond(HTTPSuccess(OK, { success: true }), res); } catch (error) { diff --git a/api/src/util/train.util.js b/api/src/util/train.util.js index 539deb8f..859f8d0b 100644 --- a/api/src/util/train.util.js +++ b/api/src/util/train.util.js @@ -74,7 +74,10 @@ module.exports.process = async ({ name, key, detector }) => { module.exports.add = async (name, opts = {}) => { const files = await filesystem.files().train(); database.insert('init', files); - const images = opts.images || database.files('untrained', name); + let images = opts.images || database.files('untrained', name); + if (opts.files) { + images = images.filter((obj) => opts.files.includes(obj.filename)); + } await this.queue(images); }; diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index 4324a3c5..993bd222 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -226,7 +226,7 @@ export default { folder: this.trainingFolder, matches, }); - await ApiService.get(`/train/add/${this.trainingFolder}`); + await ApiService.get(`/train/add/${this.trainingFolder}`, { files: matches.map((obj) => obj.filename) }); $this.matches.disabled = $this.matches.disabled.concat(ids); $this.matches.selected = []; From 4ee64874bdd69b07c0cefd46ea86bbe98b3e3174 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 16 Jul 2021 02:50:04 -0400 Subject: [PATCH 46/70] chore: header styling and button labels --- frontend/src/components/Header.vue | 41 ++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/Header.vue b/frontend/src/components/Header.vue index acff9195..c07d0900 100644 --- a/frontend/src/components/Header.vue +++ b/frontend/src/components/Header.vue @@ -21,7 +21,7 @@
-
+
@@ -42,18 +42,20 @@ :maxFileSize="10000000" @upload="$parent.init()" :auto="true" - chooseLabel="" + chooseLabel="Upload" class="p-d-inline" />
-
+
@@ -26,6 +32,7 @@ export default { data: () => ({ version: null, updateAvailable: false, + buildTag: null, items: [ { label: 'Matches', icon: 'pi pi-fw fa fa-portrait', to: '/' }, { label: 'Train', icon: 'pi pi-fw fa fa-images', to: '/train' }, @@ -50,11 +57,13 @@ export default { ); const [currentBuild] = actions.workflow_runs.filter((run) => run.head_sha.includes(sha7)); if (currentBuild) { - const [lastBuild] = actions.workflow_runs.filter( - (run) => - run.head_branch === currentBuild.head_branch && - run.status === 'completed' && - run.conclusion === 'success', + this.buildTag = currentBuild.head_branch === 'beta' ? 'beta' : 'latest'; + const [lastBuild] = actions.workflow_runs.filter((run) => + this.buildTag === 'latest' + ? run.event === 'release' && run.status === 'completed' && run.conclusion === 'success' + : run.head_branch === currentBuild.head_branch && + run.status === 'completed' && + run.conclusion === 'success', ); if (currentBuild.id < lastBuild.id) this.updateAvailable = true; } From 4d10e318978c17b84fc86c02fabeecb56d668da7 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 16 Jul 2021 16:00:43 -0400 Subject: [PATCH 52/70] build: package updates --- frontend/package-lock.json | 88 +++++++++++++++++++------------------- frontend/package.json | 4 +- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0d52511d..69542345 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1956,39 +1956,39 @@ } }, "@vue/compiler-core": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.1.4.tgz", - "integrity": "sha512-TnUz+1z0y74O/A4YKAbzsdUfamyHV73MihrEfvettWpm9bQKVoZd1nEmR1cGN9LsXWlwAvVQBetBlWdOjmQO5Q==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.1.5.tgz", + "integrity": "sha512-TXBhFinoBaXKDykJzY26UEuQU1K07FOp/0Ie+OXySqqk0bS0ZO7Xvl7UmiTUPYcLrWbxWBR7Bs/y55AI0MNc2Q==", "requires": { "@babel/parser": "^7.12.0", "@babel/types": "^7.12.0", - "@vue/shared": "3.1.4", + "@vue/shared": "3.1.5", "estree-walker": "^2.0.1", "source-map": "^0.6.1" } }, "@vue/compiler-dom": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.1.4.tgz", - "integrity": "sha512-3tG2ScHkghhUBuFwl9KgyZhrS8CPFZsO7hUDekJgIp5b1OMkROr4AvxHu6rRMl4WkyvYkvidFNBS2VfOnwa6Kw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.1.5.tgz", + "integrity": "sha512-ZsL3jqJ52OjGU/YiT/9XiuZAmWClKInZM2aFJh9gnsAPqOrj2JIELMbkIFpVKR/CrVO/f2VxfPiiQdQTr65jcQ==", "requires": { - "@vue/compiler-core": "3.1.4", - "@vue/shared": "3.1.4" + "@vue/compiler-core": "3.1.5", + "@vue/shared": "3.1.5" } }, "@vue/compiler-sfc": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.1.4.tgz", - "integrity": "sha512-4KDQg60Khy3SgnF+V/TB2NZqzmM4TyGRmzsxqG1SebGdMSecCweFDSlI/F1vDYk6dKiCHgmpoT9A1sLxswkJ0A==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.1.5.tgz", + "integrity": "sha512-mtMY6xMvZeSRx9MTa1+NgJWndrkzVTdJ1pQAmAKQuxyb5LsHVvrgP7kcQFvxPHVpLVTORbTJWHaiqoKrJvi1iA==", "dev": true, "requires": { "@babel/parser": "^7.13.9", "@babel/types": "^7.13.0", "@types/estree": "^0.0.48", - "@vue/compiler-core": "3.1.4", - "@vue/compiler-dom": "3.1.4", - "@vue/compiler-ssr": "3.1.4", - "@vue/shared": "3.1.4", + "@vue/compiler-core": "3.1.5", + "@vue/compiler-dom": "3.1.5", + "@vue/compiler-ssr": "3.1.5", + "@vue/shared": "3.1.5", "consolidate": "^0.16.0", "estree-walker": "^2.0.1", "hash-sum": "^2.0.0", @@ -2024,13 +2024,13 @@ } }, "@vue/compiler-ssr": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.1.4.tgz", - "integrity": "sha512-Box8fCuCFPp0FuimIswjDkjwiSDCBkHvt/xVALyFkYCiIMWv2eR53fIjmlsnEHhcBuZ+VgRC+UanCTcKvSA1gA==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.1.5.tgz", + "integrity": "sha512-CU5N7Di/a4lyJ18LGJxJYZS2a8PlLdWpWHX9p/XcsjT2TngMpj3QvHVRkuik2u8QrIDZ8OpYmTyj1WDNsOV+Dg==", "dev": true, "requires": { - "@vue/compiler-dom": "3.1.4", - "@vue/shared": "3.1.4" + "@vue/compiler-dom": "3.1.5", + "@vue/shared": "3.1.5" } }, "@vue/component-compiler-utils": { @@ -2107,36 +2107,36 @@ "dev": true }, "@vue/reactivity": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.1.4.tgz", - "integrity": "sha512-YDlgii2Cr9yAoKVZFzgY4j0mYlVT73986X3e5SPp6ifqckSEoFSUWXZK2Tb53TB/9qO29BEEbspnKD3m3wAwkA==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.1.5.tgz", + "integrity": "sha512-1tdfLmNjWG6t/CsPldh+foumYFo3cpyCHgBYQ34ylaMsJ+SNHQ1kApMIa8jN+i593zQuaw3AdWH0nJTARzCFhg==", "requires": { - "@vue/shared": "3.1.4" + "@vue/shared": "3.1.5" } }, "@vue/runtime-core": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.1.4.tgz", - "integrity": "sha512-qmVJgJuFxfT7M4qHQ4M6KqhKC66fjuswK+aBivE8dWiZ2rtIGl9gtJGpwqwjQEcKEBTOfvvrtrwBncYArJUO8Q==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.1.5.tgz", + "integrity": "sha512-YQbG5cBktN1RowQDKA22itmvQ+b40f0WgQ6CXK4VYoYICAiAfu6Cc14777ve8zp1rJRGtk5oIeS149TOculrTg==", "requires": { - "@vue/reactivity": "3.1.4", - "@vue/shared": "3.1.4" + "@vue/reactivity": "3.1.5", + "@vue/shared": "3.1.5" } }, "@vue/runtime-dom": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.1.4.tgz", - "integrity": "sha512-vbmwgTxku1BU87Kw7r29adv0OIrDXCW0PslOPQT0O/9R5SqcXgS94Yj6zsztDjvghegenwIAPNLlDR1Auh5s+w==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.1.5.tgz", + "integrity": "sha512-tNcf3JhVR0RfW0kw1p8xZgv30nvX8Y9rsz7eiQ0dHe273sfoCngAG0y4GvMaY4Xd8FsjUwFedd4suQ8Lu8meXg==", "requires": { - "@vue/runtime-core": "3.1.4", - "@vue/shared": "3.1.4", + "@vue/runtime-core": "3.1.5", + "@vue/shared": "3.1.5", "csstype": "^2.6.8" } }, "@vue/shared": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.1.4.tgz", - "integrity": "sha512-6O45kZAmkLvzGLToBxEz4lR2W6kXohCtebV2UxjH9GXjd8X9AhEn68FN9eNanFtWNzvgw1hqd6HkPRVQalqf7Q==" + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.1.5.tgz", + "integrity": "sha512-oJ4F3TnvpXaQwZJNF3ZK+kLPHKarDmJjJ6jyzVNDKH9md1dptjC7lWR//jrGuLdek/U6iltWxqAnYOu8gCiOvA==" }, "@vue/web-component-wrapper": { "version": "1.3.0", @@ -12967,13 +12967,13 @@ "dev": true }, "vue": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.1.4.tgz", - "integrity": "sha512-p8dcdyeCgmaAiZsbLyDkmOLcFGZb/jEVdCLW65V68LRCXTNX8jKsgah2F7OZ/v/Ai2V0Fb1MNO0vz/GFqsPVMA==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.1.5.tgz", + "integrity": "sha512-Ho7HNb1nfDoO+HVb6qYZgeaobt1XbY6KXFe4HGs1b9X6RhkWG/113n4/SrtM1LUclM6OrP/Se5aPHHvAPG1iVQ==", "requires": { - "@vue/compiler-dom": "3.1.4", - "@vue/runtime-dom": "3.1.4", - "@vue/shared": "3.1.4" + "@vue/compiler-dom": "3.1.5", + "@vue/runtime-dom": "3.1.5", + "@vue/shared": "3.1.5" } }, "vue-axios": { diff --git a/frontend/package.json b/frontend/package.json index 44eaffdb..1883e028 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,7 +26,7 @@ "primeicons": "^4.1.0", "primevue": "^3.5.1", "prismjs": "^1.24.1", - "vue": "^3.1.4", + "vue": "^3.1.5", "vue-axios": "^3.2.4", "vue-router": "^4.0.10", "vue3-ace-editor": "^2.0.2" @@ -35,7 +35,7 @@ "@vue/cli-plugin-babel": "^4.5.13", "@vue/cli-plugin-eslint": "^4.5.13", "@vue/cli-service": "^4.5.13", - "@vue/compiler-sfc": "^3.1.4", + "@vue/compiler-sfc": "^3.1.5", "@vue/eslint-config-airbnb": "^5.3.0", "@vue/eslint-config-prettier": "^6.0.0", "babel-eslint": "^10.1.0", From bd20f678a393f77fd2165cd9c58a0b09cac6fd6e Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 16 Jul 2021 16:56:40 -0400 Subject: [PATCH 53/70] fix(ui): only reset disabled array when new images come in --- frontend/src/views/Match.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index cc560f2b..5e4b3f88 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -152,7 +152,7 @@ export default { delete $this.matches.source[deleteDisabled[i]]; } } - $this.matches.disabled = []; + if (data.matches.length) $this.matches.disabled = []; $this.loading.files = false; } catch (error) { $this.$toast.add({ From a27321c31842ca3f19e9d72e0d2b783cccf77dd5 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Fri, 16 Jul 2021 23:58:21 -0400 Subject: [PATCH 54/70] fix: proxy frigate status check through api for docker dns resolution --- api/src/controllers/proxy.controller.js | 17 +++++++++++++++++ api/src/routes/index.js | 6 ++++++ frontend/src/views/Config.vue | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 api/src/controllers/proxy.controller.js diff --git a/api/src/controllers/proxy.controller.js b/api/src/controllers/proxy.controller.js new file mode 100644 index 00000000..5c8bd6a9 --- /dev/null +++ b/api/src/controllers/proxy.controller.js @@ -0,0 +1,17 @@ +const axios = require('axios'); +const { respond, HTTPSuccess } = require('../util/respond.util'); +const { OK } = require('../constants/http-status'); + +module.exports.url = async (req, res) => { + try { + const { url } = req.query; + const { data } = await axios({ + method: 'get', + url, + }); + console.log(url); + respond(HTTPSuccess(OK, data), res); + } catch (error) { + respond(error, res); + } +}; diff --git a/api/src/routes/index.js b/api/src/routes/index.js index 9c1ec7b1..06dea5c2 100644 --- a/api/src/routes/index.js +++ b/api/src/routes/index.js @@ -8,7 +8,11 @@ const cameras = require('../controllers/cameras.controller'); const train = require('../controllers/train.controller'); const match = require('../controllers/match.controller'); const filesystem = require('../controllers/fs.controller'); +const proxy = require('../controllers/proxy.controller'); const validators = require('../util/validators.util'); +const { expressValidator } = require('../util/validate.util'); + +const { query } = expressValidator; const router = express.Router(); @@ -38,4 +42,6 @@ router.get('/storage/matches/:filename', validate(validators.storage().matches() router.delete('/storage/train', storage.delete); router.get('/storage/train/:name/:filename', storage.train); +router.get('/proxy', validate([query('url').isLength({ min: 1 })]), proxy.url); + module.exports = router; diff --git a/frontend/src/views/Config.vue b/frontend/src/views/Config.vue index 8b0a2fbd..988694a2 100644 --- a/frontend/src/views/Config.vue +++ b/frontend/src/views/Config.vue @@ -158,7 +158,7 @@ export default { }, async checkFrigate(url) { try { - await ApiService.get(`${url}/api/version`); + await ApiService.get(`proxy?url=${url}/api/version`); this.frigate.status = 200; } catch (error) { const status = error.response && error.response.status ? error.response.status : 500; From 49a60786fe6f0d601b5c712cb16edc3e10c70f0a Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 02:41:34 -0400 Subject: [PATCH 55/70] chore: typo --- api/src/controllers/fs.controller.js | 2 +- api/src/controllers/match.controller.js | 2 +- api/src/util/respond.util.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/controllers/fs.controller.js b/api/src/controllers/fs.controller.js index 06c257f1..375b4e83 100644 --- a/api/src/controllers/fs.controller.js +++ b/api/src/controllers/fs.controller.js @@ -20,7 +20,7 @@ module.exports.folders = () => { if (!fs.existsSync(`${STORAGE.PATH}/train/${name}`)) { fs.mkdirSync(`${STORAGE.PATH}/train/${name}`); } - respond(HTTPSuccess(OK, { sucess: true }), res); + respond(HTTPSuccess(OK, { success: true }), res); } catch (error) { respond(error, res); } diff --git a/api/src/controllers/match.controller.js b/api/src/controllers/match.controller.js index 9278da70..4a747d61 100644 --- a/api/src/controllers/match.controller.js +++ b/api/src/controllers/match.controller.js @@ -99,7 +99,7 @@ module.exports.delete = async (req, res) => { db.prepare('DELETE FROM match WHERE id = ?').run(file.id); filesystem.delete(`${STORAGE.PATH}/${file.key}`); }); - respond(HTTPSuccess(OK, { sucess: true }), res); + respond(HTTPSuccess(OK, { success: true }), res); } catch (error) { respond(error, res); } diff --git a/api/src/util/respond.util.js b/api/src/util/respond.util.js index 51b04f3a..38fc8e04 100644 --- a/api/src/util/respond.util.js +++ b/api/src/util/respond.util.js @@ -4,7 +4,7 @@ module.exports.respond = (err, res) => { try { const status = err && err.status ? err.status : BAD_REQUEST; const firstChar = parseInt(status.toString().charAt(0), 10); - let message = { sucess: true }; + let message = { success: true }; if (firstChar === 4 || firstChar === 5) { message = { error: err.message }; From 0a609396e8ec1d0aceb350093276dc64cc0f6e19 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 02:45:00 -0400 Subject: [PATCH 56/70] refactor: remove patch /match in favor of updated /train/add --- api/src/controllers/match.controller.js | 15 --------------- api/src/routes/index.js | 3 ++- frontend/src/views/Match.vue | 3 --- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/api/src/controllers/match.controller.js b/api/src/controllers/match.controller.js index 4a747d61..83b71caf 100644 --- a/api/src/controllers/match.controller.js +++ b/api/src/controllers/match.controller.js @@ -76,21 +76,6 @@ module.exports.get = async (req, res) => { } }; -module.exports.patch = async (req, res) => { - try { - const { folder, matches } = req.body; - matches.forEach((obj) => { - filesystem.move( - `${STORAGE.PATH}/${obj.key}`, - `${STORAGE.PATH}/train/${folder}/${obj.filename}` - ); - }); - respond(HTTPSuccess(OK, { sucess: true }), res); - } catch (error) { - respond(error, res); - } -}; - module.exports.delete = async (req, res) => { try { const files = req.body; diff --git a/api/src/routes/index.js b/api/src/routes/index.js index 06dea5c2..a1cc7e79 100644 --- a/api/src/routes/index.js +++ b/api/src/routes/index.js @@ -25,7 +25,8 @@ router.post('/recognize', validate(validators.recognize({ post: true })), recogn router.get('/recognize', validate(validators.recognize({ get: true })), recognize.start); router.get('/recognize/test', recognize.test); -router.get('/match', match.get).patch('/match', match.patch).delete('/match', match.delete); +router.get('/match', match.get); +router.delete('/match', match.delete); router.get('/filesystem/folders', filesystem.folders().list); router.post('/filesystem/folders/:name', filesystem.folders().create); diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index 5e4b3f88..a6207039 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -224,9 +224,6 @@ export default { filename: obj.file.filename, })); const ids = $this.matches.selected.map((obj) => obj.id); - await ApiService.patch('match', { - folder: this.trainingFolder, - matches, }); await ApiService.get(`/train/add/${this.trainingFolder}`, { files: matches.map((obj) => obj.filename) }); From eef0f9b89a73eb0e727fac15d419819d602ee41a Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 03:07:50 -0400 Subject: [PATCH 57/70] refactor(ui): updates for api changes --- frontend/src/components/Asset.vue | 7 +++---- frontend/src/views/Match.vue | 10 +++------- frontend/src/views/Train.vue | 8 ++++---- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/Asset.vue b/frontend/src/components/Asset.vue index c6e4a6bc..63cabfe7 100644 --- a/frontend/src/components/Asset.vue +++ b/frontend/src/components/Asset.vue @@ -97,7 +97,7 @@
{{ createdAt.ago }} - {{ asset.file.key }} + {{ asset.name }}
@@ -198,9 +198,8 @@ export default { watch: { folder(value) { if (value) { - const file = JSON.parse(JSON.stringify(this.asset.file)); - delete file.base64; - ApiService.patch('train', { folder: value, file }); + const { id } = this.asset; + ApiService.patch(`train/${id}`, { name: value }); this.$parent.$emit('init', true); } }, diff --git a/frontend/src/views/Match.vue b/frontend/src/views/Match.vue index a6207039..2b0c26ed 100644 --- a/frontend/src/views/Match.vue +++ b/frontend/src/views/Match.vue @@ -218,15 +218,11 @@ export default { position: 'top', accept: async () => { try { - const matches = $this.matches.selected.map((obj) => ({ - id: obj.id, - key: obj.file.key, - filename: obj.file.filename, - })); - const ids = $this.matches.selected.map((obj) => obj.id); + await ApiService.post(`/train/add/${this.trainingFolder}`, { + urls: $this.matches.selected.map((obj) => `${process.env.VUE_APP_API_URL}/storage/${obj.file.key}`), }); - await ApiService.get(`/train/add/${this.trainingFolder}`, { files: matches.map((obj) => obj.filename) }); + const ids = $this.matches.selected.map((obj) => obj.id); $this.matches.disabled = $this.matches.disabled.concat(ids); $this.matches.selected = []; diff --git a/frontend/src/views/Train.vue b/frontend/src/views/Train.vue index 0b4cb97a..6193fbd2 100644 --- a/frontend/src/views/Train.vue +++ b/frontend/src/views/Train.vue @@ -13,7 +13,7 @@

Training in progress...

-
{{ name.name }}
+
{{ name.name }} - {{ name.trained }}/{{ name.total }}
@@ -158,7 +158,7 @@ export default { accept: async () => { try { names.forEach((name) => { - ApiService.get(`train/retrain/${name}`, { ids }); + ApiService.delete(`train/remove/${name}`, { ids }); }); if (untrained.length) { await ApiService.delete('storage/train', { @@ -187,7 +187,7 @@ export default { position: 'top', accept: async () => { try { - ApiService.get(`/train/remove/${this.trainingFolder}`); + await ApiService.delete(`/train/remove/${this.trainingFolder}`); await $this.init(); } catch (error) { this.toast({ @@ -209,7 +209,7 @@ export default { position: 'top', accept: async () => { try { - ApiService.get(`/train/retrain/${this.trainingFolder}`); + await ApiService.get(`/train/retrain/${this.trainingFolder}`); await $this.init(); } catch (error) { this.toast({ From 32b9104c7ca9acaab43fba802abd63809bd69f1c Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 03:18:16 -0400 Subject: [PATCH 58/70] refactor(api): refactored /train/* and db helper functions --- api/src/controllers/train.controller.js | 56 +++++-- api/src/routes/index.js | 8 +- api/src/util/db.util.js | 206 +++++++++--------------- api/src/util/fs.util.js | 95 +++++++---- api/src/util/process.util.js | 13 +- api/src/util/train.util.js | 103 ++++++------ 6 files changed, 238 insertions(+), 243 deletions(-) diff --git a/api/src/controllers/train.controller.js b/api/src/controllers/train.controller.js index dac30d52..64a94fdc 100644 --- a/api/src/controllers/train.controller.js +++ b/api/src/controllers/train.controller.js @@ -5,8 +5,8 @@ const time = require('../util/time.util'); const database = require('../util/db.util'); const train = require('../util/train.util'); const filesystem = require('../util/fs.util'); -const { respond, HTTPSuccess } = require('../util/respond.util'); -const { OK } = require('../constants/http-status'); +const { respond, HTTPSuccess, HTTPError } = require('../util/respond.util'); +const { OK, BAD_REQUEST } = require('../constants/http-status'); const { STORAGE } = require('../constants'); const { tryParseJSON } = require('../util/validators.util'); @@ -71,8 +71,9 @@ module.exports.delete = async (req, res) => { try { perf.start(); const { name } = req.params; + const { ids } = req.body; const seconds = parseFloat((perf.stop().time / 1000).toFixed(2)); - const results = await train.remove(name); + const results = await train.remove(name, { ids }); console.log(`done untraining for ${name} in ${seconds} sec`); respond(HTTPSuccess(OK, results), res); } catch (error) { @@ -84,9 +85,29 @@ module.exports.delete = async (req, res) => { module.exports.add = async (req, res) => { try { const { name } = req.params; - const files = req.query.files || []; + const { urls } = req.body; + + let files = []; + + if (req.files) { + await Promise.all( + req.files.map(async (obj) => { + const { originalname, buffer } = obj; + const ext = `.${originalname.split('.').pop()}`; + const filename = `${originalname.replace(ext, '')}-${time.unix()}${ext}`; + await filesystem.writer(`${STORAGE.PATH}/train/${name}/${filename}`, buffer); + files.push({ name, filename }); + }) + ); + } else { + files = (urls ? await filesystem.saveURLs(urls, `train/${name}`) : []).map((filename) => ({ + filename, + name, + })); + } + + train.add(name, { files }); respond(HTTPSuccess(OK, { message: `training queued for ${name}` }), res); - await train.add(name, { files }); } catch (error) { console.error(`train add error: ${error.message}`); respond(error, res); @@ -96,8 +117,8 @@ module.exports.add = async (req, res) => { module.exports.retrain = async (req, res) => { try { const { name } = req.params; - const ids = req.query.ids || []; - await train.retrain(name, { ids }); + await train.remove(name); + train.add(name); respond(HTTPSuccess(OK, { success: true }), res); } catch (error) { console.error(`retrain error: ${error.message}`); @@ -107,10 +128,21 @@ module.exports.retrain = async (req, res) => { module.exports.patch = async (req, res) => { try { - const { key, filename } = req.body.file; - const { folder } = req.body; - filesystem.move(`${STORAGE.PATH}/${key}`, `${STORAGE.PATH}/train/${folder}/${filename}`); - train.add(folder, { files: [filename] }); + const { id } = req.params; + const { name } = req.body; + + const [currentFile] = database.get.filesById([id]); + + if (!currentFile) { + throw HTTPError(BAD_REQUEST, 'no file found'); + } + + filesystem.move( + `${STORAGE.PATH}/train/${currentFile.name}/${currentFile.filename}`, + `${STORAGE.PATH}/train/${name}/${currentFile.filename}` + ); + + train.add(name, { files: [{ name, filename: currentFile.filename }] }); respond(HTTPSuccess(OK, { success: true }), res); } catch (error) { @@ -130,7 +162,7 @@ module.exports.upload = async (req, res) => { const ext = `.${originalname.split('.').pop()}`; const filename = `${originalname.replace(ext, '')}-${time.unix()}${ext}`; await filesystem.writer(`${STORAGE.PATH}/train/${name}/${filename}`, buffer); - files.push(filename); + files.push({ name, filename }); }) ); diff --git a/api/src/routes/index.js b/api/src/routes/index.js index a1cc7e79..2c15e2ba 100644 --- a/api/src/routes/index.js +++ b/api/src/routes/index.js @@ -32,11 +32,11 @@ router.get('/filesystem/folders', filesystem.folders().list); router.post('/filesystem/folders/:name', filesystem.folders().create); router.get('/train', train.get); -router.patch('/train', train.patch); +router.patch('/train/:id', train.patch); + router.get('/train/status', train.status); -router.get('/train/add/:name', train.add); -router.post('/train/add/:name', multer().array('files[]'), train.upload); -router.get('/train/remove/:name', train.delete); +router.post('/train/add/:name', multer().array('files[]'), train.add); +router.delete('/train/remove/:name', train.delete); router.get('/train/retrain/:name', train.retrain); router.get('/storage/matches/:filename', validate(validators.storage().matches()), storage.matches); diff --git a/api/src/util/db.util.js b/api/src/util/db.util.js index 87a73bb7..3186dcef 100644 --- a/api/src/util/db.util.js +++ b/api/src/util/db.util.js @@ -1,8 +1,7 @@ const Database = require('better-sqlite3'); const time = require('./time.util'); const filesystem = require('./fs.util'); - -const { STORAGE, SAVE } = require('../constants'); +const { STORAGE } = require('../constants'); const database = this; let connection = false; @@ -53,8 +52,7 @@ module.exports.init = async () => { )` ).run(); - const files = await filesystem.files().train(); - database.insert('init', files); + await this.resync.files(); } catch (error) { console.error(`db init error: ${error.message}`); } @@ -104,139 +102,81 @@ module.exports.migrations = () => { } }; -module.exports.files = (status, data) => { - const db = database.connect(); - try { - let files = []; - - if (status === 'untrained') { - files = db - .prepare( - `SELECT * FROM file WHERE id NOT IN (SELECT fileId FROM train) AND name = ? AND isActive = 1` - ) - .all(data); - } - - if (status === 'trained') { - files = db.prepare(`SELECT * FROM train`).all(); - } - - if (status === 'trained-ids') { - files = db - .prepare(`SELECT name, fileId as id, filename FROM train WHERE name = ? GROUP BY fileId`) - .all(data); - } - - if (status === 'all') { - files = db.prepare(`SELECT * FROM file WHERE isActive = 1`).all(); - } - - files.forEach((file) => { - const { name, filename } = file; - if (file.response) file.response = JSON.parse(file.response); - file.key = `${STORAGE.PATH}/train/${name}/${filename}`; - }); - - return files; - } catch (error) { - console.error(`files error: ${error.message}`); - } +module.exports.resync = { + files: async () => { + const db = database.connect(); + db.prepare(`UPDATE file SET isActive = 0`).run(); + const files = await filesystem.files.train(); + files.forEach((obj) => this.create.file(obj)); + }, }; -module.exports.insert = (type, data = []) => { - const db = database.connect(); - - if (type === 'init') { - const files = database.files('all'); - let ids = []; - - data.forEach((file) => { - const [dbRecord] = files.filter( - (obj) => obj.name === file.name && obj.filename === file.filename - ); - if (dbRecord) ids.push(dbRecord.id); - }); - - if (ids.length) { - ids = ids.map((id) => `'${id}'`).join(','); - db.prepare(`UPDATE file SET isActive = 0 WHERE id NOT IN (${ids})`).run(); - } else { - db.prepare(`UPDATE file SET isActive = 0`).run(); - } - database.insert('file', data); - } - - if (type === 'train') { - const insert = db.prepare(` - INSERT INTO train - VALUES (:id, :fileId, :name, :filename, :detector, :meta, :createdAt) - ON CONFLICT (fileId, detector) DO UPDATE SET meta = :meta; - `); - const transaction = db.transaction((items) => { - for (const { ...item } of items) { - const temp = { ...item }; - item.id = null; - item.fileId = temp.id; - item.createdAt = time.utc(); - item.meta = temp.meta || null; - insert.run(item); - } - }); - transaction(data); - } +module.exports.get = { + untrained: (name) => { + const db = database.connect(); + return db + .prepare( + `SELECT * FROM file WHERE id NOT IN (SELECT fileId FROM train WHERE meta IS NOT NULL) AND name = ? AND isActive = 1` + ) + .all(name); + }, + filesById: (ids) => { + const db = database.connect(); + return db.prepare(`SELECT * FROM file WHERE id IN (${database.params(ids)})`).all(ids); + }, + fileByFilename(name, filename) { + const db = database.connect(); + const [file] = db + .prepare(`SELECT * FROM file WHERE name = ? AND filename = ?`) + .all(name, filename); + return file || false; + }, +}; - if (type === 'file') { - const insert = db.prepare(` - INSERT INTO file - VALUES (:id, :name, :filename, :meta, :isActive, :createdAt) - ON CONFLICT (name, filename) DO UPDATE SET isActive = 1; - `); - const transaction = db.transaction((items) => { - for (const item of items) { - item.id = null; - item.createdAt = time.utc(); - item.meta = null; - item.isActive = 1; - insert.run(item); - } +module.exports.create = { + file: ({ name, filename, meta }) => { + const db = database.connect(); + db.prepare( + `INSERT INTO file + VALUES (:id, :name, :filename, :meta, :isActive, :createdAt) + ON CONFLICT (name, filename) DO UPDATE SET isActive = 1;` + ).run({ + id: null, + name, + filename, + meta: meta || null, + createdAt: time.utc(), + isActive: 1, }); - transaction(data); - } - - if (type === 'match') { - const { camera, results, zones } = data; - const records = []; - results.forEach((group) => { - group.results.forEach((attempt) => { - let combined = SAVE.UNKNOWN - ? [...attempt.matches, ...attempt.misses] - : [...attempt.matches]; - - combined = combined.map((obj) => { - obj.filename = attempt.filename; - obj.type = group.type; - obj.detector = attempt.detector; - obj.camera = camera; - obj.zones = zones; - return obj; - }); - records.push(...combined); - }); + }, + train: ({ id, name, filename, detector, meta }) => { + const db = database.connect(); + db.prepare( + `INSERT INTO train + VALUES (:id, :fileId, :name, :filename, :detector, :meta, :createdAt) + ON CONFLICT (fileId, detector) DO UPDATE SET meta = :meta;` + ).run({ + id: null, + fileId: id, + name, + filename, + detector, + meta: meta || null, + createdAt: time.utc(), }); - - const insert = db.prepare(` - INSERT INTO match - VALUES (:id, :meta, :createdAt); - `); - const transaction = db.transaction((items) => { - for (const item of items) { - insert.run({ - id: null, - meta: JSON.stringify(item), - createdAt: time.utc(), - }); - } + }, + match: ({ filename, event, response }) => { + const db = database.connect(); + db.prepare( + `INSERT INTO match (id, filename, event, response, createdAt) VALUES (:id, :filename, :event, :response, :createdAt)` + ).run({ + id: null, + filename, + event: event ? JSON.stringify(event) : null, + response: response ? JSON.stringify(response) : null, + createdAt: time.utc(), }); - transaction(records); - } + }, }; + +module.exports.params = (array) => '?,'.repeat(array.length).slice(0, -1); diff --git a/api/src/util/fs.util.js b/api/src/util/fs.util.js index b16d3f7b..f4aac7fd 100644 --- a/api/src/util/fs.util.js +++ b/api/src/util/fs.util.js @@ -1,4 +1,6 @@ const fs = require('fs'); +const axios = require('axios'); +const time = require('./time.util'); const { STORAGE } = require('../constants'); module.exports.folders = () => { @@ -18,40 +20,39 @@ module.exports.folders = () => { }; }; -module.exports.files = () => { - return { - traverse: async (path) => { - const output = []; - let folders = await fs.promises.readdir(`${STORAGE.PATH}/${path}`, { withFileTypes: true }); - folders = folders.filter((file) => file.isDirectory()).map((file) => file.name); +module.exports.files = { + traverse: async (path) => { + const output = []; + let folders = await fs.promises.readdir(`${STORAGE.PATH}/${path}`, { withFileTypes: true }); + folders = folders.filter((file) => file.isDirectory()).map((file) => file.name); - for (const folder of folders) { - let images = await fs.promises.readdir(`${STORAGE.PATH}/${path}/${folder}`, { - withFileTypes: true, - }); - images = images - .filter((file) => file.isFile()) - .map((file) => file.name) - .filter( - (file) => - file.toLowerCase().includes('.jpeg') || - file.toLowerCase().includes('.jpg') || - file.toLowerCase().includes('.png') - ); - images.forEach((filename) => { - const { birthtime } = fs.statSync(`${STORAGE.PATH}/${path}/${folder}/${filename}`); - output.push({ name: folder, filename, key: `${path}/${folder}/${filename}`, birthtime }); - }); - } - return output.sort((a, b) => (a.birthtime < b.birthtime ? 1 : -1)); - }, - train: async () => { - return this.files().traverse('train'); - }, - matches: async () => { - return this.files().traverse('matches'); - }, - }; + for (const folder of folders) { + let images = await fs.promises.readdir(`${STORAGE.PATH}/${path}/${folder}`, { + withFileTypes: true, + }); + images = images + .filter((file) => file.isFile()) + .map((file) => file.name) + .filter( + (file) => + file.toLowerCase().includes('.jpeg') || + file.toLowerCase().includes('.jpg') || + file.toLowerCase().includes('.png') + ); + images.forEach((filename) => { + const { birthtime } = fs.statSync(`${STORAGE.PATH}/${path}/${folder}/${filename}`); + output.push({ name: folder, filename, key: `${path}/${folder}/${filename}`, birthtime }); + }); + } + return output.sort((a, b) => (a.birthtime < b.birthtime ? 1 : -1)); + }, + train: async (name) => { + const files = await this.files.traverse('train'); + return name ? files.filter((obj) => obj.name === name) : files; + }, + matches: async () => { + return this.files.traverse('matches'); + }, }; module.exports.writer = async (file, data) => { @@ -114,3 +115,31 @@ module.exports.move = (source, destination) => { console.error(`move error: ${error.message}`); } }; + +module.exports.saveURLs = async (urls, path) => { + const files = []; + for (let i = 0; i < urls.length; i++) { + try { + const validOptions = ['image/jpg', 'image/jpeg', 'image/png']; + const url = urls[i]; + const { headers, data: buffer } = await axios({ + method: 'get', + url, + responseType: 'arraybuffer', + }); + + const isValid = validOptions.includes(headers['content-type']); + + if (isValid) { + let filename = url.substring(url.lastIndexOf('/') + 1); + const ext = `.${filename.split('.').pop()}`; + filename = `${filename.replace(ext, '')}-${time.unix()}${ext}`; + fs.writeFileSync(`${STORAGE.PATH}/${path}/${filename}`, buffer); + files.push(filename); + } + } catch (error) { + console.error(error.message); + } + } + return files; +}; diff --git a/api/src/util/process.util.js b/api/src/util/process.util.js index 57873df1..44ff554d 100644 --- a/api/src/util/process.util.js +++ b/api/src/util/process.util.js @@ -5,8 +5,6 @@ const { v4: uuidv4 } = require('uuid'); const sleep = require('./sleep.util'); const filesystem = require('./fs.util'); const database = require('./db.util'); -const time = require('./time.util'); -// const frigate = require('./frigate.util'); const { recognize, normalize } = require('./detectors/actions'); const { DETECTORS, STORAGE, SAVE } = require('../constants'); @@ -80,16 +78,7 @@ module.exports.polling = async (event, { retries, id, type, url, breakMatch, MAT module.exports.save = async (event, results, filename, tmp) => { try { - const db = database.connect(); - db.prepare( - `INSERT INTO match (id, filename, event, response, createdAt) VALUES (:id, :filename, :event, :response, :createdAt)` - ).run({ - id: null, - filename, - event: JSON.stringify(event), - response: JSON.stringify(results), - createdAt: time.utc(), - }); + database.create.match({ filename, event, response: results }); await filesystem.writerStream(fs.createReadStream(tmp), `${STORAGE.PATH}/matches/${filename}`); } catch (error) { console.error(`save results error: ${error.message}`); diff --git a/api/src/util/train.util.js b/api/src/util/train.util.js index 859f8d0b..52185b88 100644 --- a/api/src/util/train.util.js +++ b/api/src/util/train.util.js @@ -1,8 +1,7 @@ const perf = require('execution-time')(); const database = require('./db.util'); -const filesystem = require('./fs.util'); const { train, remove } = require('./detectors/actions'); - +const { STORAGE } = require('../constants'); const { detectors } = require('../constants/config'); module.exports.queue = async (files) => { @@ -12,32 +11,36 @@ module.exports.queue = async (files) => { perf.start(); console.log(`queuing ${files.length} file(s) for training`); - const inserts = []; - files.forEach((file, i) => { - for (const detector of detectors()) { - inserts.push({ - ...file, + const records = []; + files.forEach(({ id, name, filename }, i) => + detectors().forEach((detector) => { + const record = { number: i + 1, + id, + name, + filename, detector, - }); - } - }); - - database.insert('train', inserts); + }; + database.create.train(record); + records.push(record); + }) + ); const outputs = []; - for (let i = 0; i < inserts.length; i++) { - const { name, filename, key, detector, number } = inserts[i]; + for (let i = 0; i < records.length; i++) { + const { name, filename, detector, number } = records[i]; console.log(`file ${number} - ${detector}: ${name} - ${filename}`); - const result = await this.process({ name, key, detector }); - outputs.push({ ...result, key }); - inserts[i].meta = JSON.stringify(result); - database.insert('train', [inserts[i]]); + const result = await this.process({ + name, + key: `${STORAGE.PATH}/train/${name}/${filename}`, + detector, + }); + outputs.push({ ...result }); + records[i].meta = JSON.stringify(result); + database.create.train(records[i]); } - database.insert('train', inserts); console.log(`training complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec`); - return outputs; } catch (error) { console.error(`queue error: ${error.message}`); } @@ -72,48 +75,50 @@ module.exports.process = async ({ name, key, detector }) => { }; module.exports.add = async (name, opts = {}) => { - const files = await filesystem.files().train(); - database.insert('init', files); - let images = opts.images || database.files('untrained', name); - if (opts.files) { - images = images.filter((obj) => opts.files.includes(obj.filename)); - } - await this.queue(images); + const { ids, files } = opts; + await database.resync.files(); + const queue = files + ? files.map((obj) => database.get.fileByFilename(obj.name, obj.filename)) + : ids + ? database.get.filesById(ids) + : database.get.untrained(name); + await this.queue(queue); }; -module.exports.remove = async (name) => { +module.exports.remove = async (name, opts = {}) => { + const { ids } = opts; + const db = database.connect(); + const promises = []; - for (const detector of detectors()) { - promises.push(remove({ detector, name })); - } + detectors().forEach((detector) => promises.push(remove({ detector, name }))); + const results = (await Promise.all(promises)).map((result, i) => ({ detector: detectors()[i], results: result.data, })); - const db = database.connect(); + if (ids && ids.length) { + db.prepare( + `DELETE FROM train WHERE name = ? AND detector IN (${database.params( + detectors() + )}) AND fileId IN (${database.params(ids)})` + ).run(name, detectors(), ids); + const addIds = database.get + .trained(name) + .filter((obj) => detectors().includes(obj.detector)) + .map((obj) => obj.fileId); + + if (addIds.length) await this.add(name, { ids: addIds }); + return { success: true }; + } + db.prepare( - `DELETE FROM train WHERE detector IN (${detectors() - .map((value) => `'${value}'`) - .join(',')}) AND name = ?` - ).run(name); + `DELETE FROM train WHERE name = ? AND detector IN (${database.params(detectors())})` + ).run(name, detectors()); return results; }; -module.exports.retrain = async (name, opts = {}) => { - const { ids } = opts; - if (ids.length) { - const db = database.connect(); - db.prepare( - `DELETE FROM train WHERE name = ? AND fileID IN (${ids.map((id) => `'${id}'`).join(',')})` - ).run(name); - } - const images = ids.length ? database.files('trained-ids', name) : null; - await this.remove(name); - await this.add(name, { images }); -}; - module.exports.status = () => { const db = database.connect(); const status = db From 093651ae1d03078646c7cec161295194b1d3e3c2 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 03:18:34 -0400 Subject: [PATCH 59/70] chore: include dev build tag for local --- frontend/src/components/Toolbar.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/Toolbar.vue b/frontend/src/components/Toolbar.vue index 8d8930a9..0f42afeb 100644 --- a/frontend/src/components/Toolbar.vue +++ b/frontend/src/components/Toolbar.vue @@ -70,6 +70,8 @@ export default { // eslint-disable-next-line no-empty } catch (error) {} if (!this.updateAvailable) setTimeout(this.checkVersion, 60000); + } else { + this.buildTag = 'dev'; } }, }, From d7876d8f40148d5ae08e8ab5604df3de75affaea Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 03:20:27 -0400 Subject: [PATCH 60/70] build: package updates --- api/package-lock.json | 6 +- api/package.json | 2 +- frontend/package-lock.json | 157 +++++++++++++------------------------ frontend/package.json | 4 +- package-lock.json | 38 ++++----- package.json | 2 +- 6 files changed, 79 insertions(+), 130 deletions(-) diff --git a/api/package-lock.json b/api/package-lock.json index ecdba247..4ca79fa7 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -125,9 +125,9 @@ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "better-sqlite3": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.4.1.tgz", - "integrity": "sha512-sk1kW3PsWE7W7G9qbi5TQxCROlQVR8YWlp4srbyrwN5DrLeamKfrm3JExwOiNSAYyJv8cw5/2HOfvF/ipZj4qg==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.4.3.tgz", + "integrity": "sha512-07bKjClZg/f4KMVRkzWtoIvazVPcF1gsvVKVIXlxwleC2DxuIhnra3KCMlUT1rFeRYXXckot2a46UciF2d9KLw==", "requires": { "bindings": "^1.5.0", "prebuild-install": "^6.0.1", diff --git a/api/package.json b/api/package.json index 772a59f9..fb930869 100644 --- a/api/package.json +++ b/api/package.json @@ -18,7 +18,7 @@ "homepage": "https://github.com/jakowenko/double-take#readme", "dependencies": { "axios": "^0.21.1", - "better-sqlite3": "^7.4.1", + "better-sqlite3": "^7.4.3", "canvas": "^2.8.0", "cors": "^2.8.5", "execution-time": "^1.4.1", diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 69542345..4f002e83 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1118,9 +1118,9 @@ } }, "@eslint/eslintrc": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz", - "integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==", + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", + "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==", "dev": true, "requires": { "ajv": "^6.12.4", @@ -1134,27 +1134,10 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { - "acorn": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", - "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", - "dev": true - }, - "espree": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", - "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", - "dev": true, - "requires": { - "acorn": "^7.4.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^1.3.0" - } - }, "globals": { - "version": "13.9.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.9.0.tgz", - "integrity": "sha512-74/FduwI/JaIrr1H8e71UbDE+5x7pIPs1C2rrwC52SszOo043CsWOZEMW7o2Y58xwm9b+0RBKDxY5n2sUpEFxA==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz", + "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -2359,9 +2342,9 @@ "dev": true }, "acorn-jsx": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", - "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true }, "acorn-walk": { @@ -5113,13 +5096,13 @@ "dev": true }, "eslint": { - "version": "7.30.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz", - "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==", + "version": "7.31.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.31.0.tgz", + "integrity": "sha512-vafgJpSh2ia8tnTkNUkwxGmnumgckLh5aAbLa1xRmIn9+owi8qBNGKL+B881kNKNTy7FFqTEkpNkUvmw0n6PkA==", "dev": true, "requires": { "@babel/code-frame": "7.12.11", - "@eslint/eslintrc": "^0.4.2", + "@eslint/eslintrc": "^0.4.3", "@humanwhocodes/config-array": "^0.5.0", "ajv": "^6.10.0", "chalk": "^4.0.0", @@ -5169,12 +5152,6 @@ "@babel/highlight": "^7.10.4" } }, - "acorn": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", - "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", - "dev": true - }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -5251,29 +5228,10 @@ "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", "dev": true }, - "espree": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", - "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", - "dev": true, - "requires": { - "acorn": "^7.4.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^1.3.0" - }, - "dependencies": { - "eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", - "dev": true - } - } - }, "globals": { - "version": "13.9.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.9.0.tgz", - "integrity": "sha512-74/FduwI/JaIrr1H8e71UbDE+5x7pIPs1C2rrwC52SszOo043CsWOZEMW7o2Y58xwm9b+0RBKDxY5n2sUpEFxA==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz", + "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -5660,41 +5618,15 @@ } }, "eslint-plugin-vue": { - "version": "7.13.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-7.13.0.tgz", - "integrity": "sha512-u0+jL8h2MshRuMTCLslktxRsPTjlENNcNufhgHu01N982DmHVdeFniyMPoVLLRjACQOwdz3FdlsgYGBMBG+AKg==", + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-vue/-/eslint-plugin-vue-7.14.0.tgz", + "integrity": "sha512-IW5A2Td0wEWjFGaGVEO24JNXa8cVFzAQTXrYv/Vu3zyDVS9sjwOpZY0iqub7FOkT2AK3Imtw4U4wg48pP9oWww==", "dev": true, "requires": { "eslint-utils": "^2.1.0", "natural-compare": "^1.4.0", - "semver": "^7.3.2", - "vue-eslint-parser": "^7.8.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, - "requires": { - "yallist": "^4.0.0" - } - }, - "semver": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", - "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", - "dev": true, - "requires": { - "lru-cache": "^6.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true - } + "semver": "^6.3.0", + "vue-eslint-parser": "^7.9.0" } }, "eslint-scope": { @@ -5723,14 +5655,14 @@ "dev": true }, "espree": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", - "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", + "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", "dev": true, "requires": { - "acorn": "^7.1.1", - "acorn-jsx": "^5.2.0", - "eslint-visitor-keys": "^1.1.0" + "acorn": "^7.4.0", + "acorn-jsx": "^5.3.1", + "eslint-visitor-keys": "^1.3.0" }, "dependencies": { "acorn": { @@ -6291,9 +6223,9 @@ } }, "flatted": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.0.tgz", - "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.1.tgz", + "integrity": "sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg==", "dev": true }, "flush-write-stream": { @@ -12204,9 +12136,9 @@ }, "dependencies": { "ajv": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.1.tgz", - "integrity": "sha512-42VLtQUOLefAvKFAQIxIZDaThq6om/PrfP0CYk3/vn+y4BMNkKnbli8ON2QCiHov4KkzOSJ/xSoBJdayiiYvVQ==", + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.2.tgz", + "integrity": "sha512-9807RlWAgT564wT+DjeyU5OFMPjmzxVobvDFmNAhY+5zD6A2ly3jDp6sgnfyDtlIQ+7H97oc/DGCzzfu9rjw9w==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -12982,9 +12914,9 @@ "integrity": "sha512-8Dgws6dwt+1LoKEU4HDzIPfaRgWHl0xLDLW6kxrAk/z3GAJ2thkajcXC4qZgmqj0cZRx/Z/dzooKi5HWtUQwxA==" }, "vue-eslint-parser": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-7.8.0.tgz", - "integrity": "sha512-ehmmrLZNYLUoKayvVW8l8HyPQIfuYZHiJoQLRP3dapDlTU7bGs4tqIKVGdAEpMuXS/b4R/PImCt7Tkj4UhX1SQ==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-7.9.0.tgz", + "integrity": "sha512-QBlhZ5LteDRVy2dISfQhNEmmcqph+GTaD4SH41bYzXcVHFPJ9p34zCG6QAqOZVa8PKaVgbomFnoZpGJRZi14vg==", "dev": true, "requires": { "debug": "^4.1.1", @@ -12996,6 +12928,12 @@ "semver": "^6.3.0" }, "dependencies": { + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, "eslint-scope": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", @@ -13005,6 +12943,17 @@ "esrecurse": "^4.3.0", "estraverse": "^4.1.1" } + }, + "espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "requires": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + } } } }, diff --git a/frontend/package.json b/frontend/package.json index 1883e028..994cbcda 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -39,8 +39,8 @@ "@vue/eslint-config-airbnb": "^5.3.0", "@vue/eslint-config-prettier": "^6.0.0", "babel-eslint": "^10.1.0", - "eslint": "^7.30.0", - "eslint-plugin-vue": "^7.13.0", + "eslint": "^7.31.0", + "eslint-plugin-vue": "^7.14.0", "node-sass": "^6.0.1", "sass-loader": "^10.2.0" }, diff --git a/package-lock.json b/package-lock.json index 76d72b07..a257a1f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -288,9 +288,9 @@ } }, "@eslint/eslintrc": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.2.tgz", - "integrity": "sha512-8nmGq/4ycLpIwzvhI4tNDmQztZ8sp+hI7cyG8i1nQDhkAbRzHpXPidRAHlNvCZQpJTKw5ItIpMw9RSToGF00mg==", + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", + "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==", "dev": true, "requires": { "ajv": "^6.12.4", @@ -383,9 +383,9 @@ "dev": true }, "acorn-jsx": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", - "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true }, "ajv": { @@ -1028,13 +1028,13 @@ "dev": true }, "eslint": { - "version": "7.30.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.30.0.tgz", - "integrity": "sha512-VLqz80i3as3NdloY44BQSJpFw534L9Oh+6zJOUaViV4JPd+DaHwutqP7tcpkW3YiXbK6s05RZl7yl7cQn+lijg==", + "version": "7.31.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.31.0.tgz", + "integrity": "sha512-vafgJpSh2ia8tnTkNUkwxGmnumgckLh5aAbLa1xRmIn9+owi8qBNGKL+B881kNKNTy7FFqTEkpNkUvmw0n6PkA==", "dev": true, "requires": { "@babel/code-frame": "7.12.11", - "@eslint/eslintrc": "^0.4.2", + "@eslint/eslintrc": "^0.4.3", "@humanwhocodes/config-array": "^0.5.0", "ajv": "^6.10.0", "chalk": "^4.0.0", @@ -1485,9 +1485,9 @@ } }, "flatted": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.0.tgz", - "integrity": "sha512-XprP7lDrVT+kE2c2YlfiV+IfS9zxukiIOvNamPNsImNhXadSsQEbosItdL9bUQlCZXR13SvPk20BjWSWLA7m4A==", + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.1.tgz", + "integrity": "sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg==", "dev": true }, "fs-extra": { @@ -1599,9 +1599,9 @@ } }, "globals": { - "version": "13.9.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.9.0.tgz", - "integrity": "sha512-74/FduwI/JaIrr1H8e71UbDE+5x7pIPs1C2rrwC52SszOo043CsWOZEMW7o2Y58xwm9b+0RBKDxY5n2sUpEFxA==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.10.0.tgz", + "integrity": "sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g==", "dev": true, "requires": { "type-fest": "^0.20.2" @@ -3105,9 +3105,9 @@ }, "dependencies": { "ajv": { - "version": "8.6.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.1.tgz", - "integrity": "sha512-42VLtQUOLefAvKFAQIxIZDaThq6om/PrfP0CYk3/vn+y4BMNkKnbli8ON2QCiHov4KkzOSJ/xSoBJdayiiYvVQ==", + "version": "8.6.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.6.2.tgz", + "integrity": "sha512-9807RlWAgT564wT+DjeyU5OFMPjmzxVobvDFmNAhY+5zD6A2ly3jDp6sgnfyDtlIQ+7H97oc/DGCzzfu9rjw9w==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", diff --git a/package.json b/package.json index e7921fc1..201a3ba2 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "devDependencies": { "@commitlint/cli": "^12.1.4", "@commitlint/config-conventional": "^12.1.4", - "eslint": "^7.30.0", + "eslint": "^7.31.0", "eslint-config-airbnb-base": "^14.2.1", "eslint-config-prettier": "^8.1.0", "eslint-plugin-import": "^2.23.4", From 1fde7e0c7e8467b0050da28cef93a8c0d7bd9097 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 03:37:27 -0400 Subject: [PATCH 61/70] fix: add missing get.trained util --- api/src/util/db.util.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/src/util/db.util.js b/api/src/util/db.util.js index 3186dcef..d033772c 100644 --- a/api/src/util/db.util.js +++ b/api/src/util/db.util.js @@ -120,6 +120,10 @@ module.exports.get = { ) .all(name); }, + trained: (name) => { + const db = database.connect(); + return db.prepare(`SELECT * FROM train WHERE name = ?`).all(name); + }, filesById: (ids) => { const db = database.connect(); return db.prepare(`SELECT * FROM file WHERE id IN (${database.params(ids)})`).all(ids); From 3bdfc3dee773ad6ccc6c6cb3729679eedb63301a Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 16:39:54 -0400 Subject: [PATCH 62/70] fix: return untrained files for detectors that were never untrained before removed from config --- api/src/util/db.util.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/util/db.util.js b/api/src/util/db.util.js index d033772c..dbbae16c 100644 --- a/api/src/util/db.util.js +++ b/api/src/util/db.util.js @@ -116,9 +116,11 @@ module.exports.get = { const db = database.connect(); return db .prepare( - `SELECT * FROM file WHERE id NOT IN (SELECT fileId FROM train WHERE meta IS NOT NULL) AND name = ? AND isActive = 1` + `SELECT * FROM file WHERE id NOT IN (SELECT fileId FROM train WHERE meta IS NOT NULL AND detector IN (${database.params( + DETECTORS + )})) AND name = ? AND isActive = 1` ) - .all(name); + .all(DETECTORS, name); }, trained: (name) => { const db = database.connect(); From 76d183a214a1d1dc8165ad976b8bc0c0f171aa69 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 16:44:45 -0400 Subject: [PATCH 63/70] chore: update global detectors variable to follow constants syntax --- api/src/util/db.util.js | 1 + api/src/util/train.util.js | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/api/src/util/db.util.js b/api/src/util/db.util.js index dbbae16c..7c2dd86e 100644 --- a/api/src/util/db.util.js +++ b/api/src/util/db.util.js @@ -2,6 +2,7 @@ const Database = require('better-sqlite3'); const time = require('./time.util'); const filesystem = require('./fs.util'); const { STORAGE } = require('../constants'); +const DETECTORS = require('../constants/config').detectors(); const database = this; let connection = false; diff --git a/api/src/util/train.util.js b/api/src/util/train.util.js index 52185b88..2ac08f2b 100644 --- a/api/src/util/train.util.js +++ b/api/src/util/train.util.js @@ -2,7 +2,7 @@ const perf = require('execution-time')(); const database = require('./db.util'); const { train, remove } = require('./detectors/actions'); const { STORAGE } = require('../constants'); -const { detectors } = require('../constants/config'); +const DETECTORS = require('../constants/config').detectors(); module.exports.queue = async (files) => { try { @@ -13,7 +13,7 @@ module.exports.queue = async (files) => { const records = []; files.forEach(({ id, name, filename }, i) => - detectors().forEach((detector) => { + DETECTORS.forEach((detector) => { const record = { number: i + 1, id, @@ -90,22 +90,22 @@ module.exports.remove = async (name, opts = {}) => { const db = database.connect(); const promises = []; - detectors().forEach((detector) => promises.push(remove({ detector, name }))); + DETECTORS.forEach((detector) => promises.push(remove({ detector, name }))); const results = (await Promise.all(promises)).map((result, i) => ({ - detector: detectors()[i], + detector: DETECTORS[i], results: result.data, })); if (ids && ids.length) { db.prepare( `DELETE FROM train WHERE name = ? AND detector IN (${database.params( - detectors() + DETECTORS )}) AND fileId IN (${database.params(ids)})` - ).run(name, detectors(), ids); + ).run(name, DETECTORS, ids); const addIds = database.get .trained(name) - .filter((obj) => detectors().includes(obj.detector)) + .filter((obj) => DETECTORS.includes(obj.detector)) .map((obj) => obj.fileId); if (addIds.length) await this.add(name, { ids: addIds }); @@ -113,8 +113,9 @@ module.exports.remove = async (name, opts = {}) => { } db.prepare( - `DELETE FROM train WHERE name = ? AND detector IN (${database.params(detectors())})` - ).run(name, detectors()); + `DELETE FROM train WHERE name = ? AND detector IN (${database.params(DETECTORS)})` + ).run(name, DETECTORS); + return results; }; From acb54b0278751f16444c910500e34b4e3325b0ce Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Tue, 20 Jul 2021 16:45:27 -0400 Subject: [PATCH 64/70] chore: update training logs and location of execution-time --- api/src/controllers/train.controller.js | 5 +---- api/src/util/train.util.js | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/api/src/controllers/train.controller.js b/api/src/controllers/train.controller.js index 64a94fdc..4bed8c1a 100644 --- a/api/src/controllers/train.controller.js +++ b/api/src/controllers/train.controller.js @@ -1,4 +1,3 @@ -const perf = require('execution-time')(); const fs = require('fs'); const sharp = require('sharp'); const time = require('../util/time.util'); @@ -69,12 +68,10 @@ module.exports.get = async (req, res) => { module.exports.delete = async (req, res) => { try { - perf.start(); const { name } = req.params; const { ids } = req.body; - const seconds = parseFloat((perf.stop().time / 1000).toFixed(2)); const results = await train.remove(name, { ids }); - console.log(`done untraining for ${name} in ${seconds} sec`); + respond(HTTPSuccess(OK, results), res); } catch (error) { console.error(`train delete error: ${error.message}`); diff --git a/api/src/util/train.util.js b/api/src/util/train.util.js index 2ac08f2b..264713a9 100644 --- a/api/src/util/train.util.js +++ b/api/src/util/train.util.js @@ -8,9 +8,6 @@ module.exports.queue = async (files) => { try { if (!files.length) return []; - perf.start(); - console.log(`queuing ${files.length} file(s) for training`); - const records = []; files.forEach(({ id, name, filename }, i) => DETECTORS.forEach((detector) => { @@ -28,8 +25,7 @@ module.exports.queue = async (files) => { const outputs = []; for (let i = 0; i < records.length; i++) { - const { name, filename, detector, number } = records[i]; - console.log(`file ${number} - ${detector}: ${name} - ${filename}`); + const { name, filename, detector } = records[i]; const result = await this.process({ name, key: `${STORAGE.PATH}/train/${name}/${filename}`, @@ -39,8 +35,6 @@ module.exports.queue = async (files) => { records[i].meta = JSON.stringify(result); database.create.train(records[i]); } - - console.log(`training complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec`); } catch (error) { console.error(`queue error: ${error.message}`); } @@ -75,6 +69,7 @@ module.exports.process = async ({ name, key, detector }) => { }; module.exports.add = async (name, opts = {}) => { + perf.start(); const { ids, files } = opts; await database.resync.files(); const queue = files @@ -82,10 +77,17 @@ module.exports.add = async (name, opts = {}) => { : ids ? database.get.filesById(ids) : database.get.untrained(name); + + console.log(`${name}: queuing ${queue.length} file(s) for training`); await this.queue(queue); + console.log( + `${name}: training complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec` + ); }; module.exports.remove = async (name, opts = {}) => { + perf.start(); + const { ids } = opts; const db = database.connect(); @@ -116,6 +118,9 @@ module.exports.remove = async (name, opts = {}) => { `DELETE FROM train WHERE name = ? AND detector IN (${database.params(DETECTORS)})` ).run(name, DETECTORS); + console.log( + `${name}: untraining complete in ${parseFloat((perf.stop().time / 1000).toFixed(2))} sec` + ); return results; }; From 6347ee38780698eb3ff028708678007ec0b25282 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:13:19 -0400 Subject: [PATCH 65/70] chore: remove unused file id --- frontend/src/components/Asset.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/Asset.vue b/frontend/src/components/Asset.vue index 63cabfe7..96222e92 100644 --- a/frontend/src/components/Asset.vue +++ b/frontend/src/components/Asset.vue @@ -7,7 +7,7 @@
From ae84369575cec05a4819460d6122e429b7dbd0fa Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:13:52 -0400 Subject: [PATCH 66/70] chore: hide type label if manual --- frontend/src/components/Asset.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/Asset.vue b/frontend/src/components/Asset.vue index 96222e92..772c264c 100644 --- a/frontend/src/components/Asset.vue +++ b/frontend/src/components/Asset.vue @@ -90,7 +90,7 @@
- + From e6f715f0cdfc0c02930001dd3a1dba811be624a0 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:14:22 -0400 Subject: [PATCH 67/70] fix: catch empty cameras config --- api/src/controllers/cameras.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/controllers/cameras.controller.js b/api/src/controllers/cameras.controller.js index 313a2103..88533dcf 100644 --- a/api/src/controllers/cameras.controller.js +++ b/api/src/controllers/cameras.controller.js @@ -8,7 +8,7 @@ module.exports.event = async (req, res) => { const { camera } = req.params; const { attempts, break: breakMatch } = req.query; - const { SNAPSHOT } = Object.keys(CAMERAS) + const { SNAPSHOT } = Object.keys(CAMERAS || {}) .filter((key) => key.toLowerCase() === camera) .reduce((obj, key) => { obj = CAMERAS[key]; From b98b181d0228ea6cf289fd8a16a1c8d9420a17d3 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:23:31 -0400 Subject: [PATCH 68/70] docs: remove api docs from readme in favor of postman docs --- README.md | 353 +++++++++++------------------------------------------- 1 file changed, 71 insertions(+), 282 deletions(-) diff --git a/README.md b/README.md index bc5f51d2..997e9250 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Unified UI and API for processing and training images for facial recognition.

- +

## Why? @@ -22,7 +22,7 @@ There's a lot of great open source software to perform facial recognition, but e - [Frigate](https://github.com/blakeblackshear/frigate) v0.8.0-0.9.0 -## Use Cases +## Integrations ### [Frigate](https://github.com/blakeblackshear/frigate) @@ -56,283 +56,40 @@ If a match is found the image is saved to `/.storage/matches/${filename}`. Trigger automations / notifications when images are processed. -If the MQTT integration is configured within Home Assistant, then sensors can be created from the topics that Double Take publishes to. +If the MQTT integration is configured within Home Assistant, then sensors will automatically be created. -```yaml -sensor: - - platform: mqtt - name: David - icon: mdi:account - state_topic: 'double-take/matches/david' - json_attributes_topic: 'double-take/matches/david' - value_template: '{{ value_json.camera }}' - availability_topic: 'double-take/available' -``` - -

- -

- -## Notify Services - -### [Gotify](https://gotify.net) +#### Notification Automation ```yaml -notify: - gotify: - url: http://192.168.1.1:8080 - token: XXXXXXX -``` - -## UI - -The UI is accessible from `http://localhost:3000/#/`. - -### `/#/config` - -Make changes to the configuration and restart the API. - -### `/#/files` - -View files and training results from detectors. - -## API - -### `GET - /api/config` - -Output configuration. - -```shell -curl -X GET "http://localhost:3000/api/config" \ --H "Content-type: application/json" +alias: Notify +trigger: + - platform: state + entity_id: sensor.double_take_david + - platform: state + entity_id: sensor.double_take_unknown +condition: + - condition: template + value_template: '{{ trigger.to_state.state != trigger.from_state.state }}' +action: + - service: notify.mobile_app + data: + message: >- + {{trigger.to_state.attributes.friendly_name}} is near the + {{trigger.to_state.state}} @ + {{trigger.to_state.attributes.match.confidence}}% by + {{trigger.to_state.attributes.match.detector}}:{{trigger.to_state.attributes.match.type}} + taking {{trigger.to_state.attributes.attempts}} attempt(s) @ + {{trigger.to_state.attributes.duration}} sec + data: + attachment: + url: http://192.168.1.2:3000/api/storage/matches/{{trigger.to_state.attributes.match.filename}}?box=true + actions: + - action: URI + title: View Image + uri: http://192.168.1.2:3000/api/storage/matches/{{trigger.to_state.attributes.match.filename}}?box=true ``` -```json -{ - "confidence": { "match": 60, "unknown": 40 }, - "detectors": { - "compreface": { - "url": "http://192.168.1.1:8000", - "key": "xxx-xxx-xxx-xxx-xxx" - }, - "deepstack": { "url": "http://192.168.1.1:8001" }, - "facebox": { "url": "http://192.168.1.1:8002" } - }, - "frigate": { - "attempts": { "latest": 10, "snapshot": 0 }, - "image": { "height": 500 }, - "url": "http://192.168.1.1:5000", - "cameras": [], - "zones": [] - }, - "mqtt": { - "topics": { - "frigate": "frigate/events", - "matches": "double-take/matches", - "cameras": "double-take/cameras" - }, - "host": "192.168.1.1" - }, - "objects": { "face": { "min_area_match": 10000 } }, - "purge": { "matches": 168, "unknown": 8 }, - "save": { "matches": true, "unknown": true }, - "server": { "port": 3000 }, - "storage": { "path": "./.storage" }, - "time": { "timezone": "UTC", "format": "F" } -} -``` - -### `GET - /api/recognize` - -Process image for recognition. - -| Query Params | Default | Description | -| ------------ | ------------- | ---------------------------------------------------- | -| url | | URL of image to pass to facial recognition detectors | -| attempts | `1` | Number of attempts before stopping without a match | -| results | `best` | Options: `best`, `all` | -| break | `true` | Break attempt loop if a match is found | -| camera | `double-take` | Camera name used in output results | - -```shell -curl -X GET "http://localhost:3000/api/recognize?url=https://jakowenko.com/img/david.92f395c6.jpg" \ --H "Content-type: application/json" -``` - -```json -{ - "id": "fd0d91ee-1ecc-4b93-aee4-4e6523090f9a", - "duration": 4.04, - "timestamp": "2021-04-28T13:12:06.624-04:00", - "attempts": 1, - "camera": "double-take", - "zones": [], - "matches": [ - { - "name": "david", - "confidence": 100, - "match": true, - "box": { "top": 286, "left": 744, "width": 319, "height": 397 }, - "type": "manual", - "detector": "compreface", - "duration": 0.92, - "filename": "e4f181f2-21bd-4aa3-a2a8-9b7730d9d9dd.jpg" - } - ] -} -``` - -### `GET - /api/recognize/test` - -Process test image for recognition and output the configured detectors raw response. - -```shell -curl -X GET "http://localhost:3000/api/recognize/test" \ --H "Content-type: application/json" -``` - -```json -[ - { - "detector": "deepstack", - "response": { - "success": true, - "predictions": [ - { - "confidence": 0.0260843, - "userid": "david", - "y_min": 194, - "x_min": 215, - "y_max": 392, - "x_max": 358 - } - ], - "duration": 0 - } - }, - { - "detector": "compreface", - "response": { - "result": [ - { - "box": { - "probability": 0.93259, - "x_max": 369, - "y_max": 412, - "x_min": 190, - "y_min": 165 - }, - "subjects": [{ "subject": "david", "similarity": 0.03813 }] - } - ] - } - }, - { - "detector": "facebox", - "response": { - "success": true, - "facesCount": 1, - "faces": [ - { - "rect": { "top": 219, "left": 218, "width": 155, "height": 155 }, - "matched": false, - "confidence": 0 - } - ] - } - } -] -``` - -### `GET - /api/cameras/:camera` - -Process images via HTTP or MQTT for configured cameras. - -| Query Params | Default | Description | -| ------------ | ------- | -------------------------------------------------- | -| attempts | `1` | Number of attempts before stopping without a match | -| break | `true` | Break attempt loop if a match is found | - -```yaml -cameras: - driveway: - snapshot: - topic: driveway/snapshot - url: http://192.168.1.1/latest.jpg -``` - -```shell -curl -X GET "http://localhost:3000/api/cameras/driveway" \ --H "Content-type: application/json" -``` - -```json -{ - "id": "01da75f4-47c5-4558-bc48-d6a90ddc9f05", - "duration": 1.41, - "timestamp": "2021-06-28T04:10:21.485Z", - "attempts": 1, - "camera": "driveway", - "zones": [], - "matches": [ - { - "name": "david", - "confidence": 100, - "match": true, - "box": { "top": 91, "left": 145, "width": 101, "height": 135 }, - "type": "camera-event", - "duration": 0.83, - "detector": "deepstack", - "filename": "bd7b3ed5-4a9a-46e9-a162-d73e4ca58f1f.jpg" - } - ] -} -``` - -### `GET - /api/train/add/:name` - -Train detectors with images from `./storage/train/${name}`. Once an image is trained, it will not be reprocessed unless it is removed via the API. - -```shell -curl -X GET "http://localhost:3000/api/train/add/david" \ --H "Content-type: application/json" -``` - -```json -{ - "message": "training queued for david using 2 image(s): check logs for details" -} -``` - -### `GET - /api/train/remove/:name` - -Remove all images for the specific name from detectors. This does not delete the files from the training folder. - -```shell -curl -X GET "http://localhost:3000/api/train/remove/david" \ --H "Content-type: application/json" -``` - -```json -[ - { - "detector": "compreface", - "results": [{ "image_id": "46f0db76-38ec-4b50-b8c7-de7d4080517d", "subject": "david" }] - }, - { "detector": "deepstack", "results": { "success": true, "duration": 0 } }, - { "detector": "facebox", "results": { "success": true } } -] -``` - -### `GET - /api/storage/matches/:filename` - -Render match image. - -| Query Params | Default | Description | -| ------------ | ------- | ------------------------------------------------------- | -| box | `false` | Draw bounding box around face with name and confidence. | - -## MQTT +### MQTT Publish results to `double-take/matches/${name}` and `double-take/cameras/${camera}`. The number of results will also be published to `double-take/cameras/${camera}/person` and will reset back to `0` after 30 seconds. @@ -341,7 +98,7 @@ mqtt: host: 192.168.1.1 ``` -**double-take/matches/david** +#### double-take/matches/david ```json { @@ -364,7 +121,7 @@ mqtt: } ``` -**double-take/cameras/back-door** +#### double-take/cameras/back-door ```json { @@ -389,6 +146,43 @@ mqtt: } ``` +## Notify Services + +### [Gotify](https://gotify.net) + +```yaml +notify: + gotify: + url: http://192.168.1.1:8080 + token: XXXXXXX +``` + +## UI + +The UI is accessible from `http://localhost:3000/#/`. + +### Matches + +

+ +

+ +### Train + +

+ +

+ +### Config + +

+ +

+ +## API + +Documentation can be viewed on [Postman](https://documenter.getpostman.com/view/1013188/TzsWuAa8). + ## Usage ### Docker Run @@ -398,7 +192,6 @@ docker run -d \ --name=double-take \ --restart=unless-stopped \ -p 3000:3000 \ - -v ${PWD}/config.yml:/double-take/config.yml \ -v ${PWD}/.storage:/.storage \ jakowenko/double-take ``` @@ -414,7 +207,6 @@ services: image: jakowenko/double-take restart: unless-stopped volumes: - - ${PWD}/config.yml:/double-take/config.yml - ${PWD}/.storage:/.storage ports: - 3000:3000 @@ -493,6 +285,7 @@ time: | mqtt.username | | MQTT username | | mqtt.password | | MQTT password | | mqtt.topics.frigate | `frigate/events` | MQTT topic for Frigate message subscription | +| mqtt.topics.homeassistant | `homeassistant` | MQTT topic for Home Assistant Discovery subscription | | mqtt.topics.matches | `double-take/matches` | MQTT topic where matches are published | | mqtt.topics.cameras | `double-take/cameras` | MQTT topic where matches are published by camera name | | confidence.match | `60` | Minimum confidence needed to consider a result a match | @@ -523,7 +316,3 @@ time: | notify.gotify.zones | | Only notify from specific zones | | time.format | | Defaults to ISO 8601 format with support for [token-based formatting](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) | | time.timezone | `UTC` | Time zone used in logs | - -## Known Issues - -In rare scenarios, requesting images from Frigate's API causes Frigate to crash. There is an [open issue](https://github.com/blakeblackshear/frigate/discussions/853) with more information, but it appears sometimes the database connection isn't being closed in time causing Frigate's API to crash. This appears to be related to processing the `snapshot.jpg`. Setting `frigate.attempts.snapshot` to `0` will disable the processing of that image. From 60e3c742e06cdb9d151e8cae45b3312b3aaf3b04 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:45:16 -0400 Subject: [PATCH 69/70] docs: add donation section --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 997e9250..2b40bfe0 100644 --- a/README.md +++ b/README.md @@ -316,3 +316,7 @@ time: | notify.gotify.zones | | Only notify from specific zones | | time.format | | Defaults to ISO 8601 format with support for [token-based formatting](https://moment.github.io/luxon/docs/manual/formatting.html#table-of-tokens) | | time.timezone | `UTC` | Time zone used in logs | + +## Donations + +If you would like to make a donation to support development, please use [Github Sponsors](https://github.com/sponsors/jakowenko). From 374e4d9c836dd3e3e0d5b90bf0a4410dcd5ed6f8 Mon Sep 17 00:00:00 2001 From: David Jakowenko Date: Wed, 21 Jul 2021 02:46:19 -0400 Subject: [PATCH 70/70] refactor: decrease image quality on matches and train page to reduce load --- api/src/controllers/match.controller.js | 1 + api/src/controllers/train.controller.js | 1 + 2 files changed, 2 insertions(+) diff --git a/api/src/controllers/match.controller.js b/api/src/controllers/match.controller.js index 83b71caf..583d311a 100644 --- a/api/src/controllers/match.controller.js +++ b/api/src/controllers/match.controller.js @@ -53,6 +53,7 @@ module.exports.get = async (req, res) => { output.file = matchProp.file; } else if (fs.existsSync(`${STORAGE.PATH}/${key}`)) { const base64 = await sharp(`${STORAGE.PATH}/${key}`) + .jpeg({ quality: 70 }) .resize(500) .withMetadata() .toBuffer(); diff --git a/api/src/controllers/train.controller.js b/api/src/controllers/train.controller.js index 4bed8c1a..622ebddf 100644 --- a/api/src/controllers/train.controller.js +++ b/api/src/controllers/train.controller.js @@ -51,6 +51,7 @@ module.exports.get = async (req, res) => { if (fs.existsSync(`${STORAGE.PATH}/${key}`)) { const base64 = await sharp(`${STORAGE.PATH}/${key}`) + .jpeg({ quality: 70 }) .resize(500) .withMetadata() .toBuffer();