From 3a0383269190259988f323f1a33ad99529e6971d Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 20 Jun 2021 10:55:30 +0100 Subject: [PATCH 1/3] Initial implementation --- config/default.json | 9 ++++ package.json | 1 + src/app.js | 82 +++++++++------------------------- src/lib/cache/cacheFactory.js | 15 +++++++ src/lib/cache/redisCache.js | 40 +++++++++++++++++ src/lib/cache/standardCache.js | 21 +++++++++ 6 files changed, 107 insertions(+), 61 deletions(-) create mode 100644 src/lib/cache/cacheFactory.js create mode 100644 src/lib/cache/redisCache.js create mode 100644 src/lib/cache/standardCache.js diff --git a/config/default.json b/config/default.json index 096fdf3f5..b2431b4d2 100644 --- a/config/default.json +++ b/config/default.json @@ -110,6 +110,15 @@ "user": "poracleuser", "password": "poraclepassword", "port": 3306 + }, + "cache": "memory", // can be 'memory' or 'redis' + "redis": { + "host": "127.0.0.1", + "port": 6379, + "user": null, // default redis installation is no user/password but localhost only connections + "password": null, + "prefix": null, // usually leave as null! prefix all keys with this string (multi instance) + "database": null // usually leave as null! select this database on startup (advanced) } }, // diff --git a/package.json b/package.json index f41ca1676..e7d230144 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "pogo-protos": "git+https://github.com/Furtif/pogo-protos.git", "point-in-polygon": "^1.1.0", "readline-sync": "^1.4.10", + "redis": "^3.1.2", "s2-geometry": "^1.2.10", "strip-json-comments": "^3.1.1", "telegraf": "^4.3.0", diff --git a/src/app.js b/src/app.js index 11178342c..ac9c5a329 100644 --- a/src/app.js +++ b/src/app.js @@ -4,11 +4,9 @@ require('events').EventEmitter.prototype._maxListeners = 100 const { writeHeapSnapshot } = require('v8') const fs = require('fs') -const fsp = require('fs').promises const util = require('util') const { S2 } = require('s2-geometry') const { Worker, MessageChannel } = require('worker_threads') -const NodeCache = require('node-cache') const fastify = require('fastify')({ bodyLimit: 5242880, }) @@ -43,8 +41,6 @@ const readDir = util.promisify(fs.readdir) const telegraf = new Telegraf(config.telegram.token)// , { channelMode: true }) const telegrafChannel = config.telegram.channelToken ? new Telegraf(config.telegram.channelToken)/* , { channelMode: true }) */ : null -const cache = new NodeCache({ stdTTL: 5400, useClones: false }) // 90 minutes - const DiscordWorker = require('./lib/discord/discordWorker') const DiscordWebhookWorker = require('./lib/discord/discordWebhookWorker') const DiscordCommando = require('./lib/discord/commando') @@ -66,7 +62,6 @@ fastify.decorate('controllerLog', logs.controller) fastify.decorate('webhooks', logs.webhooks) fastify.decorate('config', config) fastify.decorate('knex', knex) -fastify.decorate('cache', cache) fastify.decorate('query', query) fastify.decorate('dts', dts) fastify.decorate('geofence', geofence) @@ -177,39 +172,6 @@ async function syncDiscordRole() { setTimeout(syncDiscordRole, config.discord.checkRoleInterval * 3600000) } -async function saveEventCache() { - // eslint-disable-next-line no-underscore-dangle - fastify.cache._checkData(false) - return fsp.writeFile('.cache/webhook-events.json', JSON.stringify(fastify.cache.data), 'utf8') -} - -async function loadEventCache() { - let loaddatatxt - - try { - loaddatatxt = await fsp.readFile('.cache/webhook-events.json', 'utf8') - } catch { - return - } - - const now = Date.now() - - try { - const data = JSON.parse(loaddatatxt) - for (const key of Object.keys(data)) { - const msgData = data[key] - - if (msgData.t > now) { - const newTtlms = Math.max(msgData.t - now, 2000) - const newTtl = Math.floor(newTtlms / 1000) - fastify.cache.set(key, msgData.v, newTtl) - } - } - } catch (err) { - log.info(`Error processing historic cache ${err}`) - } -} - function handleShutdown() { const workerSaves = [] for (const worker of discordWorkers) { @@ -218,9 +180,6 @@ function handleShutdown() { if (telegram) workerSaves.push(telegram.saveTimeouts()) if (telegramChannel) workerSaves.push(telegramChannel.saveTimeouts()) if (discordWebhookWorker) workerSaves.push(discordWebhookWorker.saveTimeouts()) - if (config.general.persistDuplicateCache) { - workerSaves.push(saveEventCache()) - } Promise.all(workerSaves) .then(() => { @@ -231,14 +190,15 @@ function handleShutdown() { }) } +const cacheFactory = require('./lib/cache/cacheFactory') + async function run() { + const cache = cacheFactory.createCache(config, 5400) + fastify.decorate('cache', cache) + process.on('SIGINT', handleShutdown) process.on('SIGTERM', handleShutdown) - if (config.general.persistDuplicateCache) { - await loadEventCache() - } - if (config.discord.enabled) { setInterval(() => { if (!fastify.discordQueue.length) { @@ -541,8 +501,8 @@ async function processOne(hook) { } fastify.webhooks.info(`pokemon ${JSON.stringify(hook.message)}`) const verifiedSpawnTime = (hook.message.verified || hook.message.disappear_time_verified) - const cacheKey = `${hook.message.encounter_id}${verifiedSpawnTime ? 'T' : 'F'}${hook.message.cp}` - if (fastify.cache.get(cacheKey)) { + const cacheKey = `${hook.message.encounter_id}${verifiedSpawnTime ? 'T' : 'F'}${hook.message.cp || 'x'}` + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.encounter_id}: Wild encounter was sent again too soon, ignoring`) break } @@ -551,7 +511,7 @@ async function processOne(hook) { const secondsRemaining = !verifiedSpawnTime ? 3600 : (Math.max((hook.message.disappear_time * 1000 - Date.now()) / 1000, 0) + 300) - fastify.cache.set(cacheKey, 'x', secondsRemaining) + await fastify.cache.set(cacheKey, 'x', secondsRemaining) processHook = hook @@ -569,12 +529,12 @@ async function processOne(hook) { fastify.webhooks.info(`raid ${JSON.stringify(hook.message)}`) const cacheKey = `${hook.message.gym_id}${hook.message.end}${hook.message.pokemon_id}` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.gym_id}: Raid was sent again too soon, ignoring`) break } - fastify.cache.set(cacheKey, 'x') + await fastify.cache.set(cacheKey, 'x') processHook = hook @@ -596,7 +556,7 @@ async function processOne(hook) { if (lureExpiration && !config.general.disableLure) { const cacheKey = `${hook.message.pokestop_id}L${lureExpiration}` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.pokestop_id}: Lure was sent again too soon, ignoring`) break } @@ -604,13 +564,13 @@ async function processOne(hook) { // Set cache expiry to calculated invasion expiry time + 5 minutes to cope with near misses const secondsRemaining = Math.max((lureExpiration * 1000 - Date.now()) / 1000, 0) + 300 - fastify.cache.set(cacheKey, 'x', secondsRemaining) + await fastify.cache.set(cacheKey, 'x', secondsRemaining) processHook = hook } else if (!config.general.disableInvasion) { - const cacheKey = `${hook.message.pokestop_id}I${lureExpiration}` + const cacheKey = `${hook.message.pokestop_id}I` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.pokestop_id}: Invasion was sent again too soon, ignoring`) break } @@ -618,7 +578,7 @@ async function processOne(hook) { // Set cache expiry to calculated invasion expiry time + 5 minutes to cope with near misses const secondsRemaining = Math.max((incidentExpiration * 1000 - Date.now()) / 1000, 0) + 300 - fastify.cache.set(cacheKey, 'x', secondsRemaining) + await fastify.cache.set(cacheKey, 'x', secondsRemaining) processHook = hook } @@ -632,11 +592,11 @@ async function processOne(hook) { fastify.webhooks.info(`quest ${JSON.stringify(hook.message)}`) const cacheKey = `${hook.message.pokestop_id}_${JSON.stringify(hook.message.rewards)}` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.pokestop_id}: Quest was sent again too soon, ignoring`) break } - fastify.cache.set(cacheKey, 'x') + await fastify.cache.set(cacheKey, 'x') processHook = hook break @@ -648,7 +608,7 @@ async function processOne(hook) { } fastify.webhooks.info(`nest ${JSON.stringify(hook.message)}`) const cacheKey = `${hook.message.nest_id}_${hook.message.pokemon_id}_${hook.message.reset_time}` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.nest_id}: Nest was sent again too soon, ignoring`) break } @@ -656,7 +616,7 @@ async function processOne(hook) { // expiry time -- 14 days (!) after reset time const secondsRemaining = Math.max(((hook.message.reset_time + 14 * 24 * 60 * 60) * 1000 - Date.now()) / 1000, 0) - fastify.cache.set(cacheKey, 'x', secondsRemaining) + await fastify.cache.set(cacheKey, 'x', secondsRemaining) processHook = hook break @@ -672,11 +632,11 @@ async function processOne(hook) { const updateTimestamp = hook.message.time_changed || hook.message.updated const hookHourTimestamp = updateTimestamp - (updateTimestamp % 3600) const cacheKey = `${hook.message.s2_cell_id}_${hookHourTimestamp}` - if (fastify.cache.get(cacheKey)) { + if (await fastify.cache.get(cacheKey)) { fastify.controllerLog.debug(`${hook.message.s2_cell_id}: Weather for this cell was sent again too soon, ignoring`) break } - fastify.cache.set(cacheKey, 'x') + await fastify.cache.set(cacheKey, 'x') // post directly to weather controller weatherWorker.queuePort.postMessage(hook) diff --git a/src/lib/cache/cacheFactory.js b/src/lib/cache/cacheFactory.js new file mode 100644 index 000000000..abde55f69 --- /dev/null +++ b/src/lib/cache/cacheFactory.js @@ -0,0 +1,15 @@ +const PoracleRedisCache = require('./redisCache') +const PoracleNodeCache = require('./standardCache') + +function createCache(config, ttl) { + if (config.database.cache === 'redis') { + return new PoracleRedisCache(ttl, config.database.redis.host, + config.database.redis.port, config.database.redis.database, + config.database.redis.user, config.database.redis.password, + config.database.redis.prefix) + } + + return new PoracleNodeCache(ttl) +} + +exports.createCache = createCache \ No newline at end of file diff --git a/src/lib/cache/redisCache.js b/src/lib/cache/redisCache.js new file mode 100644 index 000000000..a3d4dc5e7 --- /dev/null +++ b/src/lib/cache/redisCache.js @@ -0,0 +1,40 @@ +const redis = require('redis') +const { promisify } = require('util') + +class PoracleRedisCache { + constructor(ttl, host, port, database, user, password, prefix) { + const options = { + host, + port, + } + if (user) options.user = user + if (password) options.password = password + if (database) options.db = database + if (prefix) options.prefix = prefix + + this.client = redis.createClient(options) + this.defaultTtl = ttl + this.hits = 0 + this.misses = 0 + } + + async get(key) { + const getAsync = promisify(this.client.get).bind(this.client) + + const val = await getAsync(key) + if (val) { this.hits++ } else { this.misses++ } + return val + } + + async set(key, value, ttl) { + const setexAsync = promisify(this.client.setex).bind(this.client) + + return setexAsync(key, Math.floor(ttl || this.defaultTtl), value) + } + + getStats() { + return { hits: this.hits, misses: this.misses, type: 'redis' } + } +} + +module.exports = PoracleRedisCache \ No newline at end of file diff --git a/src/lib/cache/standardCache.js b/src/lib/cache/standardCache.js new file mode 100644 index 000000000..036b0d83d --- /dev/null +++ b/src/lib/cache/standardCache.js @@ -0,0 +1,21 @@ +const NodeCache = require('node-cache') + +class PoracleNodeCache { + constructor(ttl) { + this.cache = new NodeCache({ stdTTL: ttl, useClones: false }) + } + + async get(key) { + return this.cache.get(key) + } + + async set(key, value, ttl) { + this.cache.set(key, value, ttl) + } + + getStats() { + return this.cache.getStats() + } +} + +module.exports = PoracleNodeCache \ No newline at end of file From 0a4c107e3c875158cd4706ee1ec5c8b7afc500e8 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 20 Jun 2021 10:57:31 +0100 Subject: [PATCH 2/3] Don't recreate commands all the time --- src/lib/cache/redisCache.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/cache/redisCache.js b/src/lib/cache/redisCache.js index a3d4dc5e7..2e02b511f 100644 --- a/src/lib/cache/redisCache.js +++ b/src/lib/cache/redisCache.js @@ -16,20 +16,20 @@ class PoracleRedisCache { this.defaultTtl = ttl this.hits = 0 this.misses = 0 + this.commands = { + getAsync: promisify(this.client.get).bind(this.client), + setexAsync: promisify(this.client.setex).bind(this.client), + } } async get(key) { - const getAsync = promisify(this.client.get).bind(this.client) - - const val = await getAsync(key) + const val = await this.commands.getAsync(key) if (val) { this.hits++ } else { this.misses++ } return val } async set(key, value, ttl) { - const setexAsync = promisify(this.client.setex).bind(this.client) - - return setexAsync(key, Math.floor(ttl || this.defaultTtl), value) + return this.commands.setexAsync(key, Math.floor(ttl || this.defaultTtl), value) } getStats() { From b1cb8fc001430ba56d18c5e51b50aa98a5015b7b Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 28 Dec 2021 11:36:13 +0000 Subject: [PATCH 3/3] Lint fix --- src/lib/cache/cacheFactory.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lib/cache/cacheFactory.js b/src/lib/cache/cacheFactory.js index abde55f69..aeb557f70 100644 --- a/src/lib/cache/cacheFactory.js +++ b/src/lib/cache/cacheFactory.js @@ -3,10 +3,15 @@ const PoracleNodeCache = require('./standardCache') function createCache(config, ttl) { if (config.database.cache === 'redis') { - return new PoracleRedisCache(ttl, config.database.redis.host, - config.database.redis.port, config.database.redis.database, - config.database.redis.user, config.database.redis.password, - config.database.redis.prefix) + return new PoracleRedisCache( + ttl, + config.database.redis.host, + config.database.redis.port, + config.database.redis.database, + config.database.redis.user, + config.database.redis.password, + config.database.redis.prefix, + ) } return new PoracleNodeCache(ttl)