From bda65f2c14bb2981d8903cab3e7c4e26a8634913 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sun, 22 Dec 2024 12:30:54 -0700 Subject: [PATCH 1/4] Add IP lookup service Using postgres put a lot of strain on the database because of the size of the dataset A new service has been introduced to manage downloading the database to persistent storage and exposing an internal api to interact with ip data If this is successful, then we can give up paying for the ip-api service A cron was also introduced to initiate downloading the database from maxmind on a recurring basis --- apps/cron-tasks/project.json | 8 +- apps/cron-tasks/src/config/env-config.ts | 4 + apps/cron-tasks/src/geo-ip-api-updater.ts | 46 +++++ ...geo-ip-updater.ts => geo-ip-db-updater.ts} | 3 + apps/geo-ip-api/.eslintrc.json | 18 ++ apps/geo-ip-api/.gitignore | 2 + apps/geo-ip-api/jest.config.ts | 9 + apps/geo-ip-api/project.json | 72 ++++++++ apps/geo-ip-api/src/downloads/.gitkeep | 0 apps/geo-ip-api/src/main.ts | 172 ++++++++++++++++++ apps/geo-ip-api/src/maxmind.service.ts | 132 ++++++++++++++ apps/geo-ip-api/src/route.utils.ts | 68 +++++++ apps/geo-ip-api/tsconfig.app.json | 14 ++ apps/geo-ip-api/tsconfig.json | 16 ++ apps/geo-ip-api/tsconfig.spec.json | 9 + libs/api-config/src/lib/api-logger.ts | 2 +- libs/api-config/src/lib/api-telemetry.ts | 2 +- libs/api-config/src/lib/env-config.ts | 18 +- nx.json | 5 + package.json | 9 +- yarn.lock | 84 ++++++++- 21 files changed, 683 insertions(+), 10 deletions(-) create mode 100644 apps/cron-tasks/src/geo-ip-api-updater.ts rename apps/cron-tasks/src/{geo-ip-updater.ts => geo-ip-db-updater.ts} (98%) create mode 100644 apps/geo-ip-api/.eslintrc.json create mode 100644 apps/geo-ip-api/.gitignore create mode 100644 apps/geo-ip-api/jest.config.ts create mode 100644 apps/geo-ip-api/project.json create mode 100644 apps/geo-ip-api/src/downloads/.gitkeep create mode 100644 apps/geo-ip-api/src/main.ts create mode 100644 apps/geo-ip-api/src/maxmind.service.ts create mode 100644 apps/geo-ip-api/src/route.utils.ts create mode 100644 apps/geo-ip-api/tsconfig.app.json create mode 100644 apps/geo-ip-api/tsconfig.json create mode 100644 apps/geo-ip-api/tsconfig.spec.json diff --git a/apps/cron-tasks/project.json b/apps/cron-tasks/project.json index 7233404d2..7ff9e31dd 100644 --- a/apps/cron-tasks/project.json +++ b/apps/cron-tasks/project.json @@ -16,8 +16,12 @@ "entryPath": "apps/cron-tasks/src/save-analytics-summary.ts" }, { - "entryName": "geo-ip-updater", - "entryPath": "apps/cron-tasks/src/geo-ip-updater.ts" + "entryName": "geo-ip-api-updater", + "entryPath": "apps/cron-tasks/src/geo-ip-api-updater.ts" + }, + { + "entryName": "geo-ip-db-updater", + "entryPath": "apps/cron-tasks/src/geo-ip-db-updater.ts" } ], "tsConfig": "apps/cron-tasks/tsconfig.app.json", diff --git a/apps/cron-tasks/src/config/env-config.ts b/apps/cron-tasks/src/config/env-config.ts index b9d79d54f..262684707 100644 --- a/apps/cron-tasks/src/config/env-config.ts +++ b/apps/cron-tasks/src/config/env-config.ts @@ -31,4 +31,8 @@ export const ENV = { MAX_MIND_ACCOUNT_ID: process.env.MAX_MIND_ACCOUNT_ID, MAX_MIND_LICENSE_KEY: process.env.MAX_MIND_LICENSE_KEY, + + GEO_IP_API_HOSTNAME: process.env.GEO_IP_API_HOSTNAME, + GEO_IP_API_USERNAME: process.env.MAX_MIND_ACCOUNT_ID, + GEO_IP_API_PASSWORD: process.env.MAX_MIND_LICENSE_KEY, }; diff --git a/apps/cron-tasks/src/geo-ip-api-updater.ts b/apps/cron-tasks/src/geo-ip-api-updater.ts new file mode 100644 index 000000000..7c2f3232f --- /dev/null +++ b/apps/cron-tasks/src/geo-ip-api-updater.ts @@ -0,0 +1,46 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { ENV } from './config/env-config'; +import { logger } from './config/logger.config'; +import { getExceptionLog } from './utils/utils'; + +const GEO_IP_API_HOSTNAME = ENV.GEO_IP_API_HOSTNAME!; +const GEO_IP_API_USERNAME = ENV.GEO_IP_API_USERNAME!; +const GEO_IP_API_PASSWORD = ENV.GEO_IP_API_PASSWORD!; + +if (!GEO_IP_API_HOSTNAME) { + logger.error('GEO_IP_API_HOSTNAME environment variable is not set'); + process.exit(1); +} +if (!GEO_IP_API_USERNAME) { + logger.error('GEO_IP_API_USERNAME environment variable is not set'); + process.exit(1); +} +if (!GEO_IP_API_PASSWORD) { + logger.error('GEO_IP_API_PASSWORD environment variable is not set'); + process.exit(1); +} + +async function initiateDownload() { + const response = await fetch(GEO_IP_API_HOSTNAME, { + method: 'POST', + headers: { + Authorization: `Basic ${Buffer.from(`${GEO_IP_API_USERNAME}:${GEO_IP_API_PASSWORD}`).toString('base64')}`, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to download: ${response.statusText}`); + } + + return response.json(); +} + +async function main() { + const results = await initiateDownload(); + logger.info(results, 'Download completed successfully'); +} + +main().catch((error) => { + logger.error(getExceptionLog(error), 'Fatal error: %s', error.message); + process.exit(1); +}); diff --git a/apps/cron-tasks/src/geo-ip-updater.ts b/apps/cron-tasks/src/geo-ip-db-updater.ts similarity index 98% rename from apps/cron-tasks/src/geo-ip-updater.ts rename to apps/cron-tasks/src/geo-ip-db-updater.ts index 18b870f1c..98d7a5a5a 100644 --- a/apps/cron-tasks/src/geo-ip-updater.ts +++ b/apps/cron-tasks/src/geo-ip-db-updater.ts @@ -8,6 +8,9 @@ import { logger } from './config/logger.config'; import { getExceptionLog } from './utils/utils'; /** + +NOTICE: this is no longer used in production as put too much strain on the database server + CREATE TABLE IF NOT EXISTS geo_ip.network ( network cidr NOT NULL, geoname_id int, diff --git a/apps/geo-ip-api/.eslintrc.json b/apps/geo-ip-api/.eslintrc.json new file mode 100644 index 000000000..9d9c0db55 --- /dev/null +++ b/apps/geo-ip-api/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/apps/geo-ip-api/.gitignore b/apps/geo-ip-api/.gitignore new file mode 100644 index 000000000..2bc992caf --- /dev/null +++ b/apps/geo-ip-api/.gitignore @@ -0,0 +1,2 @@ +src/downloads/* +!src/downloads/.gitkeep diff --git a/apps/geo-ip-api/jest.config.ts b/apps/geo-ip-api/jest.config.ts new file mode 100644 index 000000000..f3741373c --- /dev/null +++ b/apps/geo-ip-api/jest.config.ts @@ -0,0 +1,9 @@ +export default { + displayName: 'geo-ip-api', + preset: '../../jest.preset.js', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'js', 'html'], +}; diff --git a/apps/geo-ip-api/project.json b/apps/geo-ip-api/project.json new file mode 100644 index 000000000..feae30de6 --- /dev/null +++ b/apps/geo-ip-api/project.json @@ -0,0 +1,72 @@ +{ + "name": "geo-ip-api", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "apps/geo-ip-api/src", + "projectType": "application", + "tags": ["scope:server"], + "targets": { + "build": { + "executor": "@nx/esbuild:esbuild", + "outputs": ["{options.outputPath}"], + "defaultConfiguration": "production", + "options": { + "platform": "node", + "format": ["cjs"], + "bundle": true, + "outputPath": "dist/apps/geo-ip-api", + "main": "apps/geo-ip-api/src/main.ts", + "tsConfig": "apps/geo-ip-api/tsconfig.app.json", + "assets": [ + { + "glob": "**/*", + "input": "apps/geo-ip-api/src/downloads", + "output": "downloads", + "ignore": [".gitkeep"] + } + ], + "generatePackageJson": true, + "sourcemap": true, + "esbuildOptions": { + "sourcemap": true, + "outExtension": { + ".js": ".js" + } + } + }, + "configurations": { + "development": { + "inspect": true + }, + "production": {} + } + }, + "serve": { + "executor": "@nx/js:node", + "defaultConfiguration": "development", + "options": { + "buildTarget": "geo-ip-api:build", + "inspect": "inspect", + "port": 7778 + }, + "configurations": { + "development": { + "buildTarget": "geo-ip-api:build:development" + }, + "production": { + "buildTarget": "geo-ip-api:build:production" + } + } + }, + "lint": { + "executor": "@nx/eslint:lint", + "outputs": ["{options.outputFile}"] + }, + "test": { + "executor": "@nx/jest:jest", + "outputs": ["{workspaceRoot}/coverage/{projectRoot}"], + "options": { + "jestConfig": "apps/geo-ip-api/jest.config.ts" + } + } + } +} diff --git a/apps/geo-ip-api/src/downloads/.gitkeep b/apps/geo-ip-api/src/downloads/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/apps/geo-ip-api/src/main.ts b/apps/geo-ip-api/src/main.ts new file mode 100644 index 000000000..8b13f6604 --- /dev/null +++ b/apps/geo-ip-api/src/main.ts @@ -0,0 +1,172 @@ +import { ENV, getExceptionLog, httpLogger, logger } from '@jetstream/api-config'; +import { json, urlencoded } from 'body-parser'; +import express from 'express'; +import { z, ZodError } from 'zod'; +import { downloadMaxMindDb, initMaxMind, lookupIpAddress, validateIpAddress } from './maxmind.service'; +import { createRoute } from './route.utils'; + +const DISK_PATH = process.env.DISK_PATH ?? __dirname; + +if (ENV.ENVIRONMENT !== 'development' && (!ENV.GEO_IP_API_USERNAME || !ENV.GEO_IP_API_PASSWORD)) { + logger.error('GEO_IP_API_USERNAME/GEO_IP_API_PASSWORD environment variables are not set'); + process.exit(1); +} + +const app = express(); + +app.use(json({ limit: '20mb' })); +app.use(urlencoded({ extended: true })); + +app.use(httpLogger); + +app.use('/healthz', (req, res) => { + res.status(200).json({ + error: false, + uptime: process.uptime(), + message: 'Healthy', + }); +}); + +app.use((req, res, next) => { + try { + const authHeader = req.headers.authorization; + if (typeof authHeader !== 'string') { + throw new Error('Unauthorized'); + } + const [type, token] = authHeader.split(' '); + if (type !== 'Basic') { + throw new Error('Unauthorized'); + } + const [username, password] = Buffer.from(token, 'base64').toString().split(':'); + if (username !== ENV.GEO_IP_API_USERNAME || password !== ENV.GEO_IP_API_PASSWORD) { + throw new Error('Unauthorized'); + } + next(); + } catch (ex) { + res.header('WWW-Authenticate', 'Basic realm="Geo IP API"'); + res.status(401).json({ + success: false, + message: 'Unauthorized', + }); + } +}); + +app.post( + '/api/download', + createRoute({}, async (_, req, res, next) => { + try { + const startTime = Date.now(); + await downloadMaxMindDb(DISK_PATH); + const timeTaken = Date.now() - startTime; + res.status(200).json({ + success: true, + message: 'MaxMind database downloaded', + timeTaken, + }); + } catch (ex) { + res.log.error(getExceptionLog(ex, true), 'Failed to download MaxMind database'); + next(ex); + } + }) +); + +/** + * Lookup a single IP address + */ +app.get( + '/api/lookup', + createRoute({ query: z.object({ ip: z.string() }) }, async ({ query }, req, res, next) => { + try { + const ipAddress = query.ip; + await initMaxMind(DISK_PATH); + if (!validateIpAddress(ipAddress)) { + res.status(400).json({ success: false, message: 'IP address is invalid' }); + return; + } + const results = lookupIpAddress(ipAddress); + res.status(200).json({ success: true, results }); + } catch (ex) { + res.log.error(getExceptionLog(ex, true), 'Failed to lookup IP address'); + next(ex); + } + }) +); + +/** + * Lookup multiple IP addresses + */ +app.post( + '/api/lookup', + createRoute({ body: z.object({ ips: z.string().array() }) }, async ({ body }, req, res, next) => { + try { + const ipAddresses = body.ips; + await initMaxMind(DISK_PATH); + + const results = ipAddresses.map((ipAddress) => { + const isValid = validateIpAddress(ipAddress); + return { + ipAddress, + isValid, + results: isValid ? lookupIpAddress(ipAddress) : null, + }; + }); + + res.status(200).json({ success: true, results }); + } catch (ex) { + res.log.error(getExceptionLog(ex, true), 'Failed to lookup IP address'); + next(ex); + } + }) +); + +app.use('*', (req, res, next) => { + res.status(404).json({ + success: false, + message: 'Not found', + }); +}); + +app.use((err: Error | ZodError, req: express.Request, res: express.Response, next: express.NextFunction) => { + res.log.error('Unhandled error:', err); + + if (!res.statusCode) { + res.status(500); + } + + if (err instanceof ZodError) { + res.json({ + success: false, + message: 'Validation error', + details: err.errors, + }); + return; + } + + res.json({ + success: false, + message: err.message, + }); +}); + +const port = Number(process.env.PORT || 3334); + +const server = app.listen(port, () => { + logger.info(`Listening at http://localhost:${port}/api`); +}); +server.on('error', (error) => { + logger.error(getExceptionLog(error, true), 'Server error: %s', error.message); +}); + +process.on('SIGTERM', () => { + logger.info('SIGTERM received, shutting down gracefully'); + server.close(() => { + logger.info('Server closed'); + process.exit(0); + }); + + // Force close after 30s + setTimeout(() => { + logger.error('Could not close connections in time, forcefully shutting down'); + process.exit(1); + }, 30_000); +}); diff --git a/apps/geo-ip-api/src/maxmind.service.ts b/apps/geo-ip-api/src/maxmind.service.ts new file mode 100644 index 000000000..249ffb394 --- /dev/null +++ b/apps/geo-ip-api/src/maxmind.service.ts @@ -0,0 +1,132 @@ +import { logger } from '@jetstream/api-config'; +import fs from 'fs'; +import maxMind, { CityResponse, Reader } from 'maxmind'; +import path from 'path'; +import * as tar from 'tar'; +import { promisify } from 'util'; + +const existsAsync = promisify(fs.exists); +const writeFileAsync = promisify(fs.writeFile); +const statAsync = promisify(fs.stat); +const mkdirAsync = promisify(fs.mkdir); + +// const ASN_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-ASN/download?suffix=tar.gz'; +// const ASN_FILENAME = 'GeoLite2-ASN.tar.gz'; + +const FOLDER_NAME = 'downloads'; +const CITY_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz'; +const CITY_ZIP_FILENAME = 'GeoLite2-City.tar.gz'; +const CITY_DB_FILENAME = 'GeoLite2-City.mmdb'; + +const downloads = [ + // { url: ASN_URL, filename: ASN_FILENAME }, + { url: CITY_URL, archiveFilename: CITY_ZIP_FILENAME, dbFileName: CITY_DB_FILENAME, defaultLookup: true }, +]; + +let lookup: Reader; + +export async function initMaxMind(rootDir: string, force = false) { + if (force || !lookup) { + const filePath = path.join(rootDir, FOLDER_NAME, CITY_DB_FILENAME); + logger.info(`Initializing database: ${filePath}`); + lookup = await maxMind.open(filePath); + } +} + +export async function downloadMaxMindDb(rootDir: string): Promise { + const downloadFolderPath = path.join(rootDir, FOLDER_NAME); + logger.info(`Downloading MaxMind DB to ${downloadFolderPath}`); + + // Create downloads directory if it doesn't exist + if (!(await existsAsync(downloadFolderPath))) { + await mkdirAsync(downloadFolderPath, { recursive: true }); + } + + for (const { defaultLookup, archiveFilename, url } of downloads) { + const archiveFilePath = path.join(downloadFolderPath, archiveFilename); + + // Check if file needs to be downloaded + let needsDownload = true; + if (await existsAsync(archiveFilePath)) { + const stats = await statAsync(archiveFilePath); + const fileAge = Date.now() - stats.atime.getTime(); + const oneDayInMs = 24 * 60 * 60 * 1000; + + if (fileAge < oneDayInMs) { + needsDownload = false; + } + } + + if (needsDownload) { + logger.info(`Fetching from server: ${url}`); + const response = await fetch(url, { + headers: { + Authorization: `Basic ${Buffer.from(`${process.env.MAX_MIND_ACCOUNT_ID}:${process.env.MAX_MIND_LICENSE_KEY}`).toString( + 'base64' + )}`, + }, + }); + + if (!response.ok) { + throw new Error(`Failed to download ${archiveFilePath}: ${response.statusText}`); + } + + const buffer = Buffer.from(await response.arrayBuffer()); + await writeFileAsync(archiveFilePath, buffer); + } else { + logger.info(`File already exists, skipping download: ${url}`); + } + + // Extract .mmdb files from tar.gz + await tar.x({ + file: archiveFilePath, + cwd: downloadFolderPath, + filter: (path) => path.endsWith('.mmdb'), + strip: 1, // Remove the first directory component from paths + }); + + if (defaultLookup) { + initMaxMind(downloadFolderPath, true); + } + } +} + +export function lookupIpAddress(ipAddress: string) { + if (!lookup) { + throw new Error('MaxMind DB not initialized'); + } + + const results = lookup.get(ipAddress); + if (!results) { + return null; + } + return { + city: results.city?.names?.en ?? null, + country: results.country?.names?.en ?? null, + countryISO: results.country?.iso_code ?? null, + isEU: !!results.country?.is_in_european_union, + continent: results.continent?.names?.en ?? null, + location: results.location ?? null, + postalCode: results.postal?.code ?? null, + registeredCountry: results.registered_country + ? { + country: results.registered_country.names.en, + iso: results.registered_country.iso_code, + isEU: !!results.registered_country.is_in_european_union, + } + : null, + subdivisions: + results.subdivisions?.map((item) => ({ + name: item.names.en, + iso: item.iso_code, + })) ?? [], + foo: results.traits, + }; +} + +export function validateIpAddress(ipAddress: unknown): boolean { + if (typeof ipAddress !== 'string') { + return false; + } + return maxMind.validate(ipAddress); +} diff --git a/apps/geo-ip-api/src/route.utils.ts b/apps/geo-ip-api/src/route.utils.ts new file mode 100644 index 000000000..0339b88ea --- /dev/null +++ b/apps/geo-ip-api/src/route.utils.ts @@ -0,0 +1,68 @@ +import { getExceptionLog, rollbarServer } from '@jetstream/api-config'; +import type { Request as ExpressRequest, Response as ExpressResponse } from 'express'; +import { NextFunction } from 'express'; +import type pino from 'pino'; +import { z } from 'zod'; + +export type Request< + Params extends Record | unknown = Record, + ReqBody = unknown, + Query extends Record | unknown = Record +> = ExpressRequest & { log: pino.Logger }; + +export type Response = ExpressResponse & { log: pino.Logger }; + +export type ControllerFunction = ( + data: { + params: z.infer; + body: z.infer; + query: z.infer; + }, + req: Request, + res: Response, + next: NextFunction +) => Promise | void; + +export function createRoute( + { + params, + body, + query, + }: { + params?: TParamsSchema; + body?: TBodySchema; + query?: TQuerySchema; + }, + controllerFn: ControllerFunction +) { + return async (req: Request, res: Response, next: NextFunction) => { + try { + const data = { + params: params ? params.parse(req.params) : undefined, + body: body ? body.parse(req.body) : undefined, + query: query ? query.parse(req.query) : undefined, + }; + try { + await controllerFn(data, req, res, next); + } catch (ex) { + next(ex); + } + } catch (ex) { + rollbarServer.error('Route Validation Error', req, { + context: `route#createRoute`, + custom: { + ...getExceptionLog(ex, true), + message: ex.message, + stack: ex.stack, + url: req.url, + params: req.params, + query: req.query, + body: req.body, + }, + }); + req.log.error(getExceptionLog(ex), '[ROUTE][VALIDATION ERROR]'); + res.status(400); + next(ex); + } + }; +} diff --git a/apps/geo-ip-api/tsconfig.app.json b/apps/geo-ip-api/tsconfig.app.json new file mode 100644 index 000000000..cff0d8907 --- /dev/null +++ b/apps/geo-ip-api/tsconfig.app.json @@ -0,0 +1,14 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "esnext", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "moduleResolution": "node", + "strictNullChecks": true, + "outDir": "../../dist/out-tsc", + "types": ["node", "express", "google.accounts", "google.picker", "gapi.auth2"] + }, + "include": ["src/**/*.ts"], + "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/apps/geo-ip-api/tsconfig.json b/apps/geo-ip-api/tsconfig.json new file mode 100644 index 000000000..c1e2dd4e8 --- /dev/null +++ b/apps/geo-ip-api/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.app.json" + }, + { + "path": "./tsconfig.spec.json" + } + ], + "compilerOptions": { + "esModuleInterop": true + } +} diff --git a/apps/geo-ip-api/tsconfig.spec.json b/apps/geo-ip-api/tsconfig.spec.json new file mode 100644 index 000000000..43ab3b2a4 --- /dev/null +++ b/apps/geo-ip-api/tsconfig.spec.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node", "google.accounts", "google.picker", "gapi.auth2"] + }, + "include": ["jest.config.ts", "src/**/*.test.ts", "src/**/*.spec.ts", "src/**/*.d.ts"] +} diff --git a/libs/api-config/src/lib/api-logger.ts b/libs/api-config/src/lib/api-logger.ts index e80edee1c..3c95e75ed 100644 --- a/libs/api-config/src/lib/api-logger.ts +++ b/libs/api-config/src/lib/api-logger.ts @@ -14,7 +14,7 @@ export const logger = pino({ : undefined, }); -const ignoreLogsFileExtensions = /.*\.(js|map|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|otf|json)$/; +const ignoreLogsFileExtensions = /.*\.(js|map|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|otf|json|xml|txt)$/; export const httpLogger = pinoHttp({ logger, diff --git a/libs/api-config/src/lib/api-telemetry.ts b/libs/api-config/src/lib/api-telemetry.ts index ee4033b72..360844da3 100644 --- a/libs/api-config/src/lib/api-telemetry.ts +++ b/libs/api-config/src/lib/api-telemetry.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import { credentials, Metadata } from '@grpc/grpc-js'; -import { UserProfileSession } from '@jetstream/auth/types'; +import type { UserProfileSession } from '@jetstream/auth/types'; import telemetryApi from '@opentelemetry/api'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; diff --git a/libs/api-config/src/lib/env-config.ts b/libs/api-config/src/lib/env-config.ts index dfbc5605e..e9db73993 100644 --- a/libs/api-config/src/lib/env-config.ts +++ b/libs/api-config/src/lib/env-config.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ -import { UserProfileSession, UserProfileUiWithIdentities } from '@jetstream/auth/types'; -import { ensureBoolean } from '@jetstream/shared/utils'; +import type { UserProfileSession, UserProfileUiWithIdentities } from '@jetstream/auth/types'; +import type { Maybe } from '@jetstream/types'; import chalk from 'chalk'; import * as dotenv from 'dotenv'; import { readFileSync } from 'fs-extra'; @@ -17,6 +17,15 @@ try { // ignore errors } +function ensureBoolean(value: Maybe): boolean { + if (typeof value === 'boolean') { + return value; + } else if (typeof value === 'string') { + return value.toLowerCase().startsWith('t'); + } + return false; +} + /** * This object allows for someone to run Jetstream in a local environment * an bypass authentication - this is useful for running locally. @@ -192,6 +201,11 @@ const envSchema = z.object({ */ HONEYCOMB_ENABLED: booleanSchema, HONEYCOMB_API_KEY: z.string().optional(), + /** + * GEO-IP API (private service basic auth) + */ + GEO_IP_API_USERNAME: z.string().nullish(), + GEO_IP_API_PASSWORD: z.string().nullish(), }); const parseResults = envSchema.safeParse({ diff --git a/nx.json b/nx.json index f31e6d2f1..6dc0abd17 100644 --- a/nx.json +++ b/nx.json @@ -135,6 +135,11 @@ "cache": true, "dependsOn": ["^build"], "inputs": ["production", "^production"] + }, + "@nx/webpack:webpack": { + "cache": true, + "dependsOn": ["^build"], + "inputs": ["production", "^production"] } }, "namedInputs": { diff --git a/package.json b/package.json index 76059bd68..a3a2c5001 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "nx": "nx", "start": "nx serve", "start:api": "nx serve api", + "start:geo-ip-api": "nx serve geo-ip-api", "start:worker": "nx serve jetstream-worker", "start:landing": "nx dev landing", "start:docs": "cd apps/docs && yarn start", @@ -26,7 +27,7 @@ "start:e2e": "node dist/apps/api/main.js", "start:web-extension": "nx run jetstream-web-extension:serve", "start:cron:save-analytics-summary": "node dist/apps/cron-tasks/save-analytics-summary.js", - "start:geo-ip-db-update": "node dist/apps/cron-tasks/geo-ip-updater.js", + "start:cron:geo-ip-api-updater": "node dist/apps/cron-tasks/geo-ip-api-updater.js", "build": "cross-env NODE_ENV=production npm-run-all db:generate build:core build:landing generate:version", "build:pre-deploy": "cross-env NODE_ENV=production npm-run-all --parallel rollbar:upload-sourcemaps db:migrate", "build:core": "NODE_OPTIONS=--max_old_space_size=8192 nx run-many --output-style=dynamic --target=build --parallel=3 --projects=jetstream,api,download-zip-sw --configuration=production", @@ -42,6 +43,7 @@ "build:ui:test": "nx build ui --configuration=test", "build:api": "nx build api --prod", "build:api:docker": "nx build api --configuration=docker", + "build:geo-ip-api": "nx build geo-ip-api --prod", "build:cron": "yarn db:generate && nx build cron-tasks --prod && yarn generate:version", "build:api:test": "nx build api --configuration=test", "build:sw": "nx build download-zip-sw --prod", @@ -164,6 +166,8 @@ "@types/react-router-dom": "5.3.3", "@types/react-transition-group": "^4.4.9", "@types/sass": "^1.45.0", + "@types/tar": "^6.1.13", + "@types/unzipper": "^0.10.10", "@types/uuid": "^9.0.8", "@types/webpack": "4.41.21", "@typescript-eslint/eslint-plugin": "7.18.0", @@ -265,7 +269,6 @@ "@swc/helpers": "0.5.11", "@tanstack/react-virtual": "^3.4.0", "@tippyjs/react": "^4.2.6", - "@types/unzipper": "^0.10.10", "amplitude-js": "^8.21.9", "axios": "^1.7.7", "bcrypt": "^5.1.1", @@ -309,6 +312,7 @@ "localforage": "^1.10.0", "lodash": "^4.17.21", "mailgun.js": "^8.2.1", + "maxmind": "^4.3.23", "monaco-editor": "^0.48.0", "multer": "^1.4.5-lts.1", "nanoid": "^3.1.20", @@ -347,6 +351,7 @@ "socket.io": "^4.8.1", "socket.io-client": "^4.8.1", "split.js": "^1.6.5", + "tar": "^7.4.3", "tiny-request-router": "^1.2.2", "tslib": "^2.3.0", "unzipper": "^0.12.3", diff --git a/yarn.lock b/yarn.lock index 22161b8f7..a3061dd00 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5060,6 +5060,13 @@ wrap-ansi "^8.1.0" wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" +"@isaacs/fs-minipass@^4.0.0": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz#2d59ae3ab4b38fb4270bfa23d30f8e2e86c7fe32" + integrity sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w== + dependencies: + minipass "^7.0.4" + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz" @@ -9575,6 +9582,14 @@ dependencies: tapable "^2.2.0" +"@types/tar@^6.1.13": + version "6.1.13" + resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.13.tgz#9b5801c02175344101b4b91086ab2bbc8e93a9b6" + integrity sha512-IznnlmU5f4WcGTh2ltRu/Ijpmk8wiWXfF0VA4s+HPjHZgvFggk1YaIkbo5krX/zUCzWF8N/l4+W/LNxnvAJ8nw== + dependencies: + "@types/node" "*" + minipass "^4.0.0" + "@types/tedious@^4.0.14": version "4.0.14" resolved "https://registry.yarnpkg.com/@types/tedious/-/tedious-4.0.14.tgz#868118e7a67808258c05158e9cad89ca58a2aec1" @@ -11807,6 +11822,11 @@ chownr@^2.0.0: resolved "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz" integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== +chownr@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-3.0.0.tgz#9855e64ecd240a9cc4267ce8a4aa5d24a1da15e4" + integrity sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g== + chrome-trace-event@^1.0.2: version "1.0.3" resolved "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz" @@ -15760,7 +15780,7 @@ glob@^10.3.10: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" -glob@^10.3.3, glob@^10.4.5: +glob@^10.3.3, glob@^10.3.7, glob@^10.4.5: version "10.4.5" resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.5.tgz#f4d9f0b90ffdbab09c9d77f5f29b4262517b0956" integrity sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg== @@ -18998,6 +19018,14 @@ marked@7.0.4: resolved "https://registry.yarnpkg.com/marked/-/marked-7.0.4.tgz#e2558ee2d535b9df6a27c6e282dc603a18388a6d" integrity sha512-t8eP0dXRJMtMvBojtkcsA7n48BkauktUKzfkPSCq85ZMTJ0v76Rke4DYz01omYpPTUh4p/f7HePgRo3ebG8+QQ== +maxmind@^4.3.23: + version "4.3.23" + resolved "https://registry.yarnpkg.com/maxmind/-/maxmind-4.3.23.tgz#e6920149c6104cdf0272d378ce3adf9837dc98f5" + integrity sha512-AMm4Eem0J0Y1EQJRVSdi2xevw5bJgUDd+lHyQwu0PvGUtK/4uOb8/uidmsrRZ/ST90UfF48H4ShAeFFWKvZ7bw== + dependencies: + mmdb-lib "2.1.1" + tiny-lru "11.2.11" + md-to-react-email@5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/md-to-react-email/-/md-to-react-email-5.0.2.tgz#2ee848a7248d4df6e6a95466a269ca6b6697a704" @@ -19230,6 +19258,11 @@ minipass@^3.0.0: dependencies: yallist "^4.0.0" +minipass@^4.0.0: + version "4.2.8" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-4.2.8.tgz#f0010f64393ecfc1d1ccb5f582bcaf45f48e1a3a" + integrity sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ== + minipass@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/minipass/-/minipass-5.0.0.tgz#3e9788ffb90b694a5d0ec94479a45b5d8738133d" @@ -19240,7 +19273,7 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== -minipass@^7.1.2: +minipass@^7.0.4, minipass@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== @@ -19253,6 +19286,14 @@ minizlib@^2.1.1: minipass "^3.0.0" yallist "^4.0.0" +minizlib@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-3.0.1.tgz#46d5329d1eb3c83924eff1d3b858ca0a31581012" + integrity sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg== + dependencies: + minipass "^7.0.4" + rimraf "^5.0.5" + mkdirp@^0.5.4, mkdirp@^0.5.5: version "0.5.6" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz" @@ -19265,6 +19306,11 @@ mkdirp@^1.0.3: resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mkdirp@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-3.0.1.tgz#e44e4c5607fb279c168241713cc6e0fea9adcb50" + integrity sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg== + mlly@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/mlly/-/mlly-1.3.0.tgz#3184cb80c6437bda861a9f452ae74e3434ed9cd1" @@ -19285,6 +19331,11 @@ mlly@^1.4.2: pkg-types "^1.0.3" ufo "^1.3.2" +mmdb-lib@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/mmdb-lib/-/mmdb-lib-2.1.1.tgz#c0d0bd35dc1fca41f0ebd043e43227ab04eb1792" + integrity sha512-yx8H/1H5AfnufiLnzzPqPf4yr/dKU9IFT1rPVwSkrKWHsQEeVVd6+X+L0nUbXhlEFTu3y/7hu38CFmEVgzvyeg== + module-details-from-path@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" @@ -22672,6 +22723,13 @@ rimraf@^3.0.0, rimraf@^3.0.2: dependencies: glob "^7.1.3" +rimraf@^5.0.5: + version "5.0.10" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.10.tgz#23b9843d3dc92db71f96e1a2ce92e39fd2a8221c" + integrity sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ== + dependencies: + glob "^10.3.7" + rimraf@~2.6.2: version "2.6.3" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" @@ -24232,6 +24290,18 @@ tar@^6.1.13: mkdirp "^1.0.3" yallist "^4.0.0" +tar@^7.4.3: + version "7.4.3" + resolved "https://registry.yarnpkg.com/tar/-/tar-7.4.3.tgz#88bbe9286a3fcd900e94592cda7a22b192e80571" + integrity sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw== + dependencies: + "@isaacs/fs-minipass" "^4.0.0" + chownr "^3.0.0" + minipass "^7.1.2" + minizlib "^3.0.1" + mkdirp "^3.0.1" + yallist "^5.0.0" + telejson@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/telejson/-/telejson-7.2.0.tgz#3994f6c9a8f8d7f2dba9be2c7c5bbb447e876f32" @@ -24371,6 +24441,11 @@ tiny-invariant@^1.3.3: resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127" integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg== +tiny-lru@11.2.11: + version "11.2.11" + resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-11.2.11.tgz#5089a6a4a157f5a97b82aa930b44d550ac5c4778" + integrity sha512-27BIW0dIWTYYoWNnqSmoNMKe5WIbkXsc0xaCQHd3/3xT2XMuMJrzHdrO9QBFR14emBz1Bu0dOAs2sCBBrvgPQA== + tiny-lru@^8.0.1: version "8.0.2" resolved "https://registry.npmjs.org/tiny-lru/-/tiny-lru-8.0.2.tgz" @@ -25862,6 +25937,11 @@ yallist@^4.0.0: resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== +yallist@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533" + integrity sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw== + yaml@^1.10.0, yaml@^1.7.2: version "1.10.2" resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" From 93d77212e592a14bb422ef00410ac70cb239e063 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sun, 22 Dec 2024 12:48:10 -0700 Subject: [PATCH 2/4] Add hostname env var --- libs/api-config/src/lib/env-config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/api-config/src/lib/env-config.ts b/libs/api-config/src/lib/env-config.ts index e9db73993..c1af84268 100644 --- a/libs/api-config/src/lib/env-config.ts +++ b/libs/api-config/src/lib/env-config.ts @@ -206,6 +206,7 @@ const envSchema = z.object({ */ GEO_IP_API_USERNAME: z.string().nullish(), GEO_IP_API_PASSWORD: z.string().nullish(), + GEO_IP_API_HOSTNAME: z.string().nullish(), }); const parseResults = envSchema.safeParse({ From c6bb45eb48509dfa90a329a4ca54822843d08189 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sun, 22 Dec 2024 13:12:13 -0700 Subject: [PATCH 3/4] add prod start command --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index a3a2c5001..f77450269 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "start": "nx serve", "start:api": "nx serve api", "start:geo-ip-api": "nx serve geo-ip-api", + "start:geo-ip-api:prod": "node dist/apps/geo-ip-api/main.js", "start:worker": "nx serve jetstream-worker", "start:landing": "nx dev landing", "start:docs": "cd apps/docs && yarn start", From 0800bbb23ec883d26b44a6a57bda32fd57836ee4 Mon Sep 17 00:00:00 2001 From: Austin Turner Date: Sun, 22 Dec 2024 14:59:54 -0700 Subject: [PATCH 4/4] Adjust API format and wire up to Jetstream session api Based on env var, call new service instead of old service Output format was modified to match prior data structure to make the migration seamless --- apps/geo-ip-api/src/main.ts | 2 +- apps/geo-ip-api/src/maxmind.service.ts | 82 +++++++++++---------- libs/api-config/src/lib/env-config.ts | 7 +- libs/auth/server/src/lib/auth.db.service.ts | 34 ++++++--- libs/auth/types/src/lib/auth-types.ts | 14 ++-- 5 files changed, 78 insertions(+), 61 deletions(-) diff --git a/apps/geo-ip-api/src/main.ts b/apps/geo-ip-api/src/main.ts index 8b13f6604..b9bde4d1c 100644 --- a/apps/geo-ip-api/src/main.ts +++ b/apps/geo-ip-api/src/main.ts @@ -107,7 +107,7 @@ app.post( return { ipAddress, isValid, - results: isValid ? lookupIpAddress(ipAddress) : null, + ...(isValid ? lookupIpAddress(ipAddress) : null), }; }); diff --git a/apps/geo-ip-api/src/maxmind.service.ts b/apps/geo-ip-api/src/maxmind.service.ts index 249ffb394..fdac0fde5 100644 --- a/apps/geo-ip-api/src/maxmind.service.ts +++ b/apps/geo-ip-api/src/maxmind.service.ts @@ -1,6 +1,6 @@ import { logger } from '@jetstream/api-config'; import fs from 'fs'; -import maxMind, { CityResponse, Reader } from 'maxmind'; +import maxMind, { AsnResponse, CityResponse, Reader } from 'maxmind'; import path from 'path'; import * as tar from 'tar'; import { promisify } from 'util'; @@ -10,8 +10,9 @@ const writeFileAsync = promisify(fs.writeFile); const statAsync = promisify(fs.stat); const mkdirAsync = promisify(fs.mkdir); -// const ASN_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-ASN/download?suffix=tar.gz'; -// const ASN_FILENAME = 'GeoLite2-ASN.tar.gz'; +const ASN_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-ASN/download?suffix=tar.gz'; +const ASN_FILENAME = 'GeoLite2-ASN.tar.gz'; +const ASN_DB_FILENAME = 'GeoLite2-ASN.mmdb'; const FOLDER_NAME = 'downloads'; const CITY_URL = 'https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz'; @@ -19,17 +20,23 @@ const CITY_ZIP_FILENAME = 'GeoLite2-City.tar.gz'; const CITY_DB_FILENAME = 'GeoLite2-City.mmdb'; const downloads = [ - // { url: ASN_URL, filename: ASN_FILENAME }, - { url: CITY_URL, archiveFilename: CITY_ZIP_FILENAME, dbFileName: CITY_DB_FILENAME, defaultLookup: true }, + { url: ASN_URL, archiveFilename: ASN_FILENAME, dbFileName: ASN_DB_FILENAME }, + { url: CITY_URL, archiveFilename: CITY_ZIP_FILENAME, dbFileName: CITY_DB_FILENAME }, ]; -let lookup: Reader; +let lookupAsn: Reader; +let lookupCity: Reader; export async function initMaxMind(rootDir: string, force = false) { - if (force || !lookup) { + if (force || !lookupAsn) { + const filePath = path.join(rootDir, FOLDER_NAME, ASN_DB_FILENAME); + logger.info(`Initializing ASN database: ${filePath}`); + lookupAsn = await maxMind.open(filePath); + } + if (force || !lookupCity) { const filePath = path.join(rootDir, FOLDER_NAME, CITY_DB_FILENAME); - logger.info(`Initializing database: ${filePath}`); - lookup = await maxMind.open(filePath); + logger.info(`Initializing CITY database: ${filePath}`); + lookupCity = await maxMind.open(filePath); } } @@ -42,7 +49,7 @@ export async function downloadMaxMindDb(rootDir: string): Promise { await mkdirAsync(downloadFolderPath, { recursive: true }); } - for (const { defaultLookup, archiveFilename, url } of downloads) { + for (const { archiveFilename, url } of downloads) { const archiveFilePath = path.join(downloadFolderPath, archiveFilename); // Check if file needs to be downloaded @@ -84,43 +91,42 @@ export async function downloadMaxMindDb(rootDir: string): Promise { filter: (path) => path.endsWith('.mmdb'), strip: 1, // Remove the first directory component from paths }); - - if (defaultLookup) { - initMaxMind(downloadFolderPath, true); - } } + await initMaxMind(rootDir, true); } export function lookupIpAddress(ipAddress: string) { - if (!lookup) { + if (!lookupAsn || !lookupCity) { throw new Error('MaxMind DB not initialized'); } - const results = lookup.get(ipAddress); - if (!results) { - return null; + const asnResults = lookupAsn.get(ipAddress); + const cityResults = lookupCity.get(ipAddress); + if (!cityResults) { + return { + query: ipAddress, + status: 'fail', + }; } + return { - city: results.city?.names?.en ?? null, - country: results.country?.names?.en ?? null, - countryISO: results.country?.iso_code ?? null, - isEU: !!results.country?.is_in_european_union, - continent: results.continent?.names?.en ?? null, - location: results.location ?? null, - postalCode: results.postal?.code ?? null, - registeredCountry: results.registered_country - ? { - country: results.registered_country.names.en, - iso: results.registered_country.iso_code, - isEU: !!results.registered_country.is_in_european_union, - } - : null, - subdivisions: - results.subdivisions?.map((item) => ({ - name: item.names.en, - iso: item.iso_code, - })) ?? [], - foo: results.traits, + query: ipAddress, + status: 'success', + continent: cityResults.continent?.names?.en ?? null, + continentCode: cityResults.continent?.code ?? null, + country: cityResults.country?.names?.en ?? null, + countryCode: cityResults.country?.iso_code ?? null, + region: cityResults.subdivisions?.[0]?.iso_code ?? null, + regionName: cityResults.subdivisions?.[0]?.names?.en ?? null, + city: cityResults.city?.names?.en ?? null, + zip: cityResults.postal?.code ?? null, + lat: cityResults.location?.latitude ?? null, + lon: cityResults.location?.longitude ?? null, + timezone: cityResults.location?.time_zone ?? null, + isEU: !!cityResults.country?.is_in_european_union, + isp: cityResults.traits?.isp ?? null, + org: cityResults.traits?.organization ?? asnResults?.autonomous_system_organization ?? null, + proxy: cityResults.traits?.is_anonymous_proxy ?? false, }; } diff --git a/libs/api-config/src/lib/env-config.ts b/libs/api-config/src/lib/env-config.ts index c1af84268..e3492d664 100644 --- a/libs/api-config/src/lib/env-config.ts +++ b/libs/api-config/src/lib/env-config.ts @@ -110,6 +110,7 @@ const envSchema = z.object({ CAPTCHA_SECRET_KEY: z.string().optional(), CAPTCHA_PROPERTY: z.literal('captchaToken').optional().default('captchaToken'), IP_API_KEY: z.string().optional().describe('API Key used to get location information from IP address'), + IP_API_SERVICE: z.enum(['IP-API', 'LOCAL']).optional().describe('API Key used to get location information from IP address'), VERSION: z.string().optional(), ROLLBAR_SERVER_TOKEN: z.string().optional(), @@ -204,9 +205,9 @@ const envSchema = z.object({ /** * GEO-IP API (private service basic auth) */ - GEO_IP_API_USERNAME: z.string().nullish(), - GEO_IP_API_PASSWORD: z.string().nullish(), - GEO_IP_API_HOSTNAME: z.string().nullish(), + GEO_IP_API_USERNAME: z.string().optional(), + GEO_IP_API_PASSWORD: z.string().optional(), + GEO_IP_API_HOSTNAME: z.string().optional(), }); const parseResults = envSchema.safeParse({ diff --git a/libs/auth/server/src/lib/auth.db.service.ts b/libs/auth/server/src/lib/auth.db.service.ts index e45867338..2a6efe36f 100644 --- a/libs/auth/server/src/lib/auth.db.service.ts +++ b/libs/auth/server/src/lib/auth.db.service.ts @@ -283,21 +283,31 @@ export async function getUserSessions(userId: string, omitLocationData?: boolean ); // Fetch location data and add to each session - if (!omitLocationData && ENV.IP_API_KEY && sessions.length > 0) { + if (!omitLocationData && sessions.length > 0) { try { + let response: Awaited> | null = null; const ipAddresses = sessions.map((session) => session.ipAddress); + if (ENV.IP_API_SERVICE === 'LOCAL' && ENV.GEO_IP_API_USERNAME && ENV.GEO_IP_API_PASSWORD && ENV.GEO_IP_API_HOSTNAME) { + response = await fetch(`${ENV.GEO_IP_API_HOSTNAME}/api/lookup`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${Buffer.from(`${ENV.GEO_IP_API_USERNAME}:${ENV.GEO_IP_API_PASSWORD}`, 'utf-8').toString('base64')}`, + }, + body: JSON.stringify({ ips: ipAddresses }), + }); + } else if (ENV.IP_API_KEY) { + const params = new URLSearchParams({ + fields: 'status,country,countryCode,region,regionName,city,isp,query', + key: ENV.IP_API_KEY, + }); - const params = new URLSearchParams({ - fields: 'status,country,countryCode,region,regionName,city,isp,query', - key: ENV.IP_API_KEY, - }); - - const response = await fetch(`https://pro.ip-api.com/batch?${params.toString()}`, { - method: 'POST', - body: JSON.stringify(ipAddresses), - }); - - if (response.ok) { + response = await fetch(`https://pro.ip-api.com/batch?${params.toString()}`, { + method: 'POST', + body: JSON.stringify(ipAddresses), + }); + } + if (response?.ok) { const locations = (await response.json()) as SessionIpData[]; return sessions.map( (session, i): UserSessionWithLocation => ({ diff --git a/libs/auth/types/src/lib/auth-types.ts b/libs/auth/types/src/lib/auth-types.ts index fb80c96b7..fc77a689e 100644 --- a/libs/auth/types/src/lib/auth-types.ts +++ b/libs/auth/types/src/lib/auth-types.ts @@ -106,13 +106,13 @@ export interface SessionData { export interface SessionIpSuccess { status: 'success'; - country: string; - countryCode: string; - region: string; - regionName: string; - city: string; - isp: string; - query: string; + country: string | null; + countryCode: string | null; + region: string | null; + regionName: string | null; + city: string | null; + isp: string | null; + query: string | null; } export interface SessionIpFail {