From e0dd556d866f4922fa40935ed7de7319d1e2101d Mon Sep 17 00:00:00 2001 From: Niel Markwick Date: Wed, 4 Dec 2024 21:42:27 +0100 Subject: [PATCH] feat: convert Javascript code to Typescript --- .github/workflows/codehealth.yml | 6 +- .gitignore | 1 + cloudrun-malware-scanner/.gcloudignore | 1 + cloudrun-malware-scanner/.husky/pre-commit | 3 +- cloudrun-malware-scanner/.prettierignore | 1 + cloudrun-malware-scanner/Dockerfile | 4 +- cloudrun-malware-scanner/clamdjs.d.ts | 45 +++ cloudrun-malware-scanner/config.ts | 150 ++++--- cloudrun-malware-scanner/gcs-proxy-server.ts | 126 +++--- cloudrun-malware-scanner/logger.ts | 34 +- cloudrun-malware-scanner/metrics.ts | 273 +++++-------- cloudrun-malware-scanner/package-lock.json | 375 ++++++++++++++++++ cloudrun-malware-scanner/package.json | 11 +- cloudrun-malware-scanner/scanner.ts | 169 +++----- cloudrun-malware-scanner/server.ts | 130 +++--- cloudrun-malware-scanner/spec/config.spec.ts | 91 ++--- cloudrun-malware-scanner/spec/scanner.spec.ts | 140 +++---- .../spec/support/jasmine.json | 2 +- cloudrun-malware-scanner/tsconfig.json | 21 +- 19 files changed, 908 insertions(+), 675 deletions(-) create mode 100644 cloudrun-malware-scanner/clamdjs.d.ts diff --git a/.github/workflows/codehealth.yml b/.github/workflows/codehealth.yml index 94c4ce2..442f2b1 100644 --- a/.github/workflows/codehealth.yml +++ b/.github/workflows/codehealth.yml @@ -37,11 +37,7 @@ jobs: working-directory: cloudrun-malware-scanner/ run: npm run check-format - - name: Typescript checks - working-directory: cloudrun-malware-scanner/ - run: npm run typecheck - - - name: NPM test + - name: NPM compile and test working-directory: cloudrun-malware-scanner/ run: npm test diff --git a/.gitignore b/.gitignore index 56bf837..7884dfa 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ config.json .terraform **/.terraform/* .terraform.tfstate.lock.info +build diff --git a/cloudrun-malware-scanner/.gcloudignore b/cloudrun-malware-scanner/.gcloudignore index 598b49f..a62e08a 100644 --- a/cloudrun-malware-scanner/.gcloudignore +++ b/cloudrun-malware-scanner/.gcloudignore @@ -4,3 +4,4 @@ node_modules .gcloudignore .eslintrc.js config.json.tmpl +build diff --git a/cloudrun-malware-scanner/.husky/pre-commit b/cloudrun-malware-scanner/.husky/pre-commit index 58a4117..f3fab7a 100644 --- a/cloudrun-malware-scanner/.husky/pre-commit +++ b/cloudrun-malware-scanner/.husky/pre-commit @@ -3,7 +3,6 @@ echo "Running cloudrun-malware-scanner/.husky/pre-commit checks. Use -n/--no-ver cd cloudrun-malware-scanner || exit 1 npm run eslint npm run check-format -npm run typecheck -npm run terraform-validate npm test npm audit +npm run terraform-validate diff --git a/cloudrun-malware-scanner/.prettierignore b/cloudrun-malware-scanner/.prettierignore index 9f4ca7e..b345e08 100644 --- a/cloudrun-malware-scanner/.prettierignore +++ b/cloudrun-malware-scanner/.prettierignore @@ -11,3 +11,4 @@ CHANGELOG.md ../.release-please-manifest.json ../terraform/*/.terraform ../terraform/*/terraform.tfstate* +build diff --git a/cloudrun-malware-scanner/Dockerfile b/cloudrun-malware-scanner/Dockerfile index 52afe23..8c8117d 100644 --- a/cloudrun-malware-scanner/Dockerfile +++ b/cloudrun-malware-scanner/Dockerfile @@ -96,7 +96,7 @@ RUN set -x \ WORKDIR /app COPY . /app -# Install required NPM modules -RUN npm install --omit=dev +# Install required NPM modules and build +RUN npm install && npm run build CMD ["bash", "bootstrap.sh"] diff --git a/cloudrun-malware-scanner/clamdjs.d.ts b/cloudrun-malware-scanner/clamdjs.d.ts new file mode 100644 index 0000000..a996d00 --- /dev/null +++ b/cloudrun-malware-scanner/clamdjs.d.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Typescript module typings for clamdjs npm + */ +declare module 'clamdjs' { + export type ScanStreamFunc = ( + stream: import('node:stream').Readable, + timeout: number, + ) => Promise; + + export interface IScanner { + scanStream: ScanStreamFunc; + } + + export function createScanner(host: string, port: number): IScanner; + + export function ping( + host: string, + port: number, + timeout?: number, + ): Promise; + + export function version( + host: string, + port: number, + timeout?: number, + ): Promise; + + export function isCleanReply(reply: any): boolean; +} diff --git a/cloudrun-malware-scanner/config.ts b/cloudrun-malware-scanner/config.ts index f87ad28..604f3ea 100644 --- a/cloudrun-malware-scanner/config.ts +++ b/cloudrun-malware-scanner/config.ts @@ -1,50 +1,47 @@ /* -* Copyright 2022 Google LLC - -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at - -* https://www.apache.org/licenses/LICENSE-2.0 - -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -const {logger} = require('./logger.js'); -const {readFileSync} = require('node:fs'); - -/** @typedef {import('@google-cloud/storage').Storage} Storage */ - -/** - * @typedef {{ - * unscanned: string, - * clean: string, - * quarantined: string, - * }} BucketDefs + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -/** @type {Array} */ -const BUCKET_TYPES = ['unscanned', 'clean', 'quarantined']; +import {logger} from './logger'; +import {readFileSync} from 'node:fs'; +import {Storage} from '@google-cloud/storage'; /** * Configuration object. * * Values are read from the JSON configuration file. * See {@link readAndVerifyConfig}. - * - * @typedef {{ - * buckets: Array, - * ClamCvdMirrorBucket: string, - * fileExclusionPatterns?: Array>, - * fileExclusionRegexps: Array, - * ignoreZeroLengthFiles: boolean, - * comments?: string | string[] - * }} Config */ +export type BucketDefs = { + unscanned: string; + clean: string; + quarantined: string; +}; + +export type Config = { + buckets: Array; + ClamCvdMirrorBucket: string; + fileExclusionPatterns?: Array>; + fileExclusionRegexps: Array; + ignoreZeroLengthFiles: boolean; + comments?: string | string[]; +}; + +const BUCKET_TYPES = ['unscanned', 'clean', 'quarantined'] as Array< + keyof BucketDefs +>; /** * Read configuration from JSON configuration file, parse, verify @@ -55,28 +52,26 @@ const BUCKET_TYPES = ['unscanned', 'clean', 'quarantined']; * @param {Storage} storage * @return {Promise} */ -async function readAndVerifyConfig(configFile, storage) { +export async function readAndVerifyConfig( + configFile: string, + storage: Storage, +): Promise { logger.info(`Using configuration file: ${configFile}`); let configText; try { configText = readFileSync(configFile, {encoding: 'utf-8'}); - } catch (e) { - const err = /** @type {Error} */ (e); + } catch (e: any) { logger.fatal( - err, - `Unable to read JSON file from ${configFile}: ${err.message}`, + e, + `Unable to read JSON file from ${configFile}: ${e.message}`, ); - throw err; + throw e; } try { return validateConfig(parseConfig(configText), storage); - } catch (e) { - const err = /** @type {Error} */ (e); - logger.fatal( - err, - `Failed parsing config file: ${configFile}: ${err.message}`, - ); - throw err; + } catch (e: any) { + logger.fatal(e, `Failed parsing config file: ${configFile}: ${e.message}`); + throw e; } } @@ -84,13 +79,13 @@ async function readAndVerifyConfig(configFile, storage) { * @param {string} configText * @returns {Config} */ -function parseConfig(configText) { +function parseConfig(configText: string): Config { /** @type {Config} */ let config; try { config = JSON.parse(configText); - } catch (e) { + } catch (e: any) { throw new Error(`Failed to parse configuration as JSON: ${e}`); } return config; @@ -105,7 +100,7 @@ function parseConfig(configText) { * @param {Storage} storage * @return {Promise} */ -async function validateConfig(config, storage) { +async function validateConfig(config: any, storage: Storage): Promise { delete config.comments; if (config.buckets.length === 0) { @@ -171,28 +166,26 @@ async function validateConfig(config, storage) { // config.fileExclusionPatterns is an array, check each value and // convert to a regexp in fileExclusionRegexps[] for (const i in config.fileExclusionPatterns) { - /** @type {string|undefined} */ - let pattern; - /** @type {string|undefined} */ - let flags; + let pattern: string | undefined; + let flags: string | undefined; // Each element can either be a simple pattern: // "^.*\\.tmp$" // or an array with pattern and flags, eg for case-insensive matching: // [ "^.*\\tmp$", "i" ] - - if (typeof config.fileExclusionPatterns[i] === 'string') { + const element = config.fileExclusionPatterns[i]; + if (typeof element === 'string') { // validate regex as simple string - pattern = config.fileExclusionPatterns[i]; + pattern = element; } else if ( - config.fileExclusionPatterns[i] instanceof Array && - config.fileExclusionPatterns[i].length <= 2 && - config.fileExclusionPatterns[i].length >= 1 && - typeof config.fileExclusionPatterns[i][0] === 'string' + Array.isArray(element) && + element.length <= 2 && + element.length >= 1 && + typeof element[0] === 'string' ) { // validate regex as [pattern, flags] - pattern = config.fileExclusionPatterns[i][0]; - flags = config.fileExclusionPatterns[i][1]; + pattern = element[0]; + flags = element[1]; } else { pattern = undefined; } @@ -205,11 +198,10 @@ async function validateConfig(config, storage) { } else { try { config.fileExclusionRegexps[i] = new RegExp(pattern, flags); - } catch (e) { - const err = /** @type {Error} */ (e); + } catch (e: any) { logger.fatal( - err, - `Config Error: fileExclusionPatterns[${i}]: Regexp compile failed for ${JSON.stringify(config.fileExclusionPatterns[i])}: ${err.message}`, + e, + `Config Error: fileExclusionPatterns[${i}]: Regexp compile failed for ${JSON.stringify(config.fileExclusionPatterns[i])}: ${e.message}`, ); success = false; } @@ -222,6 +214,7 @@ async function validateConfig(config, storage) { if (!success) { throw new Error('Invalid configuration'); } + return Object.freeze(config); } @@ -233,7 +226,11 @@ async function validateConfig(config, storage) { * @param {Storage} storage * @return {Promise} */ -async function checkBucketExists(bucketName, configName, storage) { +async function checkBucketExists( + bucketName: string, + configName: string, + storage: Storage, +): Promise { if (!bucketName) { logger.fatal(`Config Error: no "${configName}" bucket defined`); return false; @@ -247,7 +244,7 @@ async function checkBucketExists(bucketName, configName, storage) { .bucket(bucketName) .getFiles({maxResults: 1, prefix: 'zzz', autoPaginate: false}); return true; - } catch (e) { + } catch (e: any) { logger.fatal( `Error in config: cannot view files in "${configName}" : ${bucketName} : ${e}`, ); @@ -256,10 +253,7 @@ async function checkBucketExists(bucketName, configName, storage) { } } -module.exports = { - readAndVerifyConfig, - TEST_ONLY: { - checkBucketExists, - validateConfig, - }, +export const TEST_ONLY = { + checkBucketExists, + validateConfig, }; diff --git a/cloudrun-malware-scanner/gcs-proxy-server.ts b/cloudrun-malware-scanner/gcs-proxy-server.ts index 00c920f..54d15c8 100644 --- a/cloudrun-malware-scanner/gcs-proxy-server.ts +++ b/cloudrun-malware-scanner/gcs-proxy-server.ts @@ -1,54 +1,40 @@ /* -* Copyright 2022 Google LLC - -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at - -* https://www.apache.org/licenses/LICENSE-2.0 - -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -const process = require('node:process'); -const {GoogleAuth} = require('google-auth-library'); -const {logger} = require('./logger.js'); -const {readAndVerifyConfig} = require('./config.js'); -const httpProxy = require('http-proxy'); -const {Storage} = require('@google-cloud/storage'); -const pkgJson = require('./package.json'); - -/** @typedef {import('./config.js').Config} Config */ -/** @typedef {import('http').IncomingMessage} IncomingMessage */ -/** @typedef {import('http').ServerResponse} ServerResponse */ -/** @typedef {import('http').ClientRequest} ClientRequest */ + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as process from 'node:process'; +import {GoogleAuth} from 'google-auth-library'; +import {logger} from './logger.js'; +import {readAndVerifyConfig, Config} from './config.js'; +import * as httpProxy from 'http-proxy'; +import {Storage} from '@google-cloud/storage'; +import {name as packageName, version as packageVersion} from './package.json'; +import {IncomingMessage, ServerResponse, ClientRequest} from 'http'; +import {Socket} from 'node:net'; const TOKEN_REFRESH_THRESHOLD_MILLIS = 60000; const googleAuth = new GoogleAuth(); -/** - * Access token for GCS requests - will be refreshed shortly before it expires - * @type {string | null | undefined} - */ -let accessToken; +let accessToken: string | null | undefined; -/** @type {NodeJS.Timeout | null} */ -let accessTokenRefreshTimeout; +let accessTokenRefreshTimeout: NodeJS.Timeout | null = null; let clamCvdMirrorBucket = 'uninitialized'; -/** - * Check to see when access token expires and refresh it just before. - * This is required because proxy requires access token to be available - * synchronously, but getAccessToken() is async. - * So a 'current' access token needs to be available. - */ -async function accessTokenRefresh() { +async function accessTokenRefresh(): Promise { if (accessTokenRefreshTimeout) { clearTimeout(accessTokenRefreshTimeout); accessTokenRefreshTimeout = null; @@ -63,13 +49,12 @@ async function accessTokenRefresh() { accessToken = await googleAuth.getAccessToken(); logger.info( `Refreshed Access token; expires at ${new Date( - /** @type {number} */ (client.credentials.expiry_date), + client.credentials.expiry_date!, // Non-null assertion is safe here due to the check above ).toISOString()}`, ); } const nextCheckDate = new Date( - /** @type {number} */ (client.credentials.expiry_date) - - TOKEN_REFRESH_THRESHOLD_MILLIS, + client.credentials.expiry_date! - TOKEN_REFRESH_THRESHOLD_MILLIS, // Non-null assertion is safe here due to the check above ); logger.debug( `Next access token refresh check at ${nextCheckDate.toISOString()}`, @@ -80,39 +65,28 @@ async function accessTokenRefresh() { ); } -/** - * Handle any internal proxy errors by returning a 500 - * - * @param {Error} err - * @param {IncomingMessage} req The request payload - * @param {ServerResponse | import('node:net').Socket} res The HTTP response object - * @param {import('http-proxy').ProxyTargetUrl=} _target - */ function handleProxyError( - err, - req, - res, - // eslint-disable-next-line no-unused-vars - _target, -) { + err: Error, + req: IncomingMessage, + res: ServerResponse | Socket, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _target?: httpProxy.ProxyTargetUrl, +): void { logger.error( `Failed to proxy to GCS for path ${req.url}, returning code 500: ${err}`, ); - /**@type {ServerResponse} */ (res).writeHead(500, { + (res as ServerResponse).writeHead(500, { 'Content-Type': 'text/plain', }); res.end(`Failed to proxy to GCS: internal error\n`); } -/** - * Handle proxy requests - check path, and add Authorization header. - * - * @param {!ClientRequest} proxyReq - * @param {!IncomingMessage} req The request payload - * @param {!ServerResponse} res The HTTP response object - */ -function handleProxyReq(proxyReq, req, res) { - if (proxyReq.path.startsWith(`/${clamCvdMirrorBucket}/`)) { +function handleProxyReq( + proxyReq: ClientRequest, + req: IncomingMessage, + res: ServerResponse, +): void { + if (proxyReq.path?.startsWith(`/${clamCvdMirrorBucket}/`)) { logger.info(`Proxying request for ${proxyReq.path} to GCS`); proxyReq.setHeader('Authorization', 'Bearer ' + accessToken); } else { @@ -124,11 +98,7 @@ function handleProxyReq(proxyReq, req, res) { } } -/** - * Set up a reverse proxy to add authentication to HTTP requests from - * freshclam and proxy it to the GCS API - */ -async function setupGcsReverseProxy() { +async function setupGcsReverseProxy(): Promise { const proxy = httpProxy.createProxyServer({ target: 'https://storage.googleapis.com/', changeOrigin: true, @@ -148,12 +118,7 @@ async function setupGcsReverseProxy() { ); } -/** - * Perform async setup and start the app. - * - * @async - */ -async function run() { +async function run(): Promise { let configFile; if (process.argv.length >= 3) { configFile = process.argv[2]; @@ -162,10 +127,9 @@ async function run() { } const storage = new Storage({ - userAgent: `cloud-solutions/${pkgJson.name}-usage-v${pkgJson.version}`, + userAgent: `cloud-solutions/${packageName}-usage-v${packageVersion}`, }); - /** @type {Config} */ const config = await readAndVerifyConfig(configFile, storage); clamCvdMirrorBucket = config.ClamCvdMirrorBucket; diff --git a/cloudrun-malware-scanner/logger.ts b/cloudrun-malware-scanner/logger.ts index e1e9510..c5be710 100644 --- a/cloudrun-malware-scanner/logger.ts +++ b/cloudrun-malware-scanner/logger.ts @@ -16,17 +16,15 @@ * @fileoverview Create a Pino Logger using structured logging to stdout. */ -const {pino} = require('pino'); -const { - createGcpLoggingPinoConfig, -} = require('@google-cloud/pino-logging-gcp-config'); -const process = require('node:process'); -const packageJson = require('./package.json'); +import {pino} from 'pino'; +import {createGcpLoggingPinoConfig} from '@google-cloud/pino-logging-gcp-config'; +import * as process from 'node:process'; +import {name as packageName, version as packageVersion} from './package.json'; /** @type {pino.Level} */ -const DEFAULT_LOG_LEVEL = 'info'; +const DEFAULT_LOG_LEVEL: pino.Level = 'info'; /** @type {pino.Level} */ -const DEFAULT_TEST_LOG_LEVEL = 'fatal'; +const DEFAULT_TEST_LOG_LEVEL: pino.Level = 'fatal'; /** * Return a pino log level based on environment variable LOG_LEVEL which can be @@ -36,17 +34,15 @@ const DEFAULT_TEST_LOG_LEVEL = 'fatal'; * * @return {pino.Level} */ -function getLogLevel() { +function getLogLevel(): pino.Level { const envLogLevel = process.env.LOG_LEVEL ? process.env.LOG_LEVEL.toLowerCase() : undefined; - // Convert the env varable to the pino level, or undefined - const pinoLevel = /** @type {pino.Level | undefined } */ ( - Object.keys(pino.levels.values).filter( - (level) => level.toLowerCase() === envLogLevel, - )[0] - ); + // Convert the env variable to the pino level, or undefined + const pinoLevel = (Object.keys(pino.levels.values).find( + (level) => level.toLowerCase() === envLogLevel, + ) || undefined) as pino.Level | undefined; if (pinoLevel) { return pinoLevel; @@ -61,8 +57,8 @@ const logger = pino( createGcpLoggingPinoConfig( { serviceContext: { - service: packageJson.name, - version: packageJson.version, + service: packageName, + version: packageVersion, }, }, { @@ -71,6 +67,4 @@ const logger = pino( ), ); -module.exports = { - logger, -}; +export {logger}; diff --git a/cloudrun-malware-scanner/metrics.ts b/cloudrun-malware-scanner/metrics.ts index ab90e96..02f4fe8 100644 --- a/cloudrun-malware-scanner/metrics.ts +++ b/cloudrun-malware-scanner/metrics.ts @@ -1,43 +1,40 @@ /* -* Copyright 2021 Google LLC - -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at - -* https://www.apache.org/licenses/LICENSE-2.0 - -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -const process = require('node:process'); + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as process from 'node:process'; -const { +import { MeterProvider, PeriodicExportingMetricReader, -} = require('@opentelemetry/sdk-metrics'); -const {Resource} = require('@opentelemetry/resources'); -const { - MetricExporter: GcpMetricExporter, -} = require('@google-cloud/opentelemetry-cloud-monitoring-exporter'); -const {GcpDetectorSync} = require('@google-cloud/opentelemetry-resource-util'); -const Semconv = require('@opentelemetry/semantic-conventions'); -const {version: packageVersion} = require('./package.json'); - -const {logger} = require('./logger.js'); +} from '@opentelemetry/sdk-metrics'; +import {Resource} from '@opentelemetry/resources'; +import {MetricExporter as GcpMetricExporter} from '@google-cloud/opentelemetry-cloud-monitoring-exporter'; +import {GcpDetectorSync} from '@google-cloud/opentelemetry-resource-util'; +import * as Semconv from '@opentelemetry/semantic-conventions'; +import {version as packageVersion} from './package.json'; + +import {logger} from './logger.js'; +import * as OpenTelemetryApi from '@opentelemetry/api'; + const METRIC_TYPE_ROOT = 'malware-scanning/'; -const OpenTelemetryApi = require('@opentelemetry/api'); -/** - * @typedef {{ - * [x: string]: string, - * }} CounterAttributes - */ -/** @type {CounterAttributes} */ -const RESOURCE_ATTRIBUTES = { +interface CounterAttributes { + [x: string]: string; +} + +const RESOURCE_ATTRIBUTES: CounterAttributes = { [Semconv.SEMRESATTRS_SERVICE_NAMESPACE]: 'googlecloudplatform', [Semconv.SEMRESATTRS_SERVICE_NAME]: 'gcs-malware-scanning', [Semconv.SEMRESATTRS_SERVICE_VERSION]: packageVersion, @@ -49,16 +46,15 @@ const COUNTERS_PREFIX = RESOURCE_ATTRIBUTES[Semconv.SEMRESATTRS_SERVICE_NAME] + '/'; -/** @enum{string} */ -const COUNTER_NAMES = { - cleanFiles: 'clean-files', - infectedFiles: 'infected-files', - ignoredFiles: 'ignored-files', - scansFailed: 'scans-failed', - bytesScanned: 'bytes-scanned', - scanDuration: 'scan-duration', - cvdUpdates: 'cvd-mirror-updates', -}; +enum COUNTER_NAMES { + cleanFiles = 'clean-files', + infectedFiles = 'infected-files', + ignoredFiles = 'ignored-files', + scansFailed = 'scans-failed', + bytesScanned = 'bytes-scanned', + scanDuration = 'scan-duration', + cvdUpdates = 'cvd-mirror-updates', +} const COUNTER_ATTRIBUTE_NAMES = { sourceBucket: 'source_bucket', @@ -70,88 +66,57 @@ const COUNTER_ATTRIBUTE_NAMES = { ignoredRegex: 'ignored_regex', }; -/** - * Global counters object, populated by createCounters. - * - * @type {Map< - * COUNTER_NAMES, - * { - * cumulative?: OpenTelemetryApi.Counter, - * histogram?: OpenTelemetryApi.Histogram - * } - * >} counter Name to counter instance - */ -const COUNTERS = new Map(); +interface Counter { + cumulative?: OpenTelemetryApi.Counter; + histogram?: OpenTelemetryApi.Histogram; +} + +const COUNTERS: Map = new Map(); const METRIC_EXPORT_INTERVAL = parseInt( process.env.EXPORT_INTERVAL || '20000', 10, ); -/** - * Wrapper class for OpenTelemetry DiagLogger to convert to Bunyan log levels - * - * @extends {OpenTelemetryApi.DiagLogger} - */ -class DiagToBunyanLogger { - /** @constructor */ +class DiagToPinoLogger implements OpenTelemetryApi.DiagLogger { + suppressErrors: boolean; + constructor() { // In some cases where errors may be expected, we want to be able to supress // them. this.suppressErrors = false; } - /** - * @param {string} message - * @param {any[]} args - */ - verbose(message, ...args) { + verbose(message: string, ...args: any[]) { logger.trace('otel: ' + message, args); } - /** - * @param {string} message - * @param {any[]} args - */ - debug(message, ...args) { + debug(message: string, ...args: any[]) { logger.debug('otel: ' + message, args); } - /** - * @param {string} message - * @param {any[]} args - */ - info(message, ...args) { + + info(message: string, ...args: any[]) { logger.info('otel: ' + message, args); } - /** - * @param {string} message - * @param {any[]} args - */ - warn(message, ...args) { + + warn(message: string, ...args: any[]) { logger.warn('otel: ' + message, args); } - /** - * @param {string} message - * @param {any[]} args - */ - error(message, ...args) { + + error(message: string, ...args: any[]) { if (!this.suppressErrors) { logger.error('otel: ' + message, args); } } } -OpenTelemetryApi.default.diag.setLogger(new DiagToBunyanLogger(), { +OpenTelemetryApi.default.diag.setLogger(new DiagToPinoLogger(), { logLevel: OpenTelemetryApi.DiagLogLevel.INFO, suppressOverrideMessage: true, }); -/** - * Writes a scan failed metric. - * @param {string} [sourceBucket] - */ -function writeScanFailedMetric(sourceBucket) { - const attrs = { +function writeScanFailedMetric(sourceBucket?: string) { + const attrs: CounterAttributes = { [COUNTER_ATTRIBUTE_NAMES.cloudRunRevision]: process.env.K_REVISION || 'no-revision', }; @@ -162,20 +127,12 @@ function writeScanFailedMetric(sourceBucket) { COUNTERS.get(COUNTER_NAMES.scansFailed)?.cumulative?.add(1, attrs); } -/** - * Writes metrics when a clean file is scanned - * @param {string} sourceBucket - * @param {string} destinationBucket - * @param {number} fileSize - * @param {number} scanDuration - * @param {string} clamVersion - */ function writeScanCleanMetric( - sourceBucket, - destinationBucket, - fileSize, - scanDuration, - clamVersion, + sourceBucket: string, + destinationBucket: string, + fileSize: number, + scanDuration: number, + clamVersion: string, ) { writeScanCompletedMetric_( COUNTER_NAMES.cleanFiles, @@ -187,23 +144,14 @@ function writeScanCleanMetric( ); } -/** - * Writes metrics when a file is ignored - * @param {string} sourceBucket - * @param {string} destinationBucket - * @param {number} fileSize - * @param {string} ignoredReason - * @param {string} [ignoredRegex] - */ function writeScanIgnoredMetric( - sourceBucket, - destinationBucket, - fileSize, - ignoredReason, - ignoredRegex, + sourceBucket: string, + destinationBucket: string, + fileSize: number, + ignoredReason: string, + ignoredRegex?: string, ) { - /** @type {Record} */ - const additionalAttrs = {}; + const additionalAttrs: CounterAttributes = {}; if (ignoredReason) { additionalAttrs[COUNTER_ATTRIBUTE_NAMES.ignoredReason] = ignoredReason; } @@ -221,20 +169,12 @@ function writeScanIgnoredMetric( ); } -/** - * Writes metrics when an infected file is scanned - * @param {string} sourceBucket - * @param {string} destinationBucket - * @param {number} fileSize - * @param {number} scanDuration - * @param {string} clamVersion - */ function writeScanInfectedMetric( - sourceBucket, - destinationBucket, - fileSize, - scanDuration, - clamVersion, + sourceBucket: string, + destinationBucket: string, + fileSize: number, + scanDuration: number, + clamVersion: string, ) { writeScanCompletedMetric_( COUNTER_NAMES.infectedFiles, @@ -245,28 +185,17 @@ function writeScanInfectedMetric( clamVersion, ); } -/** - * Writes metrics for completed scans - * @private - * @param {string} counterName - * @param {string} sourceBucket - * @param {string} destinationBucket - * @param {number} fileSize - * @param {number?} scanDuration - * @param {string?} clamVersion - * @param {Record} [additionalAttrs] - */ + function writeScanCompletedMetric_( - counterName, - sourceBucket, - destinationBucket, - fileSize, - scanDuration, - clamVersion, - additionalAttrs, + counterName: COUNTER_NAMES, + sourceBucket: string, + destinationBucket: string, + fileSize: number, + scanDuration: number | null, + clamVersion: string | null, + additionalAttrs: CounterAttributes = {}, ) { - /** @type {Record} */ - const attrs = { + const attrs: CounterAttributes = { ...additionalAttrs, [COUNTER_ATTRIBUTE_NAMES.sourceBucket]: sourceBucket, [COUNTER_ATTRIBUTE_NAMES.destinationBucket]: destinationBucket, @@ -275,7 +204,7 @@ function writeScanCompletedMetric_( }; if (clamVersion) { - [COUNTER_ATTRIBUTE_NAMES.clamVersion] = clamVersion; + attrs[COUNTER_ATTRIBUTE_NAMES.clamVersion] = clamVersion; } const counter = COUNTERS.get(counterName); @@ -284,7 +213,7 @@ function writeScanCompletedMetric_( } counter.cumulative.add(1, attrs); - COUNTERS.get('bytes-scanned')?.cumulative?.add(fileSize, attrs); + COUNTERS.get(COUNTER_NAMES.bytesScanned)?.cumulative?.add(fileSize, attrs); if (scanDuration) { COUNTERS.get(COUNTER_NAMES.scanDuration)?.histogram?.record( scanDuration, @@ -293,13 +222,7 @@ function writeScanCompletedMetric_( } } -/** - * Writes metrics when a CVD Mirror Update occurs. - * - * @param {boolean} success - * @param {boolean} isUpdated - */ -function writeCvdMirrorUpdatedMetric(success, isUpdated) { +function writeCvdMirrorUpdatedMetric(success: boolean, isUpdated: boolean) { COUNTERS.get(COUNTER_NAMES.cvdUpdates)?.cumulative?.add(1, { [COUNTER_ATTRIBUTE_NAMES.cloudRunRevision]: process.env.K_REVISION || 'no-revision', @@ -311,11 +234,7 @@ function writeCvdMirrorUpdatedMetric(success, isUpdated) { }); } -/** - * Initialize cloud monitoring and set up metrics descriptions - * @param {string} projectId - */ -async function initMetrics(projectId) { +async function initMetrics(projectId: string) { if (!projectId) { throw Error('Unable to proceed without a Project ID'); } @@ -404,9 +323,11 @@ async function initMetrics(projectId) { ); } -exports.writeScanFailed = writeScanFailedMetric; -exports.writeScanClean = writeScanCleanMetric; -exports.writeScanIgnored = writeScanIgnoredMetric; -exports.writeScanInfected = writeScanInfectedMetric; -exports.writeCvdMirrorUpdated = writeCvdMirrorUpdatedMetric; -exports.init = initMetrics; +export { + writeScanFailedMetric as writeScanFailed, + writeScanCleanMetric as writeScanClean, + writeScanIgnoredMetric as writeScanIgnored, + writeScanInfectedMetric as writeScanInfected, + writeCvdMirrorUpdatedMetric as writeCvdMirrorUpdated, + initMetrics as init, +}; diff --git a/cloudrun-malware-scanner/package-lock.json b/cloudrun-malware-scanner/package-lock.json index 2b6245c..3c76379 100644 --- a/cloudrun-malware-scanner/package-lock.json +++ b/cloudrun-malware-scanner/package-lock.json @@ -33,6 +33,7 @@ "globals": "^15.13.0", "husky": "^9.1.7", "jasmine": "^5.5.0", + "jasmine-console-reporter": "^3.1.0", "prettier": "^3.4.2", "typescript": "^5.7.2" } @@ -1503,11 +1504,44 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/ci-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", + "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==", + "dev": true, + "license": "MIT" + }, "node_modules/clamdjs": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/clamdjs/-/clamdjs-1.0.2.tgz", "integrity": "sha512-gVnX5ySMULvwYL2ykZQnP4UK4nIK7ftG6z015drJyOFgWpsqXt1Hcq4fMyPwM8LLsxfgfYKLiZi288xuTfmZBQ==" }, + "node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "restore-cursor": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -1521,6 +1555,16 @@ "node": ">=12" } }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1733,6 +1777,19 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/define-data-property": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", @@ -3114,6 +3171,82 @@ "jasmine": "bin/jasmine.js" } }, + "node_modules/jasmine-console-reporter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jasmine-console-reporter/-/jasmine-console-reporter-3.1.0.tgz", + "integrity": "sha512-fNP6XlgkIyNvfr6JVMJudZL9qWNY2K7l934Ojj4k8J09/QXf4xYf2Mc7MUgcsDhqIb2zTkLd2LsBJWFvJz41/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "chalk": "^2.4.1", + "ci-info": "^1.4.0", + "node-emoji": "^1.8.1", + "ora": "^3.0.0", + "perfy": "^1.1.5" + }, + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jasmine": ">=3.0.0" + } + }, + "node_modules/jasmine-console-reporter/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jasmine-console-reporter/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/jasmine-console-reporter/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/jasmine-console-reporter/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jasmine-console-reporter/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/jasmine-core": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-5.5.0.tgz", @@ -3271,6 +3404,13 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.camelcase": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", @@ -3330,6 +3470,74 @@ "dev": true, "license": "MIT" }, + "node_modules/log-symbols": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", + "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/log-symbols/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/log-symbols/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/log-symbols/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/long": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", @@ -3410,6 +3618,16 @@ "node": ">= 0.6" } }, + "node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -3460,6 +3678,16 @@ "node": ">= 0.6" } }, + "node_modules/node-emoji": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz", + "integrity": "sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.17.21" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -3523,6 +3751,19 @@ "wrappy": "1" } }, + "node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3540,6 +3781,102 @@ "node": ">= 0.8.0" } }, + "node_modules/ora": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-3.4.0.tgz", + "integrity": "sha512-eNwHudNbO1folBP3JsZ19v9azXWtQZjICdr3Q0TDPIaeBQ3mXLrh54wM+er0+hSp+dWKf+Z8KM58CYzEyIYxYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-spinners": "^2.0.0", + "log-symbols": "^2.2.0", + "strip-ansi": "^5.2.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/ansi-regex": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/ora/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ora/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ora/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/ora/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/ora/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/ora/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3686,6 +4023,13 @@ "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", "license": "MIT" }, + "node_modules/perfy": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/perfy/-/perfy-1.1.5.tgz", + "integrity": "sha512-/ieVBpMaPTJf83YTUl2TImsSwMEJ23qGP2w27pE6aX+NrB/ZRGqOnQZpl7J719yFwd+ebDiHguPNFeMSamyK7w==", + "dev": true, + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", @@ -3933,6 +4277,27 @@ "node": ">=8" } }, + "node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/restore-cursor/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -4486,6 +4851,16 @@ "node": ">= 0.8" } }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "dev": true, + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", diff --git a/cloudrun-malware-scanner/package.json b/cloudrun-malware-scanner/package.json index 5ea4d27..31e6908 100644 --- a/cloudrun-malware-scanner/package.json +++ b/cloudrun-malware-scanner/package.json @@ -5,18 +5,20 @@ "main": "index.js", "scripts": { "check-format": "npm run prettier-check && npm run terraform-fmt-check", - "start": "node server.js", + "start": "node --enable-source-maps build/server.js", "terraform-fmt": "terraform fmt ../terraform/*/*.tf ../terraform/*/*/*.tf", "terraform-fmt-check": "terraform fmt -check ../terraform/*/*.tf ../terraform/*/*/*.tf", "terraform-validate": "echo 'validating terraform/infra' && cd ../terraform/infra && terraform init -input=false && terraform validate && echo 'validating terraform/service' && cd ../service && terraform init -input=false && terraform validate", "prettier": "prettier --config .prettierrc.js --write ..", "prettier-check": "prettier --config .prettierrc.js --check --log-level=warn ..", - "start-proxy": "node gcs-proxy-server.js", - "test": "env NODE_ENV=test jasmine", + "start-proxy": "node --enable-source-maps build/gcs-proxy-server.js", + "test": "env NODE_ENV=test NODE_OPTIONS=--enable-source-maps jasmine", + "test:fancy": "env NODE_ENV=test jasmine --reporter=jasmine-console-reporter", "eslint": "eslint *.js", "eslint-fix": "eslint --fix *.js", "prepare": "{ git rev-parse --is-inside-work-tree >/dev/null 2>/dev/null && test \"$NODE_ENV\" != production -a \"$CI\" != true && cd .. && husky cloudrun-malware-scanner/.husky ; } || echo 'skipping husky setup'", - "typecheck": "tsc --project jsconfig.json --maxNodeModuleJsDepth 0 --noEmit" + "build": "tsc --build", + "pretest": "npm run build" }, "author": "Google Inc.", "license": "Apache-2.0", @@ -45,6 +47,7 @@ "globals": "^15.13.0", "husky": "^9.1.7", "jasmine": "^5.5.0", + "jasmine-console-reporter": "^3.1.0", "prettier": "^3.4.2", "typescript": "^5.7.2" } diff --git a/cloudrun-malware-scanner/scanner.ts b/cloudrun-malware-scanner/scanner.ts index 6de55ea..ed66547 100644 --- a/cloudrun-malware-scanner/scanner.ts +++ b/cloudrun-malware-scanner/scanner.ts @@ -1,30 +1,26 @@ /* -* Copyright 2024 Google LLC - -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at - -* https://www.apache.org/licenses/LICENSE-2.0 - -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -const {logger} = require('./logger.js'); + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -/** @typedef {import('./config.js').Config} Config */ -/** @typedef {import('./config.js').BucketDefs} BucketDefs */ -/** @typedef {import('node:stream').Readable} Readable */ -/** @typedef {typeof import('./metrics.js')} MetricsClient */ -/** @typedef {import('@google-cloud/storage').Storage} Storage */ -/** @typedef {import('@google-cloud/storage').File} File */ +import {logger} from './logger'; +import {Config, BucketDefs} from './config.js'; +import * as gcs from '@google-cloud/storage'; +import * as ClamdClient from 'clamdjs'; +import * as metrics from './metrics'; -// @ts-ignore -- TS7016: Could not find a declaration file -/** @typedef {typeof import('clamdjs')} ClamdClient */ +type MetricsClient = typeof metrics; const CLAMD_HOST = '127.0.0.1'; const CLAMD_PORT = 3310; @@ -43,71 +39,57 @@ const CLAMD_TIMEOUT = 600000; // large enough. const MAX_FILE_SIZE = 500000000; // 500MiB -/** - * StorageObjectData object defined at: - * https://github.com/googleapis/google-cloudevents/blob/main/proto/google/events/cloud/storage/v1/data.proto - * - * @typedef {{ - * name: string, - * bucket: string, - * size: string | number - * }} StorageObjectData - * - * @typedef {{ - * status: string, - * message: string, - * clam_version?: string - * }} ScanResponse - */ +export interface StorageObjectData { + name: string; + bucket: string; + size: string | number; +} -/** - * the clamd scanner.scanStream function signature. - * @typedef {(stream: Readable, timeout: number) => Promise } ScanStreamFunc - */ +export interface ScanResponse { + status: string; + message: string; + clam_version?: string; +} -/** - * Class to encapsulate the interface with ClamD and the scanning process. - */ -class Scanner { - /** - * @param {Config} config - * @param {ClamdClient} clamdClient - * @param {Storage} storageClient - * @param {MetricsClient} metricsClient - */ - constructor(config, clamdClient, storageClient, metricsClient) { +export class Scanner { + config: Config; + clamdClient: typeof ClamdClient; + storageClient: gcs.Storage; + metricsClient: MetricsClient; + clamdScanStream: ClamdClient.ScanStreamFunc; + + constructor( + config: Config, + clamdClient: typeof ClamdClient, + storageClient: gcs.Storage, + metricsClient: MetricsClient, + ) { this.config = config; this.clamdClient = clamdClient; this.storageClient = storageClient; this.metricsClient = metricsClient; - this.clamdScanStream = /** @type {ScanStreamFunc} */ ( - /** @type {any} */ (clamdClient.createScanner(CLAMD_HOST, CLAMD_PORT)) - .scanStream - ); + this.clamdScanStream = clamdClient.createScanner( + CLAMD_HOST, + CLAMD_PORT, + ).scanStream; } - /** - * Wrapper to get a clean string with the version of CLAM. - * @return {Promise} - */ - async getClamVersion() { + async getClamVersion(): Promise { return (await this.clamdClient.version(CLAMD_HOST, CLAMD_PORT)).replace( '\x00', '', ); } - /** Wrapper to return on ping success or throw. */ - async pingClamD() { + async pingClamD(): Promise { if (!(await this.clamdClient.ping(CLAMD_HOST, CLAMD_PORT))) { // ping can return false, or throw... throw new Error('clamd PING failed'); } } - /** @param {StorageObjectData} storageObject */ - validateStorageObject(storageObject) { + validateStorageObject(storageObject: StorageObjectData): void { if (storageObject == null) { throw new Error('No storage object in request'); } @@ -123,15 +105,11 @@ class Scanner { } } - /** - * Handle a POST with a GCS object payload. - * - * @param {StorageObjectData} storageObject cloud storage object data. - * @returns {Promise} - */ - async handleGcsObject(storageObject) { + async handleGcsObject( + storageObject: StorageObjectData, + ): Promise { try { - let bucketDefs; + let bucketDefs: BucketDefs | undefined; try { this.validateStorageObject(storageObject); @@ -140,10 +118,10 @@ class Scanner { )[0]; if (bucketDefs == null) { throw new Error( - 'Request has bucket name ${storageObject.bucket} which is not an unscanned bucket in config', + `Request has bucket name ${storageObject.bucket} which is not an unscanned bucket in config`, ); } - } catch (e) { + } catch (e: any) { logger.error(`Ignoring request: ${e}`); this.metricsClient.writeScanFailed(); return {message: 'ignoring invalid request', status: 'ignored'}; @@ -179,9 +157,6 @@ class Scanner { } // Check if filename is excluded: - // Iterate through the configured file exclusion patterns. - // If the file name matches any of the exclusion patterns, log an informational message and return an "ignored" status to the client. - // This allows specific files to be skipped from the scanning process based on their names. for (const regexp of this.config.fileExclusionRegexps) { if (regexp.test(storageObject.name)) { logger.info( @@ -213,9 +188,6 @@ class Scanner { return {status: 'ignored', message: 'file deleted'}; } - // Compare file size from the request body ('file.size') to the file metadata ('metadata.size'). - // If the sizes don't match, log an informational message indicating a potential incomplete file upload and return a "ignored" status to the client. - // This check helps avoid scanning partially uploaded files, which might lead to inaccurate scan results. const [metadata] = await gcsFile.getMetadata(); const metadataSize = parseInt(String(metadata.size)); if (fileSize !== metadataSize) { @@ -236,7 +208,7 @@ class Scanner { `Scan request for ${gcsFile.cloudStorageURI.href}, (${fileSize} bytes) scanning with clam ${clamdVersion}`, ); const startTime = Date.now(); - const readStream = await gcsFile.createReadStream(); + const readStream = gcsFile.createReadStream(); let result; try { result = await this.clamdScanStream(readStream, CLAMD_TIMEOUT); @@ -259,11 +231,8 @@ class Scanner { clamdVersion, ); - // Move document to the bucket that holds clean documents. This can - // fail due to permissions or if the file has been deleted. await this.moveProcessedFile(gcsFile, bucketDefs.clean); - // Respond to API client. return { status: 'clean', clam_version: clamdVersion, @@ -281,27 +250,22 @@ class Scanner { clamdVersion, ); - // Move document to the bucket that holds infected documents. This can - // fail due to permissions or if the file has been deleted. await this.moveProcessedFile(gcsFile, bucketDefs.quarantined); - // Respond to API client. return { message: result, status: 'infected', clam_version: clamdVersion, }; } - } catch (e) { + } catch (e: any) { logger.error( {err: e}, `Exception when processing gs://${storageObject.bucket}/${storageObject.name}: ${e}`, ); // Check for an API error code - const errcode = /** @type {import('@google-cloud/storage').ApiError} */ ( - e - ).code; + const errcode = e.code as number | undefined; if (errcode && [403, 404].includes(errcode)) { // Permission denied/file not found can be raised by the stream reading // and by the object move. They cannot be retried, so respond @@ -313,13 +277,10 @@ class Scanner { } } - /** - * Move the file to the appropriate bucket. - * @async - * @param {File} srcfile - * @param {string} destinationBucketName - */ - async moveProcessedFile(srcfile, destinationBucketName) { + async moveProcessedFile( + srcfile: gcs.File, + destinationBucketName: string, + ): Promise { const destinationBucket = this.storageClient.bucket(destinationBucketName); await srcfile.move(destinationBucket); @@ -328,7 +289,3 @@ class Scanner { ); } } - -module.exports = { - Scanner, -}; diff --git a/cloudrun-malware-scanner/server.ts b/cloudrun-malware-scanner/server.ts index fdedc4b..43addb6 100644 --- a/cloudrun-malware-scanner/server.ts +++ b/cloudrun-malware-scanner/server.ts @@ -1,48 +1,52 @@ /* -* Copyright 2022 Google LLC - -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at - -* https://www.apache.org/licenses/LICENSE-2.0 - -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -const process = require('node:process'); -// @ts-ignore -- TS7016: Could not find a declaration file -const clamd = require('clamdjs'); -const express = require('express'); -const {Storage} = require('@google-cloud/storage'); -const {GoogleAuth} = require('google-auth-library'); -const {logger} = require('./logger.js'); -const pkgJson = require('./package.json'); -const metrics = require('./metrics.js'); -const {Scanner} = require('./scanner.js'); -const util = require('node:util'); -const execFile = util.promisify(require('node:child_process').execFile); -const {setTimeout} = require('timers/promises'); -const {readAndVerifyConfig} = require('./config.js'); - -/** @typedef {import('./config.js').Config} Config */ -/** @typedef {import('./config.js').BucketDefs} BucketDefs */ -/** @typedef {import('express').Request} Request */ -/** @typedef {import('express').Response} Response */ -/** @typedef {import('./scanner.js').StorageObjectData} StorageObjectData */ + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as process from 'node:process'; +import * as ClamdClient from 'clamdjs'; +import * as express from 'express'; +import {Storage} from '@google-cloud/storage'; +import {GoogleAuth} from 'google-auth-library'; +import {logger} from './logger'; +import { + name as packageName, + version as packageVersion, + description as packageDescription, +} from './package.json'; +import * as metrics from './metrics'; +import {Scanner, StorageObjectData} from './scanner'; +import {promisify} from 'node:util'; +import {execFile} from 'node:child_process'; +import {setTimeout} from 'timers/promises'; +import {readAndVerifyConfig, Config} from './config'; + +const execFilePromise = promisify(execFile); /** Encapsulates the HTTP server and its methods */ class Server { + scanner: Scanner; + config: Config; + port: number; + app: express.Application; + /** * @param {Scanner} scanner * @param {Config} config * @param {number} port */ - constructor(scanner, config, port) { + constructor(scanner: Scanner, config: Config, port: number) { this.scanner = scanner; this.config = config; this.port = port; @@ -59,31 +63,26 @@ class Server { * * Use: * curl -D - -H "Authorization: Bearer $(gcloud auth print-identity-token)" CLOUD_RUN_APP_URL - * - * @param {!Response} res */ - async versionInfo(res) { + async versionInfo(res: express.Response): Promise { res .status(200) .type('text/plain') .send( - `${pkgJson.name} version ${pkgJson.version}\nUsing Clam AV version: ${await this.scanner.getClamVersion()}\n\n${pkgJson.description}\n\n`, + `${packageName} version ${packageVersion}\nUsing Clam AV version: ${await this.scanner.getClamVersion()}\n\n${packageDescription}\n\n`, ); } /** * Health check from cloud run. * Verifies that clamd is running. - * - * @param {!Response} res */ - async healthCheck(res) { + async healthCheck(res: express.Response): Promise { try { await this.scanner.pingClamD(); res.status(200).json({message: 'Health Check Suceeded'}); - } catch (e) { - const err = /** @type {Error} */ (e); - logger.fatal(err, `Health check failed to contact clamd: ${err.message}`); + } catch (e: any) { + logger.fatal(e, `Health check failed to contact clamd: ${e.message}`); res.status(500).json({message: 'Health Check Failed', status: 'error'}); } } @@ -91,11 +90,8 @@ class Server { /** * Route that is invoked by Cloud Run when a malware scan is requested * for a document uploaded to GCS. - * - * @param {!Request} req - * @param {!Response} res */ - async handlePost(req, res) { + async handlePost(req: express.Request, res: express.Response): Promise { try { switch (req.body.kind) { case 'storage#object': @@ -114,7 +110,7 @@ class Server { res.status(400).json({message: 'invalid request', status: 'error'}); break; } - } catch (e) { + } catch (e: any) { logger.error( {err: e, payload: req.body}, `Failure when processing request: ${e}`, @@ -123,10 +119,10 @@ class Server { } } - start() { + start(): void { this.app.listen(this.port, () => { logger.info( - `${pkgJson.name} version ${pkgJson.version} started on port ${this.port}`, + `${packageName} version ${packageVersion} started on port ${this.port}`, ); }); } @@ -138,10 +134,13 @@ class Server { * @param {Config} config * @returns {Promise} */ -async function handleCvdUpdate(config) { +async function handleCvdUpdate(config: Config): Promise<{ + status: string; + updated: boolean; +}> { try { logger.info('Starting CVD Mirror update'); - const result = await execFile('./updateCvdMirror.sh', [ + const result = await execFilePromise('./updateCvdMirror.sh', [ config.ClamCvdMirrorBucket, ]); logger.info('CVD Mirror update check complete. output:\n' + result.stdout); @@ -160,16 +159,13 @@ async function handleCvdUpdate(config) { status: 'CvdUpdateComplete', updated: isUpdated, }; - } catch (err) { - const e = /** @type {import('node:child_process').ExecFileException} */ ( - err - ); + } catch (err: any) { logger.error( - {err: e}, - `Failure when running ./updateCvdMirror.sh: ${e}\nstdout: ${e.stdout}\nstderr: \n${e.stderr}`, + {err}, + `Failure when running ./updateCvdMirror.sh: ${err}\nstdout: ${err.stdout}\nstderr: \n${err.stderr}`, ); metrics.writeCvdMirrorUpdated(false, false); - throw e; + throw err; } } @@ -177,7 +173,7 @@ async function handleCvdUpdate(config) { * Wait up to 5 mins for ClamD to respond * @param {Scanner} scanner */ -async function waitForClamD(scanner) { +async function waitForClamD(scanner: Scanner): Promise { const timeoutMins = 10; const endTime = Date.now() + timeoutMins * 60 * 1000; @@ -187,7 +183,7 @@ async function waitForClamD(scanner) { const version = await scanner.getClamVersion(); logger.info(`Clamd started with version ${version}`); return; - } catch (e) { + } catch (e: any) { logger.warn(`Waiting for clamd to start: ${e}`); } await setTimeout(10000); @@ -201,7 +197,7 @@ async function waitForClamD(scanner) { * * @async */ -async function run() { +async function run(): Promise { let projectId = process.env.PROJECT_ID; if (!projectId) { // Metrics needs project ID, so get it from GoogleAuth @@ -210,7 +206,7 @@ async function run() { await metrics.init(projectId); const storage = new Storage({ - userAgent: `cloud-solutions/${pkgJson.name}-usage-v${pkgJson.version}`, + userAgent: `cloud-solutions/${packageName}-usage-v${packageVersion}`, }); let configFile; @@ -222,7 +218,7 @@ async function run() { const config = await readAndVerifyConfig(configFile, storage); - const scanner = new Scanner(config, clamd, storage, metrics); + const scanner = new Scanner(config, ClamdClient, storage, metrics); await waitForClamD(scanner); diff --git a/cloudrun-malware-scanner/spec/config.spec.ts b/cloudrun-malware-scanner/spec/config.spec.ts index 30dc06b..e7faa4c 100644 --- a/cloudrun-malware-scanner/spec/config.spec.ts +++ b/cloudrun-malware-scanner/spec/config.spec.ts @@ -14,15 +14,10 @@ * limitations under the License. */ -const Config = require('../config.js'); +import * as Config from '../config.js'; +import * as gcs from '@google-cloud/storage'; -/** - * @typedef {import('@google-cloud/storage').Bucket} Bucket - * @typedef {import('@google-cloud/storage').Storage} Storage - */ - -/** @type {Config.Config} */ -const GOOD_CONFIG = { +const GOOD_CONFIG: Config.Config = { buckets: [ { unscanned: 'unscannedBucket', @@ -36,27 +31,21 @@ const GOOD_CONFIG = { }; describe('Config', () => { - /** @type {jasmine.SpyObj} */ - let storageServiceSpy; - /** @type {jasmine.SpyObj} */ - let existingBucket; + let storageServiceSpy: jasmine.SpyObj; + let existingBucket: jasmine.SpyObj; beforeEach(() => { - // Mock the 'bucket' method of the storage API object used by Config - // so that we can fake the behaviour - storageServiceSpy = /** @type {jasmine.SpyObj} */ ( - jasmine.createSpyObj('Storage', ['bucket']) - ); + storageServiceSpy = jasmine.createSpyObj('Storage', ['bucket']); - // exisiting buckets return an object to getFiles existingBucket = jasmine.createSpyObj('existingBucket', ['getFiles']); - existingBucket.getFiles.and.resolveTo(); - // buckets in GOOD_CONFIG return treat as existingBuckets + (existingBucket.getFiles as jasmine.Spy).and.resolveTo([]); [ ...Object.values(GOOD_CONFIG.buckets[0]), GOOD_CONFIG.ClamCvdMirrorBucket, ].forEach((bucket) => - storageServiceSpy.bucket.withArgs(bucket).and.returnValue(existingBucket), + (storageServiceSpy.bucket as jasmine.Spy) + .withArgs(bucket) + .and.returnValue(existingBucket), ); }); @@ -91,18 +80,15 @@ describe('Config', () => { describe('checkBucketExists', () => { beforeEach(() => { - storageServiceSpy.bucket + (storageServiceSpy.bucket as jasmine.Spy) .withArgs('existingBucket') .and.returnValue(existingBucket); - // non exisiting buckets calls to getFiles rejects - const nonExistingBucket = /** @type {jasmine.SpyObj} */ ( - jasmine.createSpyObj('bucket', ['getFiles']) - ); - storageServiceSpy.bucket + const nonExistingBucket = jasmine.createSpyObj('bucket', ['getFiles']); + (storageServiceSpy.bucket as jasmine.Spy) .withArgs('nonExistingBucket') .and.returnValue(nonExistingBucket); - nonExistingBucket.getFiles.and.rejectWith(); + nonExistingBucket.getFiles.and.rejectWith(new Error('Bucket not found')); }); it('returns false when a bucket does not exist', async () => { @@ -128,14 +114,14 @@ describe('Config', () => { describe('validateConfig', () => { it('comments are removed', async () => { const config = structuredClone(GOOD_CONFIG); - (config.comments = ['HELLO WORLD']), - expect( - await Config.TEST_ONLY.validateConfig(config, storageServiceSpy), - ).toEqual(GOOD_CONFIG); + (config as any).comments = ['HELLO WORLD']; + expect( + await Config.TEST_ONLY.validateConfig(config, storageServiceSpy), + ).toEqual(GOOD_CONFIG); }); it('populates missing values for fileExclusionRegexps and ignoreZeroLengthFiles', async () => { - const config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + const config = structuredClone(GOOD_CONFIG) as any; delete config.fileExclusionPatterns; delete config.fileExclusionRegexps; delete config.ignoreZeroLengthFiles; @@ -146,14 +132,11 @@ describe('Config', () => { }); it('missing buckets trigger failure', async () => { - // non exisiting buckets calls to getFiles rejects - const nonExistingBucket = /** @type {jasmine.SpyObj} */ ( - jasmine.createSpyObj('bucket', ['getFiles']) - ); - storageServiceSpy.bucket + const nonExistingBucket = jasmine.createSpyObj('bucket', ['getFiles']); + (storageServiceSpy.bucket as jasmine.Spy) .withArgs('nonExistingBucket') .and.returnValue(nonExistingBucket); - nonExistingBucket.getFiles.and.rejectWith(); + nonExistingBucket.getFiles.and.rejectWith(new Error('Bucket not found')); const config = structuredClone(GOOD_CONFIG); @@ -203,7 +186,7 @@ describe('Config', () => { }); it('validates ignoreZeroLengthFiles', async () => { - let config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + let config = structuredClone(GOOD_CONFIG) as any; config.ignoreZeroLengthFiles = null; let validatedConfig = await Config.TEST_ONLY.validateConfig( @@ -212,7 +195,7 @@ describe('Config', () => { ); expect(validatedConfig.ignoreZeroLengthFiles).toEqual(false); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.ignoreZeroLengthFiles = true; validatedConfig = await Config.TEST_ONLY.validateConfig( config, @@ -220,7 +203,7 @@ describe('Config', () => { ); expect(validatedConfig.ignoreZeroLengthFiles).toEqual(true); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.ignoreZeroLengthFiles = false; validatedConfig = await Config.TEST_ONLY.validateConfig( config, @@ -228,7 +211,7 @@ describe('Config', () => { ); expect(validatedConfig.ignoreZeroLengthFiles).toEqual(false); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.ignoreZeroLengthFiles = 123; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), @@ -236,31 +219,31 @@ describe('Config', () => { }); it('validates fileExclusionPatterns, rejecting invalid ones', async () => { - let config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + let config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = 123; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), ).toBeRejectedWithError('Invalid configuration'); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = [123]; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), ).toBeRejectedWithError('Invalid configuration'); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = [[123]]; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), ).toBeRejectedWithError('Invalid configuration'); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = ['invalid_regex (']; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), ).toBeRejectedWithError('Invalid configuration'); - config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = [['invalid_regex_flag', '*']]; await expectAsync( Config.TEST_ONLY.validateConfig(config, storageServiceSpy), @@ -268,23 +251,23 @@ describe('Config', () => { }); it('converts fileExclusionPatterns, to regex', async () => { - let config = /** @type {any} */ (structuredClone(GOOD_CONFIG)); + const config = structuredClone(GOOD_CONFIG) as any; config.fileExclusionPatterns = [ 'simple.*regex', ['case-insensitve.regex$', 'i'], '^[a-z0-9]\\.tmp$', ]; - let validatedConfig = await Config.TEST_ONLY.validateConfig( + const validatedConfig = await Config.TEST_ONLY.validateConfig( config, storageServiceSpy, ); expect(validatedConfig.fileExclusionPatterns).toBeUndefined(); - expect(validatedConfig.fileExclusionRegexps.length).toEqual(3); - expect(validatedConfig.fileExclusionRegexps[0]).toEqual(/simple.*regex/); - expect(validatedConfig.fileExclusionRegexps[1]).toEqual( + expect(validatedConfig.fileExclusionRegexps!.length).toEqual(3); + expect(validatedConfig.fileExclusionRegexps![0]).toEqual(/simple.*regex/); + expect(validatedConfig.fileExclusionRegexps![1]).toEqual( /case-insensitve.regex$/i, ); - expect(validatedConfig.fileExclusionRegexps[2]).toEqual( + expect(validatedConfig.fileExclusionRegexps![2]).toEqual( /^[a-z0-9]\.tmp$/, ); }); diff --git a/cloudrun-malware-scanner/spec/scanner.spec.ts b/cloudrun-malware-scanner/spec/scanner.spec.ts index fc2b594..1921838 100644 --- a/cloudrun-malware-scanner/spec/scanner.spec.ts +++ b/cloudrun-malware-scanner/spec/scanner.spec.ts @@ -1,23 +1,27 @@ -const {Scanner} = require('../scanner.js'); -const metrics = require('../metrics'); - -/** - * @typedef {import('../config.js').Config} Config - * @typedef {import('../config.js').BucketDefs} BucketDefs - * @typedef {import('node:stream').Readable} Readable - * @typedef {typeof import('../metrics.js')} MetricsClient - * @typedef {import('@google-cloud/storage').Storage} Storage - * @typedef {import('@google-cloud/storage').Bucket} Bucket - * @typedef {import('@google-cloud/storage').File} File - * @typedef {import('@google-cloud/storage').FileExistsResponse} FileExistsResponse - * @typedef {import('../scanner').ScanStreamFunc} ScanStreamFunc +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -// @ts-ignore -- TS7016: Could not find a declaration file -/** @typedef {typeof import('clamdjs')} ClamdClient */ +import {Scanner} from '../scanner'; +import * as metrics from '../metrics'; +import {Config, BucketDefs} from '../config.js'; +import {Readable} from 'node:stream'; +import * as gcs from '@google-cloud/storage'; +import * as ClamdClient from 'clamdjs'; -/** @type {Config} */ -const CONFIG = { +const CONFIG: Config = { buckets: [ { unscanned: 'unscannedBucket', @@ -34,25 +38,16 @@ const CLAMD_VERSION_STRING = 'clamd_version_string'; const TEST_FILE_NAME = 'testFileName'; describe('Scanner', () => { - /** @type {jasmine.SpyObj} */ - let clamdClient; - /** @type {jasmine.Spy} */ - let scanStreamFunc; - /** @type {jasmine.SpyObj} */ - let metricsClient; - /** @type {jasmine.SpyObj} */ - let storageClient; - /** @type {jasmine.SpyObj} */ - let mockUnscannedBucket; - /** @type {jasmine.SpyObj} */ - let mockCleanBucket; - /** @type {jasmine.SpyObj} */ - let mockQuarantinedBucket; - /** @type {jasmine.SpyObj} */ - let mockFile; - - /** @type {Scanner} */ - let scanner; + let clamdClient: jasmine.SpyObj; + let scanStreamFunc: jasmine.Spy; + let metricsClient: jasmine.SpyObj; + let storageClient: jasmine.SpyObj; + let mockUnscannedBucket: jasmine.SpyObj; + let mockCleanBucket: jasmine.SpyObj; + let mockQuarantinedBucket: jasmine.SpyObj; + let mockFile: jasmine.SpyObj; + + let scanner: Scanner; beforeEach(() => { clamdClient = jasmine.createSpyObj('ClamdClient', [ @@ -62,26 +57,34 @@ describe('Scanner', () => { 'isCleanReply', ]); scanStreamFunc = jasmine.createSpy('scanStreamFunc'); - clamdClient.createScanner.and.returnValue({scanStream: scanStreamFunc}); - // @ts-ignore -- clamdjs.version signature incorrect - should return a promise + clamdClient.createScanner.and.returnValue({ + scanStream: scanStreamFunc, + }); clamdClient.version.and.resolveTo('clamd_version_string\x00'); - // create mock Metrics metricsClient = jasmine.createSpyObj('MetricsClient', Object.keys(metrics)); storageClient = jasmine.createSpyObj('Storage', ['bucket']); - mockUnscannedBucket = jasmine.createSpyObj('unscannedBucket', ['file']); + mockUnscannedBucket = jasmine.createSpyObj('unscannedBucket', ['file'], { + name: 'unscannedBucket', + }); storageClient.bucket .withArgs('unscannedBucket') .and.returnValue(mockUnscannedBucket); - mockCleanBucket = jasmine.createSpyObj('cleanBucket', ['file']); + mockCleanBucket = jasmine.createSpyObj('cleanBucket', ['file'], { + name: 'cleanBucket', + }); storageClient.bucket .withArgs('cleanBucket') .and.returnValue(mockCleanBucket); - mockQuarantinedBucket = jasmine.createSpyObj('quarantinedBucket', ['file']); + mockQuarantinedBucket = jasmine.createSpyObj( + 'quarantinedBucket', + ['file'], + {name: 'quarantinedBucket'}, + ); storageClient.bucket .withArgs('quarantinedBucket') .and.returnValue(mockQuarantinedBucket); @@ -108,18 +111,18 @@ describe('Scanner', () => { }); it('successful pings return', async () => { - // @ts-ignore -- clamdjs.version signature incorrect - should return a promise + // @ts-ignore - incorrect ClamdClient.ping() type clamdClient.ping.and.resolveTo(true); await expectAsync(scanner.pingClamD()).toBeResolved(); }); it('unsuccessful pings throw', async () => { - // @ts-ignore -- clamdjs.version signature incorrect - should return a promise + // @ts-ignore - incorrect ClamdClient.ping() type clamdClient.ping.and.resolveTo(false); await expectAsync(scanner.pingClamD()).toBeRejectedWithError( /clamd PING failed/, ); - // @ts-ignore -- clamdjs.version signature incorrect - should return a promise + // @ts-ignore - incorrect ClamdClient.ping() type clamdClient.ping.and.throwError('exception in Ping'); await expectAsync(scanner.pingClamD()).toBeRejectedWithError( /exception in Ping/, @@ -130,13 +133,10 @@ describe('Scanner', () => { it('validates input', async () => { const response = {message: 'ignoring invalid request', status: 'ignored'}; - /** @type {import('../scanner.js').StorageObjectData} */ - // @ts-ignore -- cannot be assigned - let request = null; + let request: any = null; await expectAsync(scanner.handleGcsObject(request)).toBeResolvedTo( response, ); - // @ts-ignore -- cannot be assigned request = {}; await expectAsync(scanner.handleGcsObject(request)).toBeResolvedTo( response, @@ -228,15 +228,13 @@ describe('Scanner', () => { }); it('ignores files who get removed before processing', async () => { - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.exists.and.resolveTo([false]); + (mockFile.exists as jasmine.Spy).and.resolveTo([false]); const request = { name: TEST_FILE_NAME, bucket: 'unscannedBucket', size: 100, }; - await scanner.handleGcsObject(request); await expectAsync(scanner.handleGcsObject(request)).toBeResolvedTo({ status: 'ignored', message: 'file deleted', @@ -247,17 +245,14 @@ describe('Scanner', () => { }); it('ignores files whos size changes before processing', async () => { - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.exists.and.resolveTo([true]); - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.getMetadata.and.resolveTo([{size: 200}]); + (mockFile.exists as jasmine.Spy).and.resolveTo([true]); + (mockFile.getMetadata as jasmine.Spy).and.resolveTo([{size: 200}]); const request = { name: TEST_FILE_NAME, bucket: 'unscannedBucket', size: 100, }; - await scanner.handleGcsObject(request); await expectAsync(scanner.handleGcsObject(request)).toBeResolvedTo({ status: 'ignored', message: 'file_size_mismatch', @@ -278,18 +273,14 @@ describe('Scanner', () => { bucket: 'unscannedBucket', size: 100, }; - /** @type {jasmine.SpyObj} */ - let mockReadStream; + let mockReadStream: jasmine.SpyObj; beforeEach(() => { - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.exists.and.resolveTo([true]); - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.getMetadata.and.resolveTo([{size: 100}]); + (mockFile.exists as jasmine.Spy).and.resolveTo([true]); + (mockFile.getMetadata as jasmine.Spy).and.resolveTo([{size: 100}]); mockReadStream = jasmine.createSpyObj('readableStream', ['destroy']); - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.createReadStream.and.resolveTo(mockReadStream); + mockFile.createReadStream.and.returnValue(mockReadStream); jasmine.clock().install(); jasmine.clock().mockDate(new Date('2024-01-01T01:00:00Z')); @@ -299,12 +290,11 @@ describe('Scanner', () => { }); it('throws when getting the stream fails', async () => { - // @ts-ignore -- for some reason, the overloading is not recognised. - mockFile.createReadStream.and.rejectWith('createReadStream Fail'); + mockFile.createReadStream.and.throwError('createReadStream Fail'); - await expectAsync(scanner.handleGcsObject(request)).toBeRejectedWith( - 'createReadStream Fail', - ); + await expectAsync( + scanner.handleGcsObject(request), + ).toBeRejectedWithError('createReadStream Fail'); expect(metricsClient.writeScanFailed).toHaveBeenCalledWith( CONFIG.buckets[0].unscanned, @@ -334,8 +324,9 @@ describe('Scanner', () => { }); expect(mockReadStream.destroy).toHaveBeenCalled(); - // @ts-ignore - overloading failing to match. - expect(mockFile.move).toHaveBeenCalledWith(mockCleanBucket); + // .toHaveBeenCalledWith does not work due to overloading of func. + expect(mockFile.move.calls.count()).toBe(1); + expect(mockFile.move.calls.first().args[0]).toBe(mockCleanBucket); expect(metricsClient.writeScanClean).toHaveBeenCalledWith( CONFIG.buckets[0].unscanned, CONFIG.buckets[0].clean, @@ -357,8 +348,9 @@ describe('Scanner', () => { }); expect(mockReadStream.destroy).toHaveBeenCalled(); - // @ts-ignore - overloading failing to match. - expect(mockFile.move).toHaveBeenCalledWith(mockQuarantinedBucket); + // .toHaveBeenCalledWith does not work due to overloading of func. + expect(mockFile.move.calls.count()).toBe(1); + expect(mockFile.move.calls.first().args[0]).toBe(mockQuarantinedBucket); expect(metricsClient.writeScanInfected).toHaveBeenCalledWith( CONFIG.buckets[0].unscanned, CONFIG.buckets[0].quarantined, diff --git a/cloudrun-malware-scanner/spec/support/jasmine.json b/cloudrun-malware-scanner/spec/support/jasmine.json index 4d2193b..423cc50 100644 --- a/cloudrun-malware-scanner/spec/support/jasmine.json +++ b/cloudrun-malware-scanner/spec/support/jasmine.json @@ -1,5 +1,5 @@ { - "spec_dir": "spec", + "spec_dir": "build/spec", "spec_files": ["**/*[sS]pec.?(m)js"], "helpers": ["helpers/**/*.?(m)js"], "env": { diff --git a/cloudrun-malware-scanner/tsconfig.json b/cloudrun-malware-scanner/tsconfig.json index 3b90008..0b749f1 100644 --- a/cloudrun-malware-scanner/tsconfig.json +++ b/cloudrun-malware-scanner/tsconfig.json @@ -1,12 +1,23 @@ { "compilerOptions": { + "allowUnreachableCode": false, + "allowUnusedLabels": false, + "declaration": true, + "forceConsistentCasingInFileNames": true, + "lib": ["es2018"], "module": "commonjs", - "target": "ES6", - "checkJs": true, - "allowJs": true, - "noEmit": true, + "noEmitOnError": true, + "noFallthroughCasesInSwitch": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "outDir": "build", + "pretty": true, + "rootDir": ".", + "sourceMap": true, "strict": true, + "target": "ES2020", "resolveJsonModule": true }, - "include": ["*.js", "spec/*.js"] + "include": ["*.ts", "spec/*.ts"], + "exclude": ["node_modules"] }