diff --git a/package-lock.json b/package-lock.json index be6b45f697..f4cd70646f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,9 @@ "packages/actor-scraper/*" ], "devDependencies": { + "@apify/consts": "^2.4.1", "@apify/eslint-config-ts": "^0.2.3", + "@apify/input_secrets": "^1.1.1", "@apify/tsconfig": "^0.1.0", "@commitlint/config-conventional": "^17.0.3", "@types/content-type": "^1.1.5", @@ -105,6 +107,16 @@ "typescript": "*" } }, + "node_modules/@apify/input_secrets": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@apify/input_secrets/-/input_secrets-1.1.1.tgz", + "integrity": "sha512-gY2Oxw1fX8tbiOxxqj1pEMGTpSsC2+VJtlkr7pszGD8p9ciFx/omM0iDXCu1FKJOpJ7ggbNmIxYVwBo6c2nGtA==", + "dependencies": { + "@apify/log": "^2.2.1", + "@apify/utilities": "^2.2.3", + "ow": "^0.28.2" + } + }, "node_modules/@apify/log": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.2.1.tgz", @@ -15767,6 +15779,7 @@ } }, "packages/actor-scraper/cheerio-scraper": { + "name": "actor-cheerio-scraper", "version": "3.0.0", "license": "Apache-2.0", "dependencies": { @@ -15785,6 +15798,7 @@ } }, "packages/actor-scraper/playwright-scraper": { + "name": "actor-playwright-scraper", "version": "3.0.0", "license": "Apache-2.0", "dependencies": { @@ -15803,6 +15817,7 @@ } }, "packages/actor-scraper/puppeteer-scraper": { + "name": "actor-puppeteer-scraper", "version": "3.0.0", "license": "Apache-2.0", "dependencies": { @@ -15819,6 +15834,7 @@ } }, "packages/actor-scraper/web-scraper": { + "name": "actor-web-scraper", "version": "3.0.0", "license": "Apache-2.0", "dependencies": { @@ -15841,10 +15857,11 @@ "version": "3.0.2", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.3.0", + "@apify/consts": "^2.4.1", + "@apify/input_secrets": "^1.1.0", "@apify/log": "^2.1.4", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.1.5", + "@apify/utilities": "^2.2.2", "@crawlee/core": "^3.0.0", "@crawlee/types": "^3.0.0", "@crawlee/utils": "^3.0.0", @@ -15858,6 +15875,7 @@ } }, "packages/scraper-tools": { + "name": "@apify/scraper-tools", "version": "1.0.0", "license": "Apache-2.0", "dependencies": { @@ -15939,6 +15957,16 @@ "eslint-plugin-react-hooks": "^4.3.0" } }, + "@apify/input_secrets": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@apify/input_secrets/-/input_secrets-1.1.1.tgz", + "integrity": "sha512-gY2Oxw1fX8tbiOxxqj1pEMGTpSsC2+VJtlkr7pszGD8p9ciFx/omM0iDXCu1FKJOpJ7ggbNmIxYVwBo6c2nGtA==", + "requires": { + "@apify/log": "^2.2.1", + "@apify/utilities": "^2.2.3", + "ow": "^0.28.2" + } + }, "@apify/log": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.2.1.tgz", @@ -19714,10 +19742,11 @@ "apify": { "version": "file:packages/apify", "requires": { - "@apify/consts": "^2.3.0", + "@apify/consts": "^2.4.1", + "@apify/input_secrets": "^1.1.0", "@apify/log": "^2.1.4", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.1.5", + "@apify/utilities": "^2.2.2", "@crawlee/core": "^3.0.0", "@crawlee/types": "^3.0.0", "@crawlee/utils": "^3.0.0", diff --git a/package.json b/package.json index 8e87c2b076..80740bb835 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,9 @@ "lint:fix": "eslint packages/*/src test --fix" }, "devDependencies": { + "@apify/consts": "^2.4.1", "@apify/eslint-config-ts": "^0.2.3", + "@apify/input_secrets": "^1.1.1", "@apify/tsconfig": "^0.1.0", "@commitlint/config-conventional": "^17.0.3", "@types/content-type": "^1.1.5", @@ -64,8 +66,6 @@ "@typescript-eslint/parser": "5.39.0", "commitlint": "^17.0.3", "crawlee": "^3.0.0", - "playwright": "*", - "puppeteer": "*", "eslint": "~8.24.0", "fs-extra": "^10.1.0", "gen-esm-wrapper": "^1.1.3", @@ -73,6 +73,8 @@ "jest": "^29.1.2", "lerna": "^5.0.0", "lint-staged": "^13.0.3", + "playwright": "*", + "puppeteer": "*", "rimraf": "^3.0.2", "ts-jest": "^29.0.3", "ts-node": "^10.8.2", diff --git a/packages/apify/package.json b/packages/apify/package.json index 6918b54a71..10d3ab361c 100644 --- a/packages/apify/package.json +++ b/packages/apify/package.json @@ -54,16 +54,17 @@ "access": "public" }, "dependencies": { - "@apify/consts": "^2.3.0", + "@apify/consts": "^2.4.1", + "@apify/input_secrets": "^1.1.0", "@apify/log": "^2.1.4", "@apify/timeout": "^0.3.0", - "@apify/utilities": "^2.1.5", + "@apify/utilities": "^2.2.2", "@crawlee/core": "^3.0.0", "@crawlee/types": "^3.0.0", "@crawlee/utils": "^3.0.0", - "semver": "^7.3.7", "apify-client": "^2.6.0", "ow": "^0.28.1", + "semver": "^7.3.7", "ws": "^7.5.9" } } diff --git a/packages/apify/src/actor.ts b/packages/apify/src/actor.ts index ef09e7de0f..e71325d459 100644 --- a/packages/apify/src/actor.ts +++ b/packages/apify/src/actor.ts @@ -1,4 +1,6 @@ import ow from 'ow'; +import { createPrivateKey } from 'node:crypto'; +import { decryptInputSecrets } from '@apify/input_secrets'; import { ENV_VARS, INTEGER_ENV_VARS } from '@apify/consts'; import { addTimeoutToPromise } from '@apify/timeout'; import log from '@apify/log'; @@ -646,7 +648,17 @@ export class Actor { * @ignore */ async getInput(): Promise { - return this.getValue(this.config.get('inputKey')); + const inputSecretsPrivateKeyFile = this.config.get('inputSecretsPrivateKeyFile'); + const inputSecretsPrivateKeyPassphrase = this.config.get('inputSecretsPrivateKeyPassphrase'); + const input = await this.getValue(this.config.get('inputKey')); + if (ow.isValid(input, ow.object.nonEmpty) && inputSecretsPrivateKeyFile && inputSecretsPrivateKeyPassphrase) { + const privateKey = createPrivateKey({ + key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'), + passphrase: inputSecretsPrivateKeyPassphrase, + }); + return decryptInputSecrets({ input, privateKey }); + } + return input; } /** diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index 76dbd0fbd9..cea9fd7f40 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -18,6 +18,8 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { proxyStatusUrl?: string; isAtHome?: boolean; userId?: string; + inputSecretsPrivateKeyPassphrase?: string; + inputSecretsPrivateKeyFile?: string; } /** @@ -129,6 +131,8 @@ export class Configuration extends CoreConfiguration { APIFY_PROXY_PASSWORD: 'proxyPassword', APIFY_PROXY_STATUS_URL: 'proxyStatusUrl', APIFY_PROXY_PORT: 'proxyPort', + APIFY_INPUT_SECRETS_PRIVATE_KEY_FILE: 'inputSecretsPrivateKeyFile', + APIFY_INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE: 'inputSecretsPrivateKeyPassphrase', }; protected static override INTEGER_VARS = [...super.INTEGER_VARS, 'proxyPort', 'containerPort', 'metamorphAfterSleepMillis']; diff --git a/test/apify/actor.test.ts b/test/apify/actor.test.ts index 3f8515f628..308e7f0b78 100644 --- a/test/apify/actor.test.ts +++ b/test/apify/actor.test.ts @@ -1,5 +1,7 @@ +import { createPublicKey } from 'node:crypto'; import { ACT_JOB_STATUSES, ENV_VARS, KEY_VALUE_STORE_KEYS, WEBHOOK_EVENT_TYPES } from '@apify/consts'; import log from '@apify/log'; +import { encryptInputSecrets } from '@apify/input_secrets'; import type { ApifyEnv } from 'apify'; import { Actor, ProxyConfiguration } from 'apify'; import type { WebhookUpdateData } from 'apify-client'; @@ -45,6 +47,14 @@ const setEnv = (env: ApifyEnv) => { if (env.memoryMbytes) process.env.APIFY_MEMORY_MBYTES = env.memoryMbytes.toString(); }; +const testingPublicKey = createPublicKey({ + // eslint-disable-next-line max-len + key: Buffer.from('LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0dis3NlNXbklhOFFKWC94RUQxRQpYdnBBQmE3ajBnQnVYenJNUU5adjhtTW1RU0t2VUF0TmpOL2xacUZpQ0haZUQxU2VDcGV1MnFHTm5XbGRxNkhUCnh5cXJpTVZEbFNKaFBNT09QSENISVNVdFI4Tk5lR1Y1MU0wYkxJcENabHcyTU9GUjdqdENWejVqZFRpZ1NvYTIKQWxrRUlRZWQ4UVlDKzk1aGJoOHk5bGcwQ0JxdEdWN1FvMFZQR2xKQ0hGaWNuaWxLVFFZay9MZzkwWVFnUElPbwozbUppeFl5bWFGNmlMZTVXNzg1M0VHWUVFVWdlWmNaZFNjaGVBMEdBMGpRSFVTdnYvMEZjay9adkZNZURJOTVsCmJVQ0JoQjFDbFg4OG4wZUhzUmdWZE5vK0NLMDI4T2IvZTZTK1JLK09VaHlFRVdPTi90alVMdGhJdTJkQWtGcmkKOFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==', 'base64'), +}); +// eslint-disable-next-line max-len +const testingPrivateKeyFile = 'LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpQcm9jLVR5cGU6IDQsRU5DUllQVEVECkRFSy1JbmZvOiBERVMtRURFMy1DQkMsNTM1QURERjIzNUQ4QkFGOQoKMXFWUzl0S0FhdkVhVUVFMktESnpjM3plMk1lZkc1dmVEd2o1UVJ0ZkRaMXdWNS9VZmIvcU5sVThTSjlNaGhKaQp6RFdrWExueUUzSW0vcEtITVZkS0czYWZkcFRtcis2TmtidXptd0dVMk0vSWpzRjRJZlpad0lGbGJoY09jUnp4CmZmWVIvTlVyaHNrS1RpNGhGV0lBUDlLb3Z6VDhPSzNZY3h6eVZQWUxYNGVWbWt3UmZzeWkwUU5Xb0tGT3d0ZC8KNm9HYzFnd2piRjI5ZDNnUThZQjFGWmRLa1AyMTJGbkt1cTIrUWgvbE1zTUZrTHlTQTRLTGJ3ZG1RSXExbE1QUwpjbUNtZnppV3J1MlBtNEZoM0dmWlQyaE1JWHlIRFdEVzlDTkxKaERodExOZ2RRamFBUFpVT1E4V2hwSkE5MS9vCjJLZzZ3MDd5Z2RCcVd5dTZrc0pXcjNpZ1JpUEJ5QmVNWEpEZU5HY3NhaUZ3Q2c5eFlja1VORXR3NS90WlRsTjIKSEdZV0NpVU5Ed0F2WllMUHR1SHpIOFRFMGxsZm5HR0VuVC9QQlp1UHV4andlZlRleE1mdzFpbGJRU3lkcy9HMgpOOUlKKzkydms0N0ZXR2NOdGh1Q3lCbklva0NpZ0c1ZlBlV2IwQTdpdjk0UGtwRTRJZ3plc0hGQ0ZFQWoxWldLCnpQdFRBQlkwZlJrUzBNc3UwMHYxOXloTTUrdFUwYkVCZWo2eWpzWHRoYzlwS01hcUNIZWlQTC9TSHRkaWsxNVMKQmU4Sml4dVJxZitUeGlYWWVuNTg2aDlzTFpEYzA3cGpkUGp2NVNYRnBYQjhIMlVxQ0tZY2p4R3RvQWpTV0pjWApMNHc3RHNEby80bVg1N0htR09iamlCN1ZyOGhVWEJDdFh2V0dmQXlmcEFZNS9vOXowdm4zREcxaDc1NVVwdDluCkF2MFZrbm9qcmJVYjM1ZlJuU1lYTVltS01LSnpNRlMrdmFvRlpwV0ZjTG10cFRWSWNzc0JGUEYyZEo3V1c0WHMKK0d2Vkl2eFl3S2wyZzFPTE1TTXRZa09vekdlblBXTzdIdU0yMUVKVGIvbHNEZ25GaTkrYWRGZHBLY3R2cm0zdgpmbW1HeG5pRmhLU05GU0xtNms5YStHL2pjK3NVQVBhb2FZNEQ3NHVGajh0WGp0eThFUHdRRGxVUGRVZld3SE9PClF3bVgyMys1REh4V0VoQy91Tm8yNHNNY2ZkQzFGZUpBV281bUNuVU5vUVVmMStNRDVhMzNJdDhhMmlrNUkxUWoKeSs1WGpRaG0xd3RBMWhWTWE4aUxBR0toT09lcFRuK1VBZHpyS0hvNjVtYzNKbGgvSFJDUXJabnVxWkErK0F2WgpjeWU0dWZGWC8xdmRQSTdLb2Q0MEdDM2dlQnhweFFNYnp1OFNUcGpOcElJRkJvRVc5dFRhemUzeHZXWnV6dDc0CnFjZS8xWURuUHBLeW5lM0xGMk94VWoyYWVYUW5YQkpYcGhTZTBVTGJMcWJtUll4bjJKWkl1d09RNHV5dm94NjUKdG9TWGNac054dUs4QTErZXNXR3JSN3pVc0djdU9QQTFERE9Ja2JjcGtmRUxMNjk4RTJRckdqTU9JWnhrcWdxZQoySE5VNktWRmV2NzdZeEJDbm1VcVdXZEhYMjcyU2NPMUYzdWpUdFVnRVBNWGN0aEdBckYzTWxEaUw1Q0k0RkhqCnhHc3pVemxzalRQTmpiY2MzdUE2MjVZS3VVZEI2c1h1Rk5NUHk5UDgwTzBpRWJGTXl3MWxmN2VpdFhvaUUxWVoKc3NhMDVxTUx4M3pPUXZTLzFDdFpqaFp4cVJMRW5pQ3NWa2JVRlVYclpodEU4dG94bGpWSUtpQ25qbitORmtqdwo2bTZ1anpBSytZZHd2Nk5WMFB4S0gwUk5NYVhwb1lmQk1oUmZ3dGlaS3V3Y2hyRFB5UEhBQ2J3WXNZOXdtUE9rCnpwdDNxWi9JdDVYTmVqNDI0RzAzcGpMbk1sd1B1T1VzYmFQUWQ2VHU4TFhsckZReUVjTXJDNHdjUTA1SzFVN3kKM1NNN3RFaTlnbjV3RjY1YVI5eEFBR0grTUtMMk5WNnQrUmlTazJVaWs1clNmeDE4Mk9wYmpSQ2grdmQ4UXhJdwotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo='; +const testingPrivateKeyPassphrase = 'pwd1234'; + describe('Actor', () => { const localStorageEmulator = new MemoryStorageEmulator(); @@ -937,23 +947,43 @@ describe('Actor', () => { }); describe('Actor.getInput', () => { + const TestingActor = new Actor(); + test('should work', async () => { - const defaultStore = await KeyValueStore.open(); - // Uses default value. - const oldGet = defaultStore.getValue; - // @ts-expect-error TODO use spyOn instead of this - defaultStore.getValue = async (key) => expect(key).toEqual(KEY_VALUE_STORE_KEYS.INPUT); - await Actor.getInput(); + const mockGetValue = jest.spyOn(TestingActor, 'getValue'); + mockGetValue.mockImplementation(async (key) => expect(key).toEqual(KEY_VALUE_STORE_KEYS.INPUT)); + + await TestingActor.getInput(); // Uses value from env var. process.env[ENV_VARS.INPUT_KEY] = 'some-value'; - // @ts-expect-error TODO use spyOn instead of this - defaultStore.getValue = async (key) => expect(key).toBe('some-value'); - await Actor.getInput(); + mockGetValue.mockImplementation(async (key) => expect(key).toBe('some-value')); + await TestingActor.getInput(); delete process.env[ENV_VARS.INPUT_KEY]; + mockGetValue.mockRestore(); + }); - defaultStore.getValue = oldGet; + test('should work with input secrets', async () => { + const mockGetValue = jest.spyOn(TestingActor, 'getValue'); + const originalInput = { secret: 'foo', nonSecret: 'bar' }; + const likeInputSchema = { properties: { secret: { type: 'string', isSecret: true } }, nonSecret: { type: 'string' } }; + const encryptedInput = encryptInputSecrets({ input: originalInput, inputSchema: likeInputSchema, publicKey: testingPublicKey }); + // Checks if encrypts the right value + expect(encryptedInput.secret.startsWith('ENCRYPTED_')).toBe(true); + expect(encryptedInput.nonSecret).toBe(originalInput.nonSecret); + + mockGetValue.mockImplementation(async (key) => encryptedInput); + + process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_FILE] = testingPrivateKeyFile; + process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE] = testingPrivateKeyPassphrase; + const input = await TestingActor.getInput(); + + expect(input).toStrictEqual(originalInput); + + delete process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_FILE]; + delete process.env[ENV_VARS.INPUT_SECRETS_PRIVATE_KEY_PASSPHRASE]; + mockGetValue.mockRestore(); }); });