From eb795c4564af8977ac3023d477fc3164e3679e6d Mon Sep 17 00:00:00 2001 From: Tom Mayne Date: Wed, 6 Mar 2024 16:12:08 +0000 Subject: [PATCH 1/2] fest: revert axios vuln fixes This reverts commit b1d9ccbf82711b41e20c5fe7156c88444146ab11, reversing changes made to 7e39e8cc6912a61c6bbb0df778727ab51f1562bd. --- .circleci/config.yml | 2 + .releaserc | 81 ++++++++++++++++++- README.md | 24 +++++- .../aws-automation-example.md | 7 +- docs/import-data.md | 18 ++--- docs/import.md | 14 ++-- docs/mirror-bitbucket-cloud.md | 18 ++--- docs/mirror-bitbucket-server.md | 18 ++--- docs/mirror-github.md | 20 ++--- docs/mirror-gitlab.md | 18 ++--- docs/orgs.md | 16 ++-- docs/sync.md | 24 +++--- jest.config.js | 3 - package.json | 6 +- 14 files changed, 182 insertions(+), 87 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0d242cd..f66e598a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,6 +18,8 @@ jobs: - run: npm install - run: npm test - run: npx tsc + - run: npm run pkg-binaries-linux + - run: ./snyk-api-import-linux help build-test-from-fork: docker: - image: circleci/node:12 diff --git a/.releaserc b/.releaserc index 94efd72a..92e61c16 100644 --- a/.releaserc +++ b/.releaserc @@ -1,10 +1,89 @@ { "prepare": [ "@semantic-release/npm", + { + "//": "adds a file to identify a build as a standalone binary", + "path": "@semantic-release/exec", + "cmd": "echo '' > dist/STANDALONE" + }, + { + "//": "build the macos", + "path": "@semantic-release/exec", + "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t mac-x64-12.18.2 -o snyk-api-import-macos" + }, + { + "//": "build the linux", + "path": "@semantic-release/exec", + "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t linux-x64-12.16.2 -o snyk-api-import-linux" + }, + { + "//": "build the alpine", + "path": "@semantic-release/exec", + "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t alpine-x64-12.9.1 -o snyk-api-import-alpine" + }, + { + "//": "build the windows binaries", + "path": "@semantic-release/exec", + "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t windows-x64-12.18.2 -o snyk-api-import-win.exe" + }, + { + "//": "shasum all binaries", + "path": "@semantic-release/exec", + "cmd": "shasum -a 256 snyk-api-import-linux > snyk-api-import-linux.sha256 && shasum -a 256 snyk-api-import-macos > snyk-api-import-macos.sha256 && shasum -a 256 snyk-api-import-win.exe > snyk-api-import-win.exe.sha256 && shasum -a 256 snyk-api-import-alpine > snyk-api-import-alpine.sha256" + }, + { + "//": "removes the file we use to identify a build as a standalone binary", + "path": "@semantic-release/exec", + "cmd": "rm dist/STANDALONE" + } ], "publish": [ "@semantic-release/npm", - "path": "@semantic-release/github" + { + "path": "@semantic-release/github", + "assets": [ + { + "path": "./snyk-api-import-linux", + "name": "snyk-api-import-linux", + "label": "snyk-api-import-linux" + }, + { + "path": "./snyk-api-import-linux.sha256", + "name": "snyk-api-import-linux.sha256", + "label": "snyk-api-import-linux.sha256" + }, + { + "path": "./snyk-api-import-alpine", + "name": "snyk-api-import-alpine", + "label": "snyk-api-import-alpine" + }, + { + "path": "./snyk-api-import-alpine.sha256", + "name": "snyk-api-import-alpine.sha256", + "label": "snyk-api-import-alpine.sha256" + }, + { + "path": "./snyk-api-import-macos", + "name": "snyk-api-import-macos", + "label": "snyk-api-import-macos" + }, + { + "path": "./snyk-api-import-macos.sha256", + "name": "snyk-api-import-macos.sha256", + "label": "snyk-api-import-macos.sha256" + }, + { + "path": "./snyk-api-import-win.exe", + "name": "snyk-api-import-win.exe", + "label": "snyk-api-import-win.exe" + }, + { + "path": "./snyk-api-import-win.exe.sha256", + "name": "snyk-api-import-win.exe.sha256", + "label": "snyk-api-import-win.exe.sha256" + } + ] + } ], "branches": ["master"] } diff --git a/README.md b/README.md index cf4706c4..129c66ec 100644 --- a/README.md +++ b/README.md @@ -39,15 +39,33 @@ If you need to adjust concurrency you can stop the script, change the concurrenc - [AWS automation example](docs/example-workflows/aws-automation-example.md) # Installation +`snyk-api-import` CLI can be installed through multiple channels. -Snyk snyk-api-import CLI is available as an [npm package](https://www.npmjs.com/package/snyk-api-import). +## Standalone executables (macOS, Linux, Windows) -Ensure you have [node.js installed](https://nodejs.org/en/learn/getting-started/how-to-install-nodejs), the `snyk-api-import` can then be installed by running: +Use [GitHub Releases](https://github.com/snyk-tech-services/snyk-api-import/releases) to download a standalone executable of Snyk CLI for your platform. + +## More installation methods + +
+ Install with npm or Yarn + +### Install with npm or Yarn + +[Snyk snyk-api-import CLI is available as an npm package](https://www.npmjs.com/package/snyk-api-import). If you have Node.js installed locally, you can install it by running: ```bash npm install snyk-api-import@latest -g ``` +or if you are using Yarn: + +```bash +yarn global add snyk-api-import +``` + +
+ # Usage By default the `import` command will run if no command specified. - `import` - kick off a an API powered import of repos/targets into existing Snyk orgs defined in [import configuration file](./docs/import.md). 100% support available for all project types supported via [Import API](https://snyk.docs.apiary.io/#reference/import-projects/import/import-targets). @@ -91,3 +109,5 @@ The logs can be explored using [Bunyan CLI](http://trentm.com/node-bunyan/bunyan snyk-api-import supports 100% of the same integration types and project sources as the Import API documentation. If an example is not in the docs for your use case please see the API documentation

+ + diff --git a/docs/example-workflows/aws-automation-example.md b/docs/example-workflows/aws-automation-example.md index d0db219f..4d18336b 100644 --- a/docs/example-workflows/aws-automation-example.md +++ b/docs/example-workflows/aws-automation-example.md @@ -45,8 +45,7 @@ To kick off an import, your Lambda function will need your `SNYK_TOKEN`, `SNYK_O ### Suggested architecture for the Lambda function 1. Get the "repository-name" and the "image-tag" from the event -2. Install the latest snyk-api-import tool -`npm install snyk-api-import@latest -g` +2. Install or download the latest [snyk-api-import tool](https://github.com/snyk-tech-services/snyk-api-import/releases) 3. Create a targets file for the tool using your SNYK_ORG_ID, ORG_INTEGRATION_ID as explained [here](https://github.com/snyk-tech-services/snyk-api-import/blob/master/docs/import.md#1-create-the-import-projectsjson-file). The import file shsould have the neccessary fileds as needed by our [import endpoint](https://snyk.docs.apiary.io/#reference/import-projects/import-targets), for example - an import file for a new image will look something like this: ``` { @@ -64,8 +63,8 @@ To kick off an import, your Lambda function will need your `SNYK_TOKEN`, `SNYK_O 4. Set your `SNYK_TOKEN` and your `SNYK_LOG_PATH` as an enviroment variables 5. Kick off an import by running the import command with refernce to the created import file, for example: ``` -DEBUG=snyk* npx snyk-api-import import --file=path/to/import-targets.json +DEBUG=snyk* snyk-api-import import --file=path/to/import-targets.json ``` 6. Wait for the imported targets log file - "imported-targets.log" to be created 7. Check that the log indicates a successful import -8. Done! +8. Done! \ No newline at end of file diff --git a/docs/import-data.md b/docs/import-data.md index a351530d..01134436 100644 --- a/docs/import-data.md +++ b/docs/import-data.md @@ -38,9 +38,9 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Github.com:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github` - - **Github Enterprise Server:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise --sourceUrl=https://ghe.custom.com` - - **Github Enterprise Cloud:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise` + - **Github.com:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github` + - **Github Enterprise Server:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise --sourceUrl=https://ghe.custom.com` + - **Github Enterprise Cloud:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -67,8 +67,8 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Gitlab.com:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab ` - - **Hosted Gitlab:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab --sourceUrl=https://gitlab.custom.com` + - **Gitlab.com:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab ` + - **Hosted Gitlab:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab --sourceUrl=https://gitlab.custom.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -97,8 +97,8 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **dev.azure.com:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos` - - **Hosted Azure:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos --sourceUrl=https://azure.custom.com` + - **dev.azure.com:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos` + - **Hosted Azure:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos --sourceUrl=https://azure.custom.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -127,7 +127,7 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Bitbucket Server:** `DEBUG=snyk* BITBUCKET_SERVER_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-server --sourceUrl=https://bitbucket-server.dev.example.com` + - **Bitbucket Server:** `DEBUG=snyk* BITBUCKET_SERVER_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-server --sourceUrl=https://bitbucket-server.dev.example.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -160,7 +160,7 @@ export BITBUCKET_CLOUD_PASSWORD=your_bitbucket_cloud_password - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Bitbucket Cloud:** `DEBUG=snyk* BITBUCKET_CLOUD_USERNAME=*** BITBUCKET_CLOUD_PASSWORD=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-cloud` + - **Bitbucket Cloud:** `DEBUG=snyk* BITBUCKET_CLOUD_USERNAME=*** BITBUCKET_CLOUD_PASSWORD=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-cloud` 4. Use the generated data to feed into [import command](/import.md) to generate kick off the import. diff --git a/docs/import.md b/docs/import.md index 6fe0147a..9526ef2c 100644 --- a/docs/import.md +++ b/docs/import.md @@ -192,11 +192,9 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit. - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` -## 3. Install and run +## 3. Download & run -Install the tool from [npm](https://www.npmjs.com/): -`npm install snyk-api-import@latest -g` -and run with `DEBUG=snyk* npx snyk-api-import import --file=path/to/imported-targets.json` +Grab a binary from the [releases page](https://github.com/snyk-tech-services/snyk-api-import/releases) and run with `DEBUG=snyk* snyk-api-import-macos import --file=path/to/imported-targets.json` ## 4. Review logs When import is started via Snyk API, many files & targets will be added to an import job. This job when complete will provide logs of what projects could be detected, which failed and any errors that were encountered. For more details see [Import API documentation](https://snyk.docs.apiary.io/#reference/import-projects/import/import-targets) @@ -229,11 +227,11 @@ Note: Command to run: - skip all previously imported into all orgs in a Group: - `npx snyk-api-import list:imported --integrationType= --groupId=` + `snyk-api-import-macos list:imported --integrationType= --groupId=` - skip all previously imported for a specific Organization: - `npx snyk-api-import list:imported --integrationType= --orgId=` -- a single integration / projects source `npx snyk-api-import list:imported --integrationType= --groupId=` -- multiple integrations / projects sources `npx snyk-api-import list:imported --integrationType= --integrationType= --orgId=` + `snyk-api-import-macos list:imported --integrationType= --orgId=` +- a single integration / projects source `snyk-api-import-macos list:imported --integrationType= --groupId=` +- multiple integrations / projects sources `snyk-api-import-macos list:imported --integrationType= --integrationType= --orgId=` Supported integration types: diff --git a/docs/mirror-bitbucket-cloud.md b/docs/mirror-bitbucket-cloud.md index e2b65ee9..1416f79b 100644 --- a/docs/mirror-bitbucket-cloud.md +++ b/docs/mirror-bitbucket-cloud.md @@ -4,17 +4,17 @@ You will need to configure Bitbucket Cloud username and password and Snyk token Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export BITBUCKET_CLOUD_USERNAME=***`, `export BITBUCKET_CLOUD_PASSWORD=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export BITBUCKET_CLOUD_USERNAME=***`, `export BITBUCKET_CLOUD_PASSWORD=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=bitbucket-cloud --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`npx snyk-api-import list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* npx snyk-api-import import` [Full instructions](./import.md) +`snyk-api-import-macos list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* snyk-api-import import` [Full instructions](./import.md) \ No newline at end of file diff --git a/docs/mirror-bitbucket-server.md b/docs/mirror-bitbucket-server.md index 480568a4..474eedb1 100644 --- a/docs/mirror-bitbucket-server.md +++ b/docs/mirror-bitbucket-server.md @@ -4,17 +4,17 @@ You will need to configure both Bitbucket Server token and Snyk token as environ Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export BITBUCKET_SERVER_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=bitbucket-server --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `snyk-api-import orgs:data --source=bitbucket-server --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export BITBUCKET_SERVER_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=bitbucket-server --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=bitbucket-server --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`npx snyk-api-import list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +`snyk-api-import-macos list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) diff --git a/docs/mirror-github.md b/docs/mirror-github.md index d3bda9ec..51135b06 100644 --- a/docs/mirror-github.md +++ b/docs/mirror-github.md @@ -9,22 +9,22 @@ You will need to configure both Github token and Snyk token as environment varia Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export GITHUB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=github --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `snyk-api-import orgs:data --source=github --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & organizations only while mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export GITHUB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=github --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=github --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) 5. Generate the previously imported log to skip all previously imported repos in a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`npx snyk-api-import list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +`snyk-api-import-macos list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) ## Syncing previously imported repos For repos already monitored in Snyk use the `sync` command to detect changes and update projects in Snyk. 1. Get a list of Snyk Organizations in the Group by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) -2. For every public Organization ID, run `sync` command [Full instructions](./sync.md) +2. For every public Organization ID, run `sync` command [Full instructions](./sync.md) \ No newline at end of file diff --git a/docs/mirror-gitlab.md b/docs/mirror-gitlab.md index 18352586..b53563c5 100644 --- a/docs/mirror-gitlab.md +++ b/docs/mirror-gitlab.md @@ -4,17 +4,17 @@ You will need to configure both Gitlab token and Snyk token as environment varia Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export GITLAB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=gitlab --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab --integrationType=gitlab` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `snyk-api-import orgs:data --source=gitlab --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab --integrationType=gitlab` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export GITLAB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=gitlab --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=gitlab --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`npx snyk-api-import list:imported --integrationType=gitlab --groupId=` -6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) +`snyk-api-import-macos list:imported --integrationType=gitlab --groupId=` +6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) diff --git a/docs/orgs.md b/docs/orgs.md index 8922a224..5e782210 100644 --- a/docs/orgs.md +++ b/docs/orgs.md @@ -38,8 +38,8 @@ This is an opinionated util and will assume every organization in Github.com / G ## Github.com / Github Enterprise 1. set the [Github.com personal access token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) as an environment variable: `export GITHUB_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - **Github.com:** `npx snyk-api-import orgs:data --source=github --groupId=` - - **Github Enterprise:** `npx snyk-api-import orgs:data --source=github-enterprise --groupId= -- sourceUrl=https://ghe.custom.github.com/` + - **Github.com:** `snyk-api-import orgs:data --source=github --groupId=` + - **Github Enterprise:** `snyk-api-import orgs:data --source=github-enterprise --groupId= -- sourceUrl=https://ghe.custom.github.com/` This will create the organization data in a file `group--github--orgs.json` @@ -47,8 +47,8 @@ This will create the organization data in a file `group--github-< ## Gitlab.com / Hosted Gitlab 1. set the [Gitlab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) as an environment variable: `export GITLAB_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - **Gitlab:** `npx snyk-api-import orgs:data --source=gitlab --groupId=` - - **Hosted Gitlab:** `npx snyk-api-import orgs:data --source=gitlab --groupId= -- sourceUrl=https://gitlab.custom.com` + - **Gitlab:** `snyk-api-import orgs:data --source=gitlab --groupId=` + - **Hosted Gitlab:** `snyk-api-import orgs:data --source=gitlab --groupId= -- sourceUrl=https://gitlab.custom.com` This will create the organization data in a file `group--gitlab-orgs.json`. Both groups & sub-groups will be [listed](https://docs.gitlab.com/ee/api/groups.html) and then these will become Organizations in Snyk. @@ -57,7 +57,7 @@ This will create the organization data in a file `group--gitlab-o **Please note that Bitbucket Server is a hosted environment and you must provide the custom URL for your Bitbucket Server instance in the command** 1. set the [Bitbucket Server access token](https://www.jetbrains.com/help/youtrack/standalone/integration-with-bitbucket-server.html#enable-youtrack-integration-bbserver) as an environment variable: `export BITBUCKET_SERVER_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - `npx snyk-api-import orgs:data --source=bitbucket-server --groupId= --sourceUrl=https://bitbucket-server.custom.com` + - `snyk-api-import orgs:data --source=bitbucket-server --groupId= --sourceUrl=https://bitbucket-server.custom.com` This will create the organization data in a file `group--bitbucket-server-orgs.json` @@ -66,7 +66,7 @@ This will create the organization data in a file `group--bitbucke **Note that the URL for Bitbucket Cloud is https://bitbucket.org/** 1. set the Bitbucket Cloud Username and Password as an environment variables: `export BITBUCKET_CLOUD_USERNAME=your_bitbucket_cloud_username` and `export BITBUCKET_CLOUD_PASSWORD=your_bitbucket_cloud_password` 2. Run the command to generate organization data: - - `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` + - `snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` This will create the organization data in a file `group--bitbucket-cloud-orgs.json` @@ -102,11 +102,11 @@ Use the generated data to feed into Snyk [Orgs API](https://snyk.docs.apiary.io/ ## via `orgs:create` util 1. set the `SNYK_TOKEN` environment variable - your [Snyk api token](https://app.snyk.io/account) 2. Run the command to create Orgs: -`npx snyk-api-import orgs:create --noDuplicateNames --includeExistingOrgsInOutput --file=group--github--orgs.json` +`snyk-api-import orgs:create --noDuplicateNames --includeExistingOrgsInOutput --file=group--github--orgs.json` - Using the `noDuplicateNames` flag (optional) will Skip creating an organization if the given name is already taken within the Group. - Using the `includeExistingOrgsInOutput` flag (optional, default is "true") will Log existing organization information as well as newly created. To set this flag as false, please use "--no-includeExistingOrgsInOutput" in the command, like so: -`npx snyk-api-import orgs:create --no-includeExistingOrgsInOutput --file=group--github--orgs.json` +`snyk-api-import orgs:create --no-includeExistingOrgsInOutput --file=group--github--orgs.json` The file format required for this looks like so: ``` diff --git a/docs/sync.md b/docs/sync.md index 6228939b..1f48aca6 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -95,11 +95,9 @@ The command will produce detailed logs for projects that were `updated` and thos - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` - `GITHUB_TOKEN` - SCM token that has read level or similar permissions to see information about repos like default branch & can list files in a repo -## 2. Install & run +## 2. Download & run -Install the tool from [npm](https://www.npmjs.com/): -`npm install snyk-api-import@latest -g` -and run with `DEBUG=snyk* npx snyk-api-import import --file=path/to/imported-targets.json` +Grab a binary from the [releases page](https://github.com/snyk-tech-services/snyk-api-import/releases) and run with `DEBUG=snyk* snyk-api-import-macos import --file=path/to/imported-targets.json` ## 3. Review logs @@ -122,39 +120,39 @@ When running `sync` in `--dryRun` mode the logs will have `dryRun` as `true` so ### Github.com In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github --exclusionGlobs=**/package.json,logs --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github --exclusionGlobs=**/package.json,logs --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github` ### GitHub Enterprise Server In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --exclusionGlobs=**/*.yaml,logs` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --exclusionGlobs=**/*.yaml,logs` ### GitHub Enterprise Cloud In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise` ### Only syncing Container projects (Dockerfiles) `--snykProduct` can be used to specify to sync projects belonging to Open Source, Container (Dockerfiles) or IaC products which represent files in Git repos. -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=container` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=container` ### Only syncing Open Source + Iac projects (Dockerfiles) -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac` ### Exclude from syncing certain files & directories -`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac --exclusionGlobs=**/*.yaml,logs,system-test` +`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac --exclusionGlobs=**/*.yaml,logs,system-test` # Known limitations diff --git a/jest.config.js b/jest.config.js index f8cdc48d..039da88a 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,7 +3,4 @@ module.exports = { testEnvironment: 'node', collectCoverageFrom: ['lib/**/*.ts'], coverageReporters: ['text-summary', 'html'], - moduleNameMapper: { - '^axios$': 'axios/dist/node/axios.cjs', - }, }; diff --git a/package.json b/package.json index d142cb32..fa86f14b 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "build": "tsc", "build-watch": "tsc -w", "prepare": "npm run build", - "snyk-test": "snyk test" + "snyk-test": "snyk test", + "pkg-binaries-linux": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t linux-x64-12.16.2 -o snyk-api-import-linux", + "pkg-binaries": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t mac-x64-12.18.2 -o snyk-api-import-macos" }, "types": "./dist/index.d.ts", "repository": { @@ -53,7 +55,7 @@ "rimraf": "3.0.2", "simple-git": "3.16.0", "sleep-promise": "8.0.1", - "snyk-request-manager": "1.8.4", + "snyk-request-manager": "1.8.3", "source-map-support": "^0.5.16", "split": "1.0.1", "yargs": "16.2.0" From ee13115abb84e66210f99dbe8e28f120dfff75e9 Mon Sep 17 00:00:00 2001 From: Tom Mayne Date: Wed, 6 Mar 2024 16:31:06 +0000 Subject: [PATCH 2/2] fix: ignore vulns until packaging can be fixed --- .snyk | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.snyk b/.snyk index 47571b38..76bc817c 100644 --- a/.snyk +++ b/.snyk @@ -7,4 +7,14 @@ ignore: reason: there is no fix available expires: 2023-12-30T17:38:57.751Z created: 2023-11-30T17:38:57.755Z + SNYK-JS-AXIOS-6032459O: + - '*': + reason: vuln fix broke binary packaging + expires: 2024-04-05T16:28:10.379Z + created: 2024-03-06T16:28:10.387Z + SNYK-JS-AXIOS-6144788O: + - '*': + reason: vuln fix broke binary packaging + expires: 2024-04-05T16:28:56.455Z + created: 2024-03-06T16:28:56.463Z patch: {}