refactor: move workers and indexer out of docker/ into gcp/

.gitmodules

@@ -2,7 +2,7 @@
 	path = gcp/api/googleapis
 	url = https://github.com/googleapis/googleapis
 [submodule "docker/worker/osv-test"]
-	path = docker/worker/osv-test
+	path = gcp/workers/worker/osv-test
 	url = https://github.com/oliverchang/osv-test.git
 [submodule "osv-schema"]
 	path = osv/osv-schema

Makefile

@@ -20,13 +20,13 @@ lib-tests:
 
 worker-tests:
 	git submodule update --init --recursive
-	cd docker/worker && ./run_tests.sh
+	cd gcp/workers/worker && ./run_tests.sh
 
 importer-tests:
-	cd docker/importer && ./run_tests.sh
+	cd gcp/workers/importer && ./run_tests.sh
 
 alias-tests:
-	cd docker/alias && ./run_tests.sh
+	cd gcp/workers/alias && ./run_tests.sh
 
 website-tests:
 	cd gcp/website && ./run_tests.sh

README.md

@@ -25,12 +25,14 @@ consists of:
 | directory       | what |
 |-----------------|------|
 | `deployment/`   | Terraform & Cloud Deploy config files <br /> A few Cloud Build config yamls |
-| `docker/`       | CI docker files (`ci`, `deployment`, `terraform`) <br /> Workers for bisection and impact analysis (`worker`, `importer`, `exporter`, `alias`, `worker-base`) <br /> The determine version `indexer`<br /> `cron/` jobs for database backups and processing oss-fuzz records |
+| `docker/`       | CI docker files (`ci`, `deployment`, `terraform`) <br /> `worker-base` docker image for `gcp/workers/worker` |
 | `docs/`         | Jekyll files for https://google.github.io/osv.dev/ <br /> `build_swagger.py` and `tools.go` |
 | `gcp/api`       | OSV API server files (including files for the local ESP server) <br /> protobuf files in `/v1`|
 | `gcp/datastore` | The datastore index file (`index.yaml`) |
 | `gcp/functions` | The Cloud Function for publishing PyPI vulnerabilities (maintained, but not developed) |
-| `gcp/website  ` | The backend of the osv.dev web interface, with the frontend in `frontend3` <br /> Blog posts (in `blog`) |
+| `gcp/indexer`   | The determine version `indexer` |
+| `gcp/website`   | The backend of the osv.dev web interface, with the frontend in `frontend3` <br /> Blog posts (in `blog`) |
+| `gcp/workers/`  | Workers for bisection and impact analysis (`worker`, `importer`, `exporter`, `alias`) <br /> `cron/` jobs for database backups and processing oss-fuzz records |
 | `osv/`          | The core OSV Python library, used in basically all Python services <br /> OSV ecosystem package versioning helpers in `ecosystems/` <br /> Datastore model definitions in `models.py` |
 | `tools/`        | Misc scripts/tools, mostly intended for development (datastore stuff, linting) <br /> The `indexer-api-caller` for indexer calling |
 | `vulnfeeds/`    | Go module for (mostly) the NVD CVE conversion <br /> The Alpine feed converter (`cmd/alpine`) <br /> The Debian feed converter (`tools/debian`, which is written in Python) |

cloudbuild.yaml

@@ -41,7 +41,7 @@ steps:
 
 - name: 'gcr.io/oss-vdb/ci'
   id: 'worker-tests'
-  dir: docker/worker
+  dir: gcp/workers/worker
   args: ['bash', '-ex', 'run_tests.sh']
   env:
     # Each concurrent test that uses the datastore emulator must have a unique port number
@@ -51,7 +51,7 @@ steps:
 
 - name: 'gcr.io/oss-vdb/ci'
   id: 'importer-tests'
-  dir: docker/importer
+  dir: gcp/workers/importer
   args: ['bash', '-ex', 'run_tests.sh']
   env:
     - CLOUD_BUILD=1
@@ -62,7 +62,7 @@ steps:
 
 - name: 'gcr.io/oss-vdb/ci'
   id: 'alias-tests'
-  dir: docker/alias
+  dir: gcp/workers/alias
   args: ['bash', '-ex', 'run_tests.sh']
   env:
     # same as worker/importer

deployment/build-and-stage.yaml

@@ -64,7 +64,7 @@ steps:
 
 # Build/push core worker/importer/exporter/alias images.
 - name: gcr.io/cloud-builders/docker
-  args: ['build', '-t', 'gcr.io/oss-vdb/worker:latest', '-t', 'gcr.io/oss-vdb/worker:$COMMIT_SHA', '-f', 'docker/worker/Dockerfile', '.']
+  args: ['build', '-t', 'gcr.io/oss-vdb/worker:latest', '-t', 'gcr.io/oss-vdb/worker:$COMMIT_SHA', '-f', 'gcp/workers/worker/Dockerfile', '.']
   id: 'build-worker'
   waitFor: ['build-worker-base']
 - name: gcr.io/cloud-builders/docker
@@ -73,7 +73,7 @@ steps:
 
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/importer:latest', '-t', 'gcr.io/oss-vdb/importer:$COMMIT_SHA', '.']
-  dir: 'docker/importer'
+  dir: 'gcp/workers/importer'
   id: 'build-importer'
   waitFor: ['build-worker']
 - name: gcr.io/cloud-builders/docker
@@ -82,7 +82,7 @@ steps:
 
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/exporter:latest', '-t', 'gcr.io/oss-vdb/exporter:$COMMIT_SHA', '.']
-  dir: 'docker/exporter'
+  dir: 'gcp/workers/exporter'
   id: 'build-exporter'
   waitFor: ['build-worker']
 - name: gcr.io/cloud-builders/docker
@@ -91,7 +91,7 @@ steps:
 
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/alias-computation:latest', '-t', 'gcr.io/oss-vdb/alias-computation:$COMMIT_SHA', '.']
-  dir: 'docker/alias'
+  dir: 'gcp/workers/alias'
   id: 'build-alias-computation'
   waitFor: ['build-worker']
 - name: gcr.io/cloud-builders/docker
@@ -101,7 +101,7 @@ steps:
 # Build/push staging-api-test images to gcr.io/oss-vdb-test.
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb-test/staging-api-test:latest', '-t', 'gcr.io/oss-vdb-test/staging-api-test:$COMMIT_SHA', '.']
-  dir: 'docker/staging_api_test'
+  dir: 'gcp/workers/staging_api_test'
   id: 'build-staging-api-test'
   waitFor: ['build-worker']
 - name: gcr.io/cloud-builders/docker
@@ -111,7 +111,7 @@ steps:
 # Build/push cron job images.
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/cron:latest', '-t', 'gcr.io/oss-vdb/cron:$COMMIT_SHA', '.']
-  dir: 'docker/cron'
+  dir: 'gcp/workers/cron'
   id: 'build-cron'
   waitFor: ['build-worker']
 - name: gcr.io/cloud-builders/docker
@@ -169,7 +169,7 @@ steps:
   waitFor: ['setup']
 - name: gcr.io/cloud-builders/docker
   args: ['build', '-t', 'gcr.io/oss-vdb/indexer:latest', '-t', 'gcr.io/oss-vdb/indexer:$COMMIT_SHA', '--cache-from', 'gcr.io/oss-vdb/indexer:latest', '--pull', '.']
-  dir: 'docker/indexer'
+  dir: 'gcp/indexer'
   id: 'build-indexer'
   waitFor: ['pull-indexer']
 - name: gcr.io/cloud-builders/docker

docker/worker-base/Dockerfile

@@ -72,4 +72,4 @@ RUN curl -fsSL https://gvisor.dev/archive.key | gpg --dearmor -o /etc/apt/truste
     apt-get update && apt-get install -y runsc
 
 # Make gVisor the default Docker runtime.
-COPY docker/worker/daemon.json /etc/docker/daemon.json
+COPY docker/worker-base/daemon.json /etc/docker/daemon.json

docs/contributing/architecture.md

@@ -35,7 +35,7 @@ Workers are Docker containers, which use [gVisor] for sandboxing untrusted
 workloads.
 
 [GKE]: https://cloud.google.com/kubernetes-engine
-[workers]: https://github.com/google/osv/tree/master/docker/worker
+[workers]: https://github.com/google/osv/tree/master/gcp/workers
 [gVisor]: https://gvisor.dev/
 [Cloud Pub/Sub]: https://cloud.google.com/pubsub
 

docs/faq.md

@@ -140,7 +140,7 @@ If you work on a project (like a Linux distribution) and would like to contribut
 1. Version enumeration (for non-SemVer ecosystems where [supporting version enumeration code](https://github.com/google/osv.dev/tree/master/osv/ecosystems) exists)
 2. [Package URL](https://github.com/package-url/purl-spec) [computation](https://github.com/google/osv.dev/blob/a751ceb26522f093edf26c0ad167cfd0967716d9/osv/models.py#L361-L365) (if necessary)
 3. [Git affected commit enumeration and commit to tag mapping](https://github.com/google/osv.dev/blob/a751ceb26522f093edf26c0ad167cfd0967716d9/osv/impact.py#L422)
-4. Repeat [batch](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/alias-computation.yaml) [computation](https://github.com/google/osv.dev/tree/master/docker/alias) of [aliases](https://ossf.github.io/osv-schema/#aliases-field) (**Note**: any time the `aliases` field changes, the record's [`modified`](https://ossf.github.io/osv-schema/#id-modified-fields) field is updated)
+4. Repeat [batch](https://github.com/google/osv.dev/blob/master/deployment/clouddeploy/gke-workers/base/alias-computation.yaml) [computation](https://github.com/google/osv.dev/tree/master/gcp/workers/alias) of [aliases](https://ossf.github.io/osv-schema/#aliases-field) (**Note**: any time the `aliases` field changes, the record's [`modified`](https://ossf.github.io/osv-schema/#id-modified-fields) field is updated)
 
 Both version and commit enumeration populate the [`affected.versions[]`](https://ossf.github.io/osv-schema/#affectedversions-field) field, which assists with precise version matching.
 

gcp/api/server.py

@@ -76,9 +76,7 @@
 _BUCKET_SIZE = 512
 
 # This needs to be kept in sync with
-# https://github.com/google/osv.dev/blob/
-# 666a43e6ae7690fbfa283e9a6f0b08a986be4d32/
-# docker/indexer/stages/processing/processing.go#L77
+# https://github.com/google/osv.dev/blob/master/docker/indexer/stages/processing/processing.go#L77
 _VENDORED_LIB_NAMES = frozenset((
     '3rdparty',
     'dep',

docker/indexer/go.mod → gcp/indexer/go.mod

@@ -1,4 +1,4 @@
-module github.com/google/osv.dev/docker/indexer
+module github.com/google/osv.dev/gcp/indexer
 
 go 1.23.1
 

docker/indexer/indexer.go → gcp/indexer/indexer.go

@@ -22,12 +22,12 @@ import (
 
 	"cloud.google.com/go/pubsub"
 	"cloud.google.com/go/storage"
-	"github.com/google/osv.dev/docker/indexer/config"
-	"github.com/google/osv.dev/docker/indexer/stages/preparation"
-	"github.com/google/osv.dev/docker/indexer/stages/processing"
+	"github.com/google/osv.dev/gcp/indexer/config"
+	"github.com/google/osv.dev/gcp/indexer/stages/preparation"
+	"github.com/google/osv.dev/gcp/indexer/stages/processing"
 
 	log "github.com/golang/glog"
-	idxStorage "github.com/google/osv.dev/docker/indexer/storage"
+	idxStorage "github.com/google/osv.dev/gcp/indexer/storage"
 )
 
 var (

docker/indexer/stages/preparation/preparation.go → gcp/indexer/stages/preparation/preparation.go

@@ -32,8 +32,8 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/google/osv.dev/docker/indexer/config"
-	"github.com/google/osv.dev/docker/indexer/shared"
+	"github.com/google/osv.dev/gcp/indexer/config"
+	"github.com/google/osv.dev/gcp/indexer/shared"
 	"golang.org/x/sync/semaphore"
 
 	log "github.com/golang/glog"

docker/indexer/stages/processing/processing.go → gcp/indexer/stages/processing/processing.go

@@ -33,8 +33,8 @@ import (
 	"cloud.google.com/go/pubsub"
 	"cloud.google.com/go/storage"
 	"github.com/go-git/go-git/v5"
-	"github.com/google/osv.dev/docker/indexer/shared"
-	"github.com/google/osv.dev/docker/indexer/stages/preparation"
+	"github.com/google/osv.dev/gcp/indexer/shared"
+	"github.com/google/osv.dev/gcp/indexer/stages/preparation"
 
 	log "github.com/golang/glog"
 )

docker/indexer/storage/storage.go → gcp/indexer/storage/storage.go

@@ -24,9 +24,9 @@ import (
 
 	"cloud.google.com/go/datastore"
 	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/google/osv.dev/docker/indexer/shared"
-	"github.com/google/osv.dev/docker/indexer/stages/preparation"
-	"github.com/google/osv.dev/docker/indexer/stages/processing"
+	"github.com/google/osv.dev/gcp/indexer/shared"
+	"github.com/google/osv.dev/gcp/indexer/stages/preparation"
+	"github.com/google/osv.dev/gcp/indexer/stages/processing"
 )
 
 const (

docker/indexer/storage/storage_test.go → gcp/indexer/storage/storage_test.go

@@ -20,7 +20,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/osv.dev/docker/indexer/stages/preparation"
+	"github.com/google/osv.dev/gcp/indexer/stages/preparation"
 )
 
 func getRepoInfo(t *testing.T) *preparation.Result {

docker/alias/alias_computation_test.py → gcp/workers/alias/alias_computation_test.py

@@ -20,7 +20,7 @@
 from google.protobuf import timestamp_pb2
 
 import osv
-from docker.alias import alias_computation
+import alias_computation
 from osv import tests
 
 TEST_DATA_DIR = os.path.join(

docker/importer/importer_test.py → gcp/workers/importer/importer_test.py

@@ -30,7 +30,7 @@
 from google.cloud import storage
 from google.cloud.storage import retry
 import pygit2
-from docker.mock_test.mock_test_handler import MockDataHandler
+from gcp.workers.mock_test.mock_test_handler import MockDataHandler
 import importer
 import osv
 from osv import tests

docker/staging_api_test/Dockerfile → gcp/workers/staging_api_test/Dockerfile

@@ -19,7 +19,7 @@ WORKDIR /staging_api_test
 COPY retrieve_bugs_from_bucket.py perform_api_calls.py run.sh ./
 
 # Add aiohttp lib
-RUN cd /env/docker/worker && POETRY_VIRTUALENVS_CREATE=false poetry add aiohttp
+RUN cd /env/gcp/workers/worker && POETRY_VIRTUALENVS_CREATE=false poetry add aiohttp
 
 RUN chmod 755 retrieve_bugs_from_bucket.py perform_api_calls.py run.sh
 

docker/worker/Dockerfile → gcp/workers/worker/Dockerfile

@@ -16,19 +16,19 @@ FROM gcr.io/oss-vdb/worker-base
 
 RUN apt-get update && apt-get upgrade -y
 
-RUN mkdir /work && mkdir -p /env/docker/worker
+RUN mkdir /work && mkdir -p /env/gcp/workers/worker
 VOLUME /var/lib/docker
 
 # Replicate project structure to make relative editable poetry dependency work.
 # TODO(ochang): Just copy the entire project (needs a clean checkout).
-COPY docker/worker/poetry.lock docker/worker/pyproject.toml /env/docker/worker/
+COPY gcp/workers/worker/poetry.lock gcp/workers/worker/pyproject.toml /env/gcp/workers/worker/
 COPY poetry.lock pyproject.toml README.md /env/
 COPY osv /env/osv
 
 # Set virtualenv creation to false to install globally
-RUN cd /env/docker/worker && POETRY_VIRTUALENVS_CREATE=false poetry install
+RUN cd /env/gcp/workers/worker && POETRY_VIRTUALENVS_CREATE=false poetry install
 
-COPY docker/worker/oss_fuzz.py docker/worker/worker.py /usr/local/bin/
+COPY gcp/workers/worker/oss_fuzz.py gcp/workers/worker/worker.py /usr/local/bin/
 RUN chmod 755 /usr/local/bin/worker.py
 
 ENTRYPOINT ["worker.py"]

docker/worker/build.sh → gcp/workers/worker/build.sh

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 # Build from root context.
-cd ../../
+cd ../../../
 
-docker build -t gcr.io/oss-vdb/worker:$1 -t gcr.io/oss-vdb/worker:latest -f docker/worker/Dockerfile . && \
+docker build -t gcr.io/oss-vdb/worker:$1 -t gcr.io/oss-vdb/worker:latest -f gcp/workers/worker/Dockerfile . && \
 gcloud docker -- push gcr.io/oss-vdb/worker:$1 && \
 gcloud docker -- push gcr.io/oss-vdb/worker:latest

docker/worker/pyproject.toml → gcp/workers/worker/pyproject.toml

@@ -17,7 +17,7 @@ pygit2 = "==1.16.0"
 requests = "==2.32.3"
 jsonschema = "==4.23.0"
 
-osv = { path = "../../", develop = true }
+osv = { path = "../../../", develop = true }
 
 [tool.poetry.dev-dependencies]
 yapf = "*"

docker/worker/worker_test.py → gcp/workers/worker/worker_test.py

@@ -16,7 +16,7 @@
 import codecs
 import datetime
 import hashlib
-from docker.mock_test.mock_test_handler import MockDataHandler
+from gcp.workers.mock_test.mock_test_handler import MockDataHandler
 import http.server
 import os
 import shutil

renovate.json

@@ -74,7 +74,7 @@
     },
     {
       "matchFileNames": [
-        "docker/**"
+        "gcp/workers/**"
       ],
       "matchCategories": [
         "python"
@@ -83,7 +83,7 @@
     },
     {
       "matchFileNames": [
-        "docker/**"
+        "gcp/indexer/**"
       ],
       "matchCategories": [
         "golang"