diff --git a/.github/actions/packages/action.yaml b/.github/actions/packages/action.yaml index 8ea3f42675..5405cf9110 100644 --- a/.github/actions/packages/action.yaml +++ b/.github/actions/packages/action.yaml @@ -11,7 +11,6 @@ inputs: required: false default: 'true' - runs: using: composite steps: diff --git a/.github/workflows/test-bigbang.yml b/.github/workflows/test-bigbang.yml index d8f252c3b9..871f9bfc56 100644 --- a/.github/workflows/test-bigbang.yml +++ b/.github/workflows/test-bigbang.yml @@ -37,6 +37,7 @@ jobs: - name: Build Zarf binary uses: ./.github/actions/packages with: + init-package: 'false' build-examples: 'false' - name: Setup K3d @@ -52,6 +53,9 @@ jobs: username: ${{ secrets.IRONBANK_USERNAME }} password: ${{ secrets.IRONBANK_PASSWORD }} + - name: Build a registry1.dso.mil Zarf 'init' package + run: make ib-init-package + - name: Run tests if: ${{ env.IRONBANK_USERNAME != '' }} env: diff --git a/Makefile b/Makefile index 9cf40a491a..5886418645 100644 --- a/Makefile +++ b/Makefile @@ -55,12 +55,12 @@ ensure-ui-build-dir: # INTERNAL: used to build the UI only if necessary check-ui: @ if [ ! -z "$(shell command -v shasum)" ]; then\ - if test "$(shell ./hack/print-ui-diff.sh | shasum)" != "$(shell cat build/ui/git-info.txt | shasum)" ; then\ - $(MAKE) build-ui;\ - ./hack/print-ui-diff.sh > build/ui/git-info.txt;\ - fi;\ + if test "$(shell ./hack/print-ui-diff.sh | shasum)" != "$(shell cat build/ui/git-info.txt | shasum)" ; then\ + $(MAKE) build-ui;\ + ./hack/print-ui-diff.sh > build/ui/git-info.txt;\ + fi;\ else\ - $(MAKE) build-ui;\ + $(MAKE) build-ui;\ fi build-ui: ## Build the Zarf UI @@ -119,6 +119,14 @@ init-package: ## Create the zarf init package (must `brew install coreutils` on release-init-package: $(ZARF_BIN) package create -o build -a $(ARCH) --set AGENT_IMAGE_TAG=$(AGENT_IMAGE_TAG) --confirm . +# INTERNAL: used to build an iron bank version of the init package with an ib version of the registry image +ib-init-package: + @test -s $(ZARF_BIN) || $(MAKE) build-cli + $(ZARF_BIN) package create -o build -a $(ARCH) --confirm . \ + --set REGISTRY_IMAGE_DOMAIN="registry1.dso.mil/" \ + --set REGISTRY_IMAGE="ironbank/opensource/docker/registry-v2" \ + --set REGISTRY_IMAGE_TAG="2.8.2" + build-examples: ## Build all of the example packages @test -s $(ZARF_BIN) || $(MAKE) build-cli diff --git a/src/injector/src/main.rs b/src/injector/src/main.rs index a2d449b529..ee9d0fad4f 100644 --- a/src/injector/src/main.rs +++ b/src/injector/src/main.rs @@ -12,11 +12,13 @@ use std::path::{Path, PathBuf}; use flate2::read::GzDecoder; use glob::glob; use hex::ToHex; -use rouille::{accept, router, Response}; +use rouille::{accept, router, Request, Response}; use serde_json::Value; use sha2::{Digest, Sha256}; use tar::Archive; +const DOCKER_MIME_TYPE: &str = "application/vnd.docker.distribution.manifest.v2+json"; + // Reads the binary contents of a file fn get_file(path: &PathBuf) -> io::Result> { // open the file @@ -100,77 +102,97 @@ fn start_seed_registry() { (GET) (/v2/) => { // returns empty json w/ Docker-Distribution-Api-Version header set Response::text("{}") - .with_unique_header("Content-Type", "application/json; charset=utf-8") - .with_additional_header("Docker-Distribution-Api-Version", "registry/2.0") - .with_additional_header("X-Content-Type-Options", "nosniff") - }, - - (GET) (/v2/registry/manifests/{_tag :String}) => { - handle_get_manifest(&root) - }, - - (GET) (/v2/{_namespace :String}/registry/manifests/{_ref :String}) => { - handle_get_manifest(&root) - }, - - (HEAD) (/v2/registry/manifests/{_ref :String}) => { - // a normal HEAD response has an empty body, but due to rouille not allowing for an override - // on Content-Length, we respond the same as a GET - accept!( - request, - "application/vnd.docker.distribution.manifest.v2+json" => { - handle_get_manifest(&root) - }, - "*/*" => Response::empty_406() - ) - }, - - (HEAD) (/v2/{_namespace :String}/registry/manifests/{_ref :String}) => { - // a normal HEAD response has an empty body, but due to rouille not allowing for an override - // on Content-Length, we respond the same as a GET - accept!( - request, - "application/vnd.docker.distribution.manifest.v2+json" => { - handle_get_manifest(&root) - }, - "*/*" => Response::empty_406() - ) - }, - - (GET) (/v2/registry/blobs/{digest :String}) => { - handle_get_digest(&root, &digest) - }, - - (GET) (/v2/{_namespace :String}/registry/blobs/{digest :String}) => { - handle_get_digest(&root, &digest) + .with_unique_header("Content-Type", "application/json; charset=utf-8") + .with_additional_header("Docker-Distribution-Api-Version", "registry/2.0") + .with_additional_header("X-Content-Type-Options", "nosniff") }, _ => { - Response::empty_404() + handle_request(&root, &request) } ) }) }); } +fn handle_request(root: &Path, request: &Request) -> Response { + let url = request.url(); + let url_segments: Vec<_> = url.split("/").collect(); + let url_seg_len = url_segments.len(); + + if url_seg_len >= 4 && url_segments[1] == "v2" { + let tag_index = url_seg_len - 1; + let object_index = url_seg_len - 2; + + let object_type = url_segments[object_index]; + + if object_type == "manifests" { + let tag_or_digest = url_segments[tag_index].to_owned(); + + let namespaced_name = url_segments[2..object_index].join("/"); + + // this route handles (GET) (/v2/**/manifests/) + if request.method() == "GET" { + return handle_get_manifest(&root, &namespaced_name, &tag_or_digest); + // this route handles (HEAD) (/v2/**/manifests/) + } else if request.method() == "HEAD" { + // a normal HEAD response has an empty body, but due to rouille not allowing for an override + // on Content-Length, we respond the same as a GET + return accept!( + request, + DOCKER_MIME_TYPE => { + handle_get_manifest(&root, &namespaced_name, &tag_or_digest) + }, + "*/*" => Response::empty_406() + ); + } + // this route handles (GET) (/v2/**/blobs/) + } else if object_type == "blobs" && request.method() == "GET" { + let digest = url_segments[tag_index].to_owned(); + return handle_get_digest(&root, &digest); + } + } + + Response::empty_404() +} + /// Handles the GET request for the manifest (only returns a OCI manifest regardless of Accept header) -fn handle_get_manifest(root: &Path) -> Response { +fn handle_get_manifest(root: &Path, name: &String, tag: &String) -> Response { let index = fs::read_to_string(root.join("index.json")).expect("read index.json"); let json: Value = serde_json::from_str(&index).expect("unable to parse index.json"); - let sha_manifest = json["manifests"][0]["digest"] - .as_str() - .unwrap() - .strip_prefix("sha256:") - .unwrap() - .to_owned(); - let file = File::open(&root.join("blobs").join("sha256").join(&sha_manifest)).unwrap(); - Response::from_file("application/vnd.docker.distribution.manifest.v2+json", file) - .with_additional_header( - "Docker-Content-Digest", - format!("sha256:{}", sha_manifest.to_owned()), - ) - .with_additional_header("Etag", format!("sha256:{}", sha_manifest)) - .with_additional_header("Docker-Distribution-Api-Version", "registry/2.0") + let mut sha_manifest = "".to_owned(); + + if tag.starts_with("sha256:") { + sha_manifest = tag.strip_prefix("sha256:").unwrap().to_owned(); + } else { + for manifest in json["manifests"].as_array().unwrap() { + let image_base_name = manifest["annotations"]["org.opencontainers.image.base.name"] + .as_str() + .unwrap(); + let requested_reference = name.to_owned() + ":" + tag; + if requested_reference == image_base_name { + sha_manifest = manifest["digest"] + .as_str() + .unwrap() + .strip_prefix("sha256:") + .unwrap() + .to_owned(); + } + } + } + + if sha_manifest != "" { + let file = File::open(&root.join("blobs").join("sha256").join(&sha_manifest)).unwrap(); + Response::from_file(DOCKER_MIME_TYPE, file) + .with_additional_header( + "Docker-Content-Digest", + format!("sha256:{}", sha_manifest.to_owned()), + ) + .with_additional_header("Etag", format!("sha256:{}", sha_manifest)) + .with_additional_header("Docker-Distribution-Api-Version", "registry/2.0") + } else { + Response::empty_404() + } } /// Handles the GET request for a blob diff --git a/src/internal/cluster/injector.go b/src/internal/cluster/injector.go index c887cd0120..fe45567ec3 100644 --- a/src/internal/cluster/injector.go +++ b/src/internal/cluster/injector.go @@ -134,7 +134,8 @@ func (c *Cluster) StopInjectionMadness() error { } func (c *Cluster) loadSeedImages(tempPath types.TempPaths, injectorSeedTags []string, spinner *message.Spinner) ([]transform.Image, error) { - var seedImages []transform.Image + seedImages := []transform.Image{} + tagToDigest := make(map[string]string) // Load the injector-specific images and save them as seed-images for _, src := range injectorSeedTags { @@ -152,6 +153,18 @@ func (c *Cluster) loadSeedImages(tempPath types.TempPaths, injectorSeedTags []st return seedImages, err } seedImages = append(seedImages, imgRef) + + // Get the image digest so we can set an annotation in the image.json later + imgDigest, err := img.Digest() + if err != nil { + return seedImages, err + } + // This is done _without_ the domain (different from pull.go) since the injector only handles local images + tagToDigest[imgRef.Path+imgRef.TagOrDigest] = imgDigest.String() + } + + if err := utils.AddImageNameAnnotation(tempPath.SeedImages, tagToDigest); err != nil { + return seedImages, fmt.Errorf("unable to format OCI layout: %w", err) } return seedImages, nil diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 6d57733529..ee058c6e67 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -6,7 +6,6 @@ package images import ( "context" - "encoding/json" "errors" "fmt" "os" @@ -25,7 +24,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/cache" "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/moby/moby/client" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pterm/pterm" ) @@ -140,7 +138,7 @@ func (i *ImgConfig) PullAll() error { tagToDigest[tag.String()] = imgDigest.String() } - if err := addImageNameAnnotation(i.ImagesPath, tagToDigest); err != nil { + if err := utils.AddImageNameAnnotation(i.ImagesPath, tagToDigest); err != nil { return fmt.Errorf("unable to format OCI layout: %w", err) } @@ -213,57 +211,3 @@ func (i *ImgConfig) PullImage(src string, spinner *message.Spinner) (img v1.Imag return img, nil } - -// IndexJSON represents the index.json file in an OCI layout. -type IndexJSON struct { - SchemaVersion int `json:"schemaVersion"` - Manifests []struct { - MediaType string `json:"mediaType"` - Size int `json:"size"` - Digest string `json:"digest"` - Annotations map[string]string `json:"annotations"` - } `json:"manifests"` -} - -// addImageNameAnnotation adds an annotation to the index.json file so that the deploying code can figure out what the image tag <-> digest shasum will be. -func addImageNameAnnotation(ociPath string, tagToDigest map[string]string) error { - indexPath := filepath.Join(ociPath, "index.json") - - // Read the file contents and turn it into a usable struct that we can manipulate - var index IndexJSON - byteValue, err := os.ReadFile(indexPath) - if err != nil { - return fmt.Errorf("unable to read the contents of the file (%s) so we can add an annotation: %w", indexPath, err) - } - if err = json.Unmarshal(byteValue, &index); err != nil { - return fmt.Errorf("unable to process the conents of the file (%s): %w", indexPath, err) - } - - // Loop through the manifests and add the appropriate OCI Base Image Name Annotation - for idx, manifest := range index.Manifests { - if manifest.Annotations == nil { - manifest.Annotations = make(map[string]string) - } - - var baseImageName string - - for tag, digest := range tagToDigest { - if digest == manifest.Digest { - baseImageName = tag - } - } - - if baseImageName != "" { - manifest.Annotations[ocispec.AnnotationBaseImageName] = baseImageName - index.Manifests[idx] = manifest - delete(tagToDigest, baseImageName) - } - } - - // Write the file back to the package - indexJSONBytes, err := json.Marshal(index) - if err != nil { - return err - } - return os.WriteFile(indexPath, indexJSONBytes, 0600) -} diff --git a/src/pkg/utils/image.go b/src/pkg/utils/image.go index 57c4717122..b007df3fd3 100644 --- a/src/pkg/utils/image.go +++ b/src/pkg/utils/image.go @@ -5,7 +5,10 @@ package utils import ( + "encoding/json" "fmt" + "os" + "path/filepath" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" @@ -35,3 +38,46 @@ func LoadOCIImage(imgPath, imgTag string) (v1.Image, error) { return nil, fmt.Errorf("unable to find image (%s) at the path (%s)", imgTag, imgPath) } + +// AddImageNameAnnotation adds an annotation to the index.json file so that the deploying code can figure out what the image tag <-> digest shasum will be. +func AddImageNameAnnotation(ociPath string, tagToDigest map[string]string) error { + indexPath := filepath.Join(ociPath, "index.json") + + // Read the file contents and turn it into a usable struct that we can manipulate + var index ocispec.Index + byteValue, err := os.ReadFile(indexPath) + if err != nil { + return fmt.Errorf("unable to read the contents of the file (%s) so we can add an annotation: %w", indexPath, err) + } + if err = json.Unmarshal(byteValue, &index); err != nil { + return fmt.Errorf("unable to process the contents of the file (%s): %w", indexPath, err) + } + + // Loop through the manifests and add the appropriate OCI Base Image Name Annotation + for idx, manifest := range index.Manifests { + if manifest.Annotations == nil { + manifest.Annotations = make(map[string]string) + } + + var baseImageName string + + for tag, digest := range tagToDigest { + if digest == manifest.Digest.String() { + baseImageName = tag + } + } + + if baseImageName != "" { + manifest.Annotations[ocispec.AnnotationBaseImageName] = baseImageName + index.Manifests[idx] = manifest + delete(tagToDigest, baseImageName) + } + } + + // Write the file back to the package + indexJSONBytes, err := json.Marshal(index) + if err != nil { + return err + } + return os.WriteFile(indexPath, indexJSONBytes, 0600) +} diff --git a/zarf-config.toml b/zarf-config.toml index a729ff9055..e1e2ba100e 100644 --- a/zarf-config.toml +++ b/zarf-config.toml @@ -5,7 +5,7 @@ agent_image = 'defenseunicorns/zarf/agent' agent_image_tag = 'local' # Tag for the zarf injector binary to use -injector_version = '2023-02-09' +injector_version = '2023-07-19' # The image reference to use for the registry that Zarf deploys into the cluster registry_image_domain = ''