Merge branch 'main' into hotfix-merge-manifests

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -975,6 +975,22 @@ Must be one of:
 </blockquote>
 </details>
 
+<details>
+<summary>
+<strong> <a name="components_items_files_items_extractPath"></a>extractPath</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+**Description:** Local folder or file to be extracted from a 'source' archive
+
+|          |          |
+| -------- | -------- |
+| **Type** | `string` |
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 

packages/distros/eks/zarf.yaml

@@ -29,37 +29,34 @@ variables:
 components:
   - name: load-eksctl
     required: true
-    actions:
-      onDeploy:
-        after:
-          # Remove existing eksctl
-          - cmd: rm -f eksctl
-          # Extract the correct linux or mac binary from the tarball
-          - cmd: ./zarf tools archiver decompress archives/eksctl_$(uname -s)_$(uname -m).tar.gz .
-          # Cleanup temp files
-          - cmd: rm -fr archives
     files:
       - source: eks.yaml
         target: eks.yaml
       - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Darwin_amd64.tar.gz
-        target: archives/eksctl_Darwin_x86_64.tar.gz
-        shasum: d3b2a204f68eaf151b8b79bb3a28857d45d5d56353b5c430a4cd34161c8fe6fe
+        target: binaries/eksctl_Darwin_x86_64
+        executable: true
+        shasum: 6d72fe0bafa5ac62e1da3889b6987af6192b330abd9c50491bcf1c5966358f89
+        extractPath: eksctl
       - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Darwin_arm64.tar.gz
-        target: archives/eksctl_Darwin_arm64.tar.gz
-        shasum: bfc14880a3c5c8fec0e338726fdfa52e375dce0a8bfa766a34e4c4224ec5c929
+        target: binaries/eksctl_Darwin_arm64
+        executable: true
+        shasum: 1d7dd5b9907de1cb3fa7832659db29f50530444d10e77b4a8eb27aa648da6fab
+        extractPath: eksctl
       - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Linux_amd64.tar.gz
-        target: archives/eksctl_Linux_x86_64.tar.gz
-        shasum: 56e5746160381a288d5ad70846f0f0b4cd7f5d51e1dfe0880043cf120a2eb10a
+        target: binaries/eksctl_Linux_x86_64
+        executable: true
+        shasum: 2a47bb9c86c7531a166542aa2d8cb8e1e0be326308ebcfaf724d016abe31636b
+        extractPath: eksctl
 
   - name: deploy-eks-cluster
     description: Create an EKS cluster!
     actions:
       onDeploy:
         before:
-          - cmd: ./eksctl create cluster --dry-run -f eks.yaml
+          - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) create cluster --dry-run -f eks.yaml
           - cmd: sleep 15
-          - cmd: ./eksctl create cluster -f eks.yaml
-          - cmd: ./eksctl utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME}
+          - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) create cluster -f eks.yaml
+          - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME}
           - cmd: ./zarf tools kubectl create namespace zarf
           - cmd: ./zarf tools kubectl create secret generic zarf-eks-yaml -n zarf --from-file=eks.yaml
 
@@ -72,8 +69,8 @@ components:
           - cmd: ./zarf tools kubectl get secret -n zarf zarf-eks-yaml -o jsonpath='{.data.*}' | base64 -d > eks.yaml
           # TODO: Error handling in case the eks.yaml isn't what we expect ???
           # Use eksctl to delete the cluster
-          - cmd: ./eksctl delete cluster -f eks.yaml --disable-nodegroup-eviction --wait
+          - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) delete cluster -f eks.yaml --disable-nodegroup-eviction --wait
         after:
           # clean up after ourselves
+          - cmd: rm -rf binaries
           - cmd: rm -f eks.yaml
-          - cmd: rm -f eksctl

src/config/lang/english.go

@@ -29,6 +29,8 @@ const (
 	ErrRemoveFile          = "failed to remove file %s: %s"
 	ErrUnarchive           = "failed to unarchive %s: %s"
 	ErrConfirmCancel       = "confirm selection canceled: %s"
+	ErrFileExtract         = "failed to extract filename %s from archive %s: %s"
+	ErrFileNameExtract     = "failed to extract filename from URL %s: %s"
 )
 
 // Zarf CLI commands.

src/pkg/packager/create.go

@@ -5,7 +5,6 @@
 package packager
 
 import (
-	"crypto"
 	"errors"
 	"fmt"
 	"os"
@@ -295,7 +294,6 @@ func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*types.Compone
 
 	for filesIdx, file := range component.Files {
 		path := filepath.Join(componentPath.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target))
-
 		appendSBOMFiles(path)
 	}
 
@@ -384,28 +382,78 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkel
 
 		rel := filepath.Join(types.FilesFolder, strconv.Itoa(filesIdx), filepath.Base(file.Target))
 		dst := filepath.Join(componentPath.Base, rel)
+		destinationDir := filepath.Dir(dst)
 
 		if helpers.IsURL(file.Source) {
 			if isSkeleton {
 				continue
 			}
-			if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil {
-				return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error())
+
+			if file.ExtractPath != "" {
+
+				// get the compressedFileName from the source
+				compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source)
+				if err != nil {
+					return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error())
+				}
+
+				compressedFile := filepath.Join(componentPath.Temp, compressedFileName)
+
+				// If the file is an archive, download it to the componentPath.Temp
+				if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil {
+					return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error())
+				}
+
+				err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir)
+				if err != nil {
+					return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error())
+				}
+
+			} else {
+				if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil {
+					return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error())
+				}
 			}
+
 		} else {
-			if err := utils.CreatePathAndCopy(file.Source, dst); err != nil {
-				return fmt.Errorf("unable to copy file %s: %w", file.Source, err)
+			if file.ExtractPath != "" {
+				err = archiver.Extract(file.Source, file.ExtractPath, destinationDir)
+				if err != nil {
+					return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error())
+				}
+			} else {
+				if err := utils.CreatePathAndCopy(file.Source, dst); err != nil {
+					return fmt.Errorf("unable to copy file %s: %w", file.Source, err)
+				}
 			}
-			if isSkeleton {
-				p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel
+
+		}
+
+		if file.ExtractPath != "" {
+			// Make sure dst reflects the actual file or directory.
+			updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath)
+			if updatedExtractedFileOrDir != dst {
+				err = os.Rename(updatedExtractedFileOrDir, dst)
+				if err != nil {
+					return fmt.Errorf(lang.ErrWritingFile, dst, err)
+				}
 			}
 		}
 
+		if isSkeleton {
+			// Change the source to the new relative source directory (any remote files will have been skipped above)
+			p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel
+			// Remove the extractPath from a skeleton since it will already extract it
+			p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = ""
+		}
+
 		// Abort packaging on invalid shasum (if one is specified).
 		if file.Shasum != "" {
-			if actualShasum, _ := utils.GetCryptoHashFromFile(dst, crypto.SHA256); actualShasum != file.Shasum {
-				return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", file.Source, file.Shasum, actualShasum)
+			actualShasum, _ := utils.GetSHA256OfFile(dst)
+			if actualShasum != file.Shasum {
+				return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", dst, file.Shasum, actualShasum)
 			}
+
 		}
 
 		if file.Executable || utils.IsDir(dst) {

src/pkg/utils/helpers/url.go

@@ -7,6 +7,9 @@ package helpers
 import (
 	"fmt"
 	"net/url"
+	"path"
+
+	"github.com/defenseunicorns/zarf/src/config/lang"
 )
 
 // Nonstandard URL schemes or prefixes
@@ -40,3 +43,17 @@ func DoHostnamesMatch(url1 string, url2 string) (bool, error) {
 
 	return parsedURL1.Hostname() == parsedURL2.Hostname(), nil
 }
+
+// ExtractBasePathFromURL returns filename from URL string
+func ExtractBasePathFromURL(urlStr string) (string, error) {
+	if !IsURL(urlStr) {
+		return "", fmt.Errorf(lang.PkgValidateErrImportURLInvalid, urlStr)
+	}
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		return "", err
+	}
+
+	filename := path.Base(parsedURL.Path)
+	return filename, nil
+}

src/pkg/utils/helpers/url_test.go

@@ -5,6 +5,7 @@
 package helpers
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -90,6 +91,43 @@ func (suite *TestURLSuite) Test_2_DoHostnamesMatch() {
 	suite.False(b)
 }
 
+func (suite *TestURLSuite) Test_3_ExtractBasePathFromURL() {
+	goodURLs := []string{
+		"https://zarf.dev/file.txt",
+		"https://docs.zarf.dev/file.txt",
+		"https://zarf.dev/docs/file.tar.gz",
+		"https://defenseunicorns.com/file.yaml",
+		"https://google.com/file.md",
+	}
+	badURLs := []string{
+		"invalid-url",
+		"am",
+		"not",
+		"a url",
+		"[email protected]",
+		"12345",
+		"kubernetes.svc.default.svc.cluster.local",
+	}
+	expectations := []string{
+		"file.txt",
+		"file.txt",
+		"file.tar.gz",
+		"file.yaml",
+		"file.md",
+	}
+
+	for idx, url := range goodURLs {
+		actualURL, err := ExtractBasePathFromURL(url)
+		suite.NoError(err)
+		suite.Equal(actualURL, expectations[idx])
+	}
+	for _, url := range badURLs {
+		url, err := ExtractBasePathFromURL(url)
+		fmt.Println(url)
+		suite.Error(err)
+	}
+
+}
 func TestURL(t *testing.T) {
 	suite.Run(t, new(TestURLSuite))
 }

src/test/e2e/00_use_cli_test.go

@@ -116,6 +116,22 @@ func TestUseCLI(t *testing.T) {
 		require.Error(t, err, stdOut, stdErr)
 	})
 
+	t.Run("zarf package to test archive path", func(t *testing.T) {
+		t.Parallel()
+		stdOut, stdErr, err := e2e.Zarf("package", "create", "packages/distros/eks", "--confirm")
+		require.NoError(t, err, stdOut, stdErr)
+
+		path := "zarf-package-distro-eks-multi-0.0.2.tar.zst"
+		stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--confirm")
+		require.NoError(t, err, stdOut, stdErr)
+
+		require.FileExists(t, "binaries/eksctl_Darwin_x86_64")
+		require.FileExists(t, "binaries/eksctl_Darwin_arm64")
+		require.FileExists(t, "binaries/eksctl_Linux_x86_64")
+
+		e2e.CleanFiles("binaries/eksctl_Darwin_x86_64", "binaries/eksctl_Darwin_arm64", "binaries/eksctl_Linux_x86_64")
+	})
+
 	t.Run("zarf package create with tmpdir and cache", func(t *testing.T) {
 		t.Parallel()
 		tmpdir := t.TempDir()

src/types/component.go

@@ -79,11 +79,12 @@ type ZarfComponentOnlyCluster struct {
 
 // ZarfFile defines a file to deploy.
 type ZarfFile struct {
-	Source     string   `json:"source" jsonschema:"description=Local folder or file path or remote URL to pull into the package"`
-	Shasum     string   `json:"shasum,omitempty" jsonschema:"description=(files only) Optional SHA256 checksum of the file"`
-	Target     string   `json:"target" jsonschema:"description=The absolute or relative path where the file or folder should be copied to during package deploy"`
-	Executable bool     `json:"executable,omitempty" jsonschema:"description=(files only) Determines if the file should be made executable during package deploy"`
-	Symlinks   []string `json:"symlinks,omitempty" jsonschema:"description=List of symlinks to create during package deploy"`
+	Source      string   `json:"source" jsonschema:"description=Local folder or file path or remote URL to pull into the package"`
+	Shasum      string   `json:"shasum,omitempty" jsonschema:"description=(files only) Optional SHA256 checksum of the file"`
+	Target      string   `json:"target" jsonschema:"description=The absolute or relative path where the file or folder should be copied to during package deploy"`
+	Executable  bool     `json:"executable,omitempty" jsonschema:"description=(files only) Determines if the file should be made executable during package deploy"`
+	Symlinks    []string `json:"symlinks,omitempty" jsonschema:"description=List of symlinks to create during package deploy"`
+	ExtractPath string   `json:"extractPath,omitempty" jsonschema:"description=Local folder or file to be extracted from a 'source' archive"`
 }
 
 // ZarfChart defines a helm chart to be deployed.

src/ui/lib/api-types.ts

@@ -719,6 +719,10 @@ export interface ZarfFile {
      * (files only) Determines if the file should be made executable during package deploy
      */
     executable?: boolean;
+    /**
+     * Local folder or file to be extracted from a 'source' archive
+     */
+    extractPath?: string;
     /**
      * (files only) Optional SHA256 checksum of the file
      */
@@ -1590,6 +1594,7 @@ const typeMap: any = {
     ], false),
     "ZarfFile": o([
         { json: "executable", js: "executable", typ: u(undefined, true) },
+        { json: "extractPath", js: "extractPath", typ: u(undefined, "") },
         { json: "shasum", js: "shasum", typ: u(undefined, "") },
         { json: "source", js: "source", typ: "" },
         { json: "symlinks", js: "symlinks", typ: u(undefined, a("")) },

zarf.schema.json

@@ -790,6 +790,10 @@
           },
           "type": "array",
           "description": "List of symlinks to create during package deploy"
+        },
+        "extractPath": {
+          "type": "string",
+          "description": "Local folder or file to be extracted from a 'source' archive"
         }
       },
       "additionalProperties": false,