hcp: generate fingerprints on each new build

Fingerprints are how we link a packer build to an iteration on HCP.
These are computed automatically from the Git SHA in the current state,
and are unique to the bucket/iteration.

The main problem with this approach is that while sound in theory, it
quickly falls apart when users want to run the same build configuration
twice, but expect a new image to be created.

With the current model, this fails, as the iteration with the current
SHA already exists.

While this is solvable through environment variables, or by committing a
change to the repository, we think this is not clear enough, and causes
an extra step to what should otherwise be a simple process.

Therefore, to lower the barrier of entry into HCP, we change this
behaviour with this commit.

Now, fingerprints are randomly generated ULIDs instead of a git SHA, and
a new one is always generated, unless one is already specified in the
environment.

This makes continuation of an existing iteration a conscious choice
rather than something automatic, and virtually eliminates conflicts such
as the ones described above.

CHANGELOG.md

@@ -3,6 +3,15 @@
 ### IMPROVEMENTS:
 * bump github.com/hashicorp/hcp-sdk-go from 0.28.0 to 0.29.0.
      [GH-12163](https://github.com/hashicorp/packer/pull/12163)
+* hcp: iteration fingerprints used to be computed from the Git SHA of the repository
+       where the template is located when running packer build. This changes with this
+       release, and now fingerprints are automatically generated as a ULID. This implies
+       that continuing an existing iteration will require users to define the
+       fingerprint in the environment manually in order to adopt this behaviour,
+       otherwise, by default, a new iteration will be created. This does not impact
+       workflows where the fingerprint was defined through the
+       HCP_PACKER_ITERATION_FINGERPRINT environment variable, and these builds will work
+       exactly as they did before.
 
 ### BUG FIXES: 
 * core/hcl2: Templates with build blocks referencing an unknown source block

command/build.go

@@ -90,12 +90,14 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
-	hcpRegistry, diags := registry.New(packerStarter)
+	hcpRegistry, diags := registry.New(packerStarter, c.Ui)
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret
 	}
 
+	defer hcpRegistry.IterationStatusSummary()
+
 	err := hcpRegistry.PopulateIteration(buildCtx)
 	if err != nil {
 		return writeDiags(c.Ui, nil, hcl.Diagnostics{

internal/hcp/registry/hcl.go

@@ -3,6 +3,7 @@ package registry
 import (
 	"context"
 	"fmt"
+	"log"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2021-04-30/models"
@@ -17,6 +18,7 @@ import (
 type HCLMetadataRegistry struct {
 	configuration *hcl2template.PackerConfig
 	bucket        *Bucket
+	ui            sdkpacker.Ui
 }
 
 const (
@@ -42,6 +44,13 @@ func (h *HCLMetadataRegistry) PopulateIteration(ctx context.Context) error {
 
 	h.configuration.HCPVars["iterationID"] = cty.StringVal(iterationID)
 
+	sha, err := getGitSHA(h.configuration.Basedir)
+	if err != nil {
+		log.Printf("failed to get GIT SHA from environment, won't set as build labels")
+	} else {
+		h.bucket.Iteration.AddSHAToBuildLabels(sha)
+	}
+
 	return nil
 }
 
@@ -70,7 +79,12 @@ func (h *HCLMetadataRegistry) CompleteBuild(
 	return h.bucket.completeBuild(ctx, name, artifacts, buildErr)
 }
 
-func NewHCLMetadataRegistry(config *hcl2template.PackerConfig) (*HCLMetadataRegistry, hcl.Diagnostics) {
+// IterationStatusSummary prints a status report in the UI if the iteration is not yet done
+func (h *HCLMetadataRegistry) IterationStatusSummary() {
+	h.bucket.Iteration.iterationStatusSummary(h.ui)
+}
+
+func NewHCLMetadataRegistry(config *hcl2template.PackerConfig, ui sdkpacker.Ui) (*HCLMetadataRegistry, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 	if len(config.Builds) > 1 {
 		diags = append(diags, &hcl.Diagnostic{
@@ -127,9 +141,12 @@ func NewHCLMetadataRegistry(config *hcl2template.PackerConfig) (*HCLMetadataRegi
 		bucket.RegisterBuildForComponent(source.String())
 	}
 
+	ui.Say(fmt.Sprintf("Tracking build on HCP Packer with fingerprint %q", bucket.Iteration.Fingerprint))
+
 	return &HCLMetadataRegistry{
 		configuration: config,
 		bucket:        bucket,
+		ui:            ui,
 	}, nil
 }
 

internal/hcp/registry/hcp.go

@@ -3,6 +3,7 @@ package registry
 import (
 	"fmt"
 
+	git "github.com/go-git/go-git/v5"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/packer/hcl2template"
 	"github.com/hashicorp/packer/internal/hcp/env"
@@ -87,9 +88,15 @@ func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts)
 		})
 	}
 
-	err := bucket.Iteration.Initialize(IterationOptions{
-		TemplateBaseDir: templateDir,
-	})
+	err := bucket.Iteration.Initialize()
+	if err != nil {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to initialize iteration",
+			Detail: fmt.Sprintf("The iteration failed to be initialized for bucket %q: %s",
+				bucket.Slug, err),
+		})
+	}
 
 	if err != nil {
 		diags = append(diags, &hcl.Diagnostic{
@@ -109,3 +116,35 @@ func withPackerEnvConfiguration(bucket *Bucket) hcl.Diagnostics {
 
 	return nil
 }
+
+// getGitSHA returns the HEAD commit for some template dir defined in baseDir.
+// If the base directory is not under version control an error is returned.
+func getGitSHA(baseDir string) (string, error) {
+	r, err := git.PlainOpenWithOptions(baseDir, &git.PlainOpenOptions{
+		DetectDotGit: true,
+	})
+
+	if err != nil {
+		return "", fmt.Errorf("Packer could not read the fingerprint from git.")
+	}
+
+	// The config can be used to retrieve user identity. for example,
+	// c.User.Email. Leaving in but commented because I'm not sure we care
+	// about this identity right now. - Megan
+	//
+	// c, err := r.ConfigScoped(config.GlobalScope)
+	// if err != nil {
+	//      return "", fmt.Errorf("Error setting git scope", err)
+	// }
+	ref, err := r.Head()
+	if err != nil {
+		// If we get there, we're in a Git dir, but HEAD cannot be read.
+		//
+		// This may happen when there's no commit in the git dir.
+		return "", fmt.Errorf("Packer could not read a git SHA in directory %q: %s", baseDir, err)
+	}
+
+	// log.Printf("Author: %v, Commit: %v\n", c.User.Email, ref.Hash())
+
+	return ref.Hash().String(), nil
+}

internal/hcp/registry/json.go

@@ -3,6 +3,7 @@ package registry
 import (
 	"context"
 	"fmt"
+	"log"
 	"path/filepath"
 
 	"github.com/hashicorp/hcl/v2"
@@ -15,9 +16,10 @@ import (
 type JSONMetadataRegistry struct {
 	configuration *packer.Core
 	bucket        *Bucket
+	ui            sdkpacker.Ui
 }
 
-func NewJSONMetadataRegistry(config *packer.Core) (*JSONMetadataRegistry, hcl.Diagnostics) {
+func NewJSONMetadataRegistry(config *packer.Core, ui sdkpacker.Ui) (*JSONMetadataRegistry, hcl.Diagnostics) {
 	bucket, diags := createConfiguredBucket(
 		filepath.Dir(config.Template.Path),
 		withPackerEnvConfiguration,
@@ -41,9 +43,12 @@ func NewJSONMetadataRegistry(config *packer.Core) (*JSONMetadataRegistry, hcl.Di
 		bucket.RegisterBuildForComponent(buildName)
 	}
 
+	ui.Say(fmt.Sprintf("Tracking build on HCP Packer with fingerprint %q", bucket.Iteration.Fingerprint))
+
 	return &JSONMetadataRegistry{
 		configuration: config,
 		bucket:        bucket,
+		ui:            ui,
 	}, nil
 }
 
@@ -63,6 +68,13 @@ func (h *JSONMetadataRegistry) PopulateIteration(ctx context.Context) error {
 		return err
 	}
 
+	sha, err := getGitSHA(h.configuration.Template.Path)
+	if err != nil {
+		log.Printf("failed to get GIT SHA from environment, won't set as build labels")
+	} else {
+		h.bucket.Iteration.AddSHAToBuildLabels(sha)
+	}
+
 	return nil
 }
 
@@ -80,3 +92,8 @@ func (h *JSONMetadataRegistry) CompleteBuild(
 ) ([]sdkpacker.Artifact, error) {
 	return h.bucket.completeBuild(ctx, build.Name(), artifacts, buildErr)
 }
+
+// IterationStatusSummary prints a status report in the UI if the iteration is not yet done
+func (h *JSONMetadataRegistry) IterationStatusSummary() {
+	h.bucket.Iteration.iterationStatusSummary(h.ui)
+}

internal/hcp/registry/null_registry.go

@@ -25,3 +25,5 @@ func (r nullRegistry) CompleteBuild(
 ) ([]sdkpacker.Artifact, error) {
 	return artifacts, nil
 }
+
+func (r nullRegistry) IterationStatusSummary() {}

internal/hcp/registry/registry.go

@@ -12,25 +12,25 @@ import (
 
 // Registry is an entity capable to orchestrate a Packer build and upload metadata to HCP
 type Registry interface {
-	//Configure(packer.Handler)
 	PopulateIteration(context.Context) error
 	StartBuild(context.Context, sdkpacker.Build) error
 	CompleteBuild(ctx context.Context, build sdkpacker.Build, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error)
+	IterationStatusSummary()
 }
 
 // New instanciates the appropriate registry for the Packer configuration template type.
 // A nullRegistry is returned for non-HCP Packer registry enabled templates.
-func New(cfg packer.Handler) (Registry, hcl.Diagnostics) {
+func New(cfg packer.Handler, ui sdkpacker.Ui) (Registry, hcl.Diagnostics) {
 	if !IsHCPEnabled(cfg) {
 		return &nullRegistry{}, nil
 	}
 
 	switch config := cfg.(type) {
 	case *hcl2template.PackerConfig:
 		// Maybe rename to what it represents....
-		return NewHCLMetadataRegistry(config)
+		return NewHCLMetadataRegistry(config, ui)
 	case *packer.Core:
-		return NewJSONMetadataRegistry(config)
+		return NewJSONMetadataRegistry(config, ui)
 	}
 
 	return nil, hcl.Diagnostics{

internal/hcp/registry/types.bucket_service_test.go

@@ -9,7 +9,6 @@ import (
 )
 
 func TestInitialize_NewBucketNewIteration(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 
 	b := &Bucket{
@@ -20,7 +19,7 @@ func TestInitialize_NewBucketNewIteration(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -62,7 +61,6 @@ func TestInitialize_NewBucketNewIteration(t *testing.T) {
 }
 
 func TestInitialize_UnsetTemplateTypeError(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testunsettemplate")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 
@@ -74,7 +72,7 @@ func TestInitialize_UnsetTemplateTypeError(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -88,7 +86,6 @@ func TestInitialize_UnsetTemplateTypeError(t *testing.T) {
 }
 
 func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 
@@ -100,7 +97,7 @@ func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -142,7 +139,6 @@ func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
 }
 
 func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 	mockService.IterationAlreadyExist = true
@@ -155,7 +151,7 @@ func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -212,7 +208,6 @@ func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
 }
 
 func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 	mockService.IterationAlreadyExist = true
@@ -227,7 +222,7 @@ func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -258,7 +253,6 @@ func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
 }
 
 func TestUpdateBuildStatus(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 	mockService.IterationAlreadyExist = true
@@ -271,7 +265,7 @@ func TestUpdateBuildStatus(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}
@@ -312,7 +306,6 @@ func TestUpdateBuildStatus(t *testing.T) {
 }
 
 func TestUpdateBuildStatus_DONENoImages(t *testing.T) {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
 	mockService := api.NewMockPackerClientService()
 	mockService.BucketAlreadyExist = true
 	mockService.IterationAlreadyExist = true
@@ -325,7 +318,7 @@ func TestUpdateBuildStatus_DONENoImages(t *testing.T) {
 	}
 
 	b.Iteration = NewIteration()
-	err := b.Iteration.Initialize(IterationOptions{})
+	err := b.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("unexpected failure: %v", err)
 	}

internal/hcp/registry/types.bucket_test.go

@@ -10,11 +10,9 @@ import (
 )
 
 func createInitialTestBucket(t testing.TB) *Bucket {
-	t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "no-fingerprint-here")
-
 	t.Helper()
 	bucket := NewBucketWithIteration()
-	err := bucket.Iteration.Initialize(IterationOptions{})
+	err := bucket.Iteration.Initialize()
 	if err != nil {
 		t.Errorf("failed to initialize Bucket: %s", err)
 		return nil
@@ -289,7 +287,7 @@ func TestBucket_PopulateIteration(t *testing.T) {
 			mockService.BuildAlreadyDone = tt.buildCompleted
 
 			bucket := NewBucketWithIteration()
-			err := bucket.Iteration.Initialize(IterationOptions{})
+			err := bucket.Iteration.Initialize()
 			if err != nil {
 				t.Fatalf("failed when calling NewBucketWithIteration: %s", err)
 			}