Make it possible to overwrite image TS with label

This adds support for overwriting the image created at timestamp with
labels which are set during build. Supported labels (for now) are the
Open Container Image (OCI) spec[1] and the (legacy) Label Schema (LS)
spec[2].

We prioritize OCI over LS, with a fallback to the CreatedAt.

This should serve a wide range of users who either have internal rules
for image builds (e.g. timestamp set to null as reported in #1797) or
want a stable reliable pointer they can have control over (#746, #891).
[1]: https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
[2]: http://label-schema.org/rc1/#build-time-labels

cmd/fluxctl/list_images_cmd.go

@@ -130,8 +130,8 @@ func (opts *imageListOpts) RunE(cmd *cobra.Command, args []string) error {
 				}
 				if printLine {
 					createdAt := ""
-					if !available.CreatedAt.IsZero() {
-						createdAt = available.CreatedAt.Format(time.RFC822)
+					if !available.CreatedTS().IsZero() {
+						createdAt = available.CreatedTS().Format(time.RFC822)
 					}
 					fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, tag, createdAt)
 				}

daemon/images.go

@@ -103,13 +103,13 @@ func calculateChanges(logger log.Logger, candidateWorkloads resources, workloads
 					continue containers
 				}
 				current := repoMetadata.FindImageWithRef(currentImageID)
-				if current.CreatedAt.IsZero() || latest.CreatedAt.IsZero() {
-					logger.Log("warning", "image with zero created timestamp", "current", fmt.Sprintf("%s (%s)", current.ID, current.CreatedAt), "latest", fmt.Sprintf("%s (%s)", latest.ID, latest.CreatedAt), "action", "skip container")
+				if current.CreatedTS().IsZero() || latest.CreatedTS().IsZero() {
+					logger.Log("warning", "image with zero created timestamp", "current", fmt.Sprintf("%s (%s)", current.ID, current.CreatedTS()), "latest", fmt.Sprintf("%s (%s)", latest.ID, latest.CreatedTS()), "action", "skip container")
 					continue containers
 				}
 				newImage := currentImageID.WithNewTag(latest.ID.Tag)
 				changes.Add(workload.ID, container, newImage)
-				logger.Log("info", "added update to automation run", "new", newImage, "reason", fmt.Sprintf("latest %s (%s) > current %s (%s)", latest.ID.Tag, latest.CreatedAt, currentImageID.Tag, current.CreatedAt))
+				logger.Log("info", "added update to automation run", "new", newImage, "reason", fmt.Sprintf("latest %s (%s) > current %s (%s)", latest.ID.Tag, latest.CreatedTS(), currentImageID.Tag, current.CreatedTS()))
 			}
 		}
 	}

image/image.go

@@ -223,6 +223,52 @@ func (i Ref) WithNewTag(t string) Ref {
 	return img
 }
 
+// Labels has all the image labels we are interested in for an image
+// ref, the JSON struct tag keys should be equal to the label.
+type Labels struct {
+	// BuildDate holds the Label Schema spec 'build date' label
+	// Ref: http://label-schema.org/rc1/#build-time-labels
+	BuildDate time.Time `json:"org.label-schema.build-date,omitempty"`
+	// Created holds the Open Container Image spec 'created' label
+	// Ref: https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
+	Created   time.Time `json:"org.opencontainers.image.created,omitempty"`
+}
+
+// MarshalJSON returns the Labels value in JSON (as bytes). It is
+// implemented so that we can omit the time values when they are
+// zero, which would otherwise be tricky for e.g., JavaScript to
+// detect.
+func (l Labels) MarshalJSON() ([]byte, error) {
+	var bd, c string
+	if !l.BuildDate.IsZero() {
+		bd = l.BuildDate.UTC().Format(time.RFC3339Nano)
+	}
+	if !l.Created.IsZero() {
+		c = l.Created.UTC().Format(time.RFC3339Nano)
+	}
+	encode := struct {
+		BuildDate string `json:"org.label-schema.build-date,omitempty"`
+		Created   string `json:"org.opencontainers.image.created,omitempty"`
+	}{BuildDate: bd, Created: c}
+	return json.Marshal(encode)
+}
+
+// UnmarshalJSON populates Labels from JSON (as bytes). It's the
+// companion to MarshalJSON above.
+func (l *Labels) UnmarshalJSON(b []byte) error {
+	unencode := struct {
+		BuildDate string `json:"org.label-schema.build-date,omitempty"`
+		Created   string `json:"org.opencontainers.image.created,omitempty"`
+	}{}
+	json.Unmarshal(b, &unencode)
+
+	var err error
+	if err = decodeTime(unencode.BuildDate, &l.BuildDate); err == nil {
+		err = decodeTime(unencode.Created, &l.Created)
+	}
+	return err
+}
+
 // Info has the metadata we are able to determine about an image ref,
 // from its registry.
 type Info struct {
@@ -235,6 +281,8 @@ type Info struct {
 	// will be the same for references that point at the same image
 	// (but does not necessarily equal Docker's image ID)
 	ImageID string `json:",omitempty"`
+	// all labels we are interested in and could find for the image ref
+	Labels Labels `json:",omitempty"`
 	// the time at which the image pointed at was created
 	CreatedAt time.Time `json:",omitempty"`
 	// the last time this image manifest was fetched
@@ -281,6 +329,27 @@ func (im *Info) UnmarshalJSON(b []byte) error {
 	return err
 }
 
+// CreatedTS returns the created at timestamp for an image,
+// prioritizing user defined timestamps from labels over the ones we
+// receive from a Docker registry API.
+//
+// The reason for this is registry vendors have different
+// interpretations of what a creation  date is, and we want the user to
+// be in control when required.
+//
+// In addition we prioritize the `Created` label over the `BuildDate`,
+// as the Label Schema Spec has been deprecated in favour of the OCI
+// Spec (but is still well known and widely used).
+func (im Info) CreatedTS() time.Time {
+	if !im.Labels.Created.IsZero() {
+		return im.Labels.Created
+	}
+	if !im.Labels.BuildDate.IsZero() {
+		return im.Labels.BuildDate
+	}
+	return im.CreatedAt
+}
+
 // RepositoryMetadata contains the image metadata information found in an
 // image repository.
 //
@@ -334,10 +403,10 @@ func decodeTime(s string, t *time.Time) error {
 // NewerByCreated returns true if lhs image should be sorted
 // before rhs with regard to their creation date descending.
 func NewerByCreated(lhs, rhs *Info) bool {
-	if lhs.CreatedAt.Equal(rhs.CreatedAt) {
+	if lhs.CreatedTS().Equal(rhs.CreatedTS()) {
 		return lhs.ID.String() < rhs.ID.String()
 	}
-	return lhs.CreatedAt.After(rhs.CreatedAt)
+	return lhs.CreatedTS().After(rhs.CreatedTS())
 }
 
 // NewerBySemver returns true if lhs image should be sorted

image/image_test.go

@@ -138,6 +138,36 @@ func TestRefSerialization(t *testing.T) {
 	}
 }
 
+func TestImageLabelsSerialisation(t *testing.T) {
+	t0 := time.Now().UTC() // UTC so it has nil location, otherwise it won't compare
+	t1 := time.Now().Add(5 * time.Minute).UTC()
+	labels := Labels{Created: t0, BuildDate: t1}
+	bytes, err := json.Marshal(labels)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var labels1 Labels
+	if err = json.Unmarshal(bytes, &labels1); err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, labels, labels1)
+}
+
+func TestImageLabelsZeroTime(t *testing.T) {
+	labels := Labels{}
+	bytes, err := json.Marshal(labels)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var labels1 map[string]interface{}
+	if err = json.Unmarshal(bytes, &labels1); err != nil {
+		t.Fatal(err)
+	}
+	if lc := len(labels1); lc >= 1 {
+		t.Errorf("serialised Labels contains %v fields; expected it to contain none\n%v", lc, labels1)
+	}
+}
+
 func mustMakeInfo(ref string, created time.Time) Info {
 	r, err := ParseRef(ref)
 	if err != nil {
@@ -146,6 +176,11 @@ func mustMakeInfo(ref string, created time.Time) Info {
 	return Info{ID: r, CreatedAt: created}
 }
 
+func (im Info) setLabels(labels Labels) Info {
+	im.Labels = labels
+	return im
+}
+
 func TestImageInfoSerialisation(t *testing.T) {
 	t0 := time.Now().UTC() // UTC so it has nil location, otherwise it won't compare
 	t1 := time.Now().Add(5 * time.Minute).UTC()
@@ -184,10 +219,10 @@ func TestImage_OrderByCreationDate(t *testing.T) {
 	time0 := testTime.Add(time.Second)
 	time2 := testTime.Add(-time.Second)
 	imA := mustMakeInfo("my/Image:2", testTime)
-	imB := mustMakeInfo("my/Image:0", time0)
-	imC := mustMakeInfo("my/Image:3", time2)
+	imB := mustMakeInfo("my/Image:0", time.Time{}).setLabels(Labels{Created: time0})
+	imC := mustMakeInfo("my/Image:3", time.Time{}).setLabels(Labels{BuildDate: time2})
 	imD := mustMakeInfo("my/Image:4", time.Time{}) // test nil
-	imE := mustMakeInfo("my/Image:1", testTime)    // test equal
+	imE := mustMakeInfo("my/Image:1", time.Time{}).setLabels(Labels{Created: testTime}) // test equal
 	imF := mustMakeInfo("my/Image:5", time.Time{}) // test nil equal
 	imgs := []Info{imA, imB, imC, imD, imE, imF}
 	Sort(imgs, NewerByCreated)
@@ -204,7 +239,7 @@ func checkSorted(t *testing.T, imgs []Info) {
 	for i, im := range imgs {
 		if strconv.Itoa(i) != im.ID.Tag {
 			for j, jim := range imgs {
-				t.Logf("%v: %v %s", j, jim.ID.String(), jim.CreatedAt)
+				t.Logf("%v: %v %s", j, jim.ID.String(), jim.CreatedTS())
 			}
 			t.Fatalf("Not sorted in expected order: %#v", imgs)
 		}

registry/client.go

@@ -133,6 +133,9 @@ interpret:
 			Created time.Time `json:"created"`
 			OS      string    `json:"os"`
 			Arch    string    `json:"architecture"`
+			Config  struct {
+				Labels image.Labels `json:"labels"`
+			} `json:"config"`
 		}
 
 		if err = json.Unmarshal([]byte(man.History[0].V1Compatibility), &v1); err != nil {
@@ -142,6 +145,7 @@ interpret:
 		// identify the image as it's the topmost layer.
 		info.ImageID = v1.ID
 		info.CreatedAt = v1.Created
+		info.Labels = v1.Config.Labels
 	case *schema2.DeserializedManifest:
 		var man schema2.Manifest = deserialised.Manifest
 		configBytes, err := repository.Blobs(ctx).Get(ctx, man.Config.Digest)
@@ -153,13 +157,17 @@ interpret:
 			Arch    string    `json:"architecture"`
 			Created time.Time `json:"created"`
 			OS      string    `json:"os"`
+			ContainerConfig struct {
+				Labels image.Labels `json:"labels"`
+			} `json:"container_config"`
 		}
 		if err = json.Unmarshal(configBytes, &config); err != nil {
 			return ImageEntry{}, err
 		}
 		// This _is_ what Docker uses as its Image ID.
 		info.ImageID = man.Config.Digest.String()
 		info.CreatedAt = config.Created
+		info.Labels = config.ContainerConfig.Labels
 	case *manifestlist.DeserializedManifestList:
 		var list manifestlist.ManifestList = deserialised.ManifestList
 		// TODO(michael): is it valid to just pick the first one that matches?