Merge pull request #3485 from nalind/conformance-mtime

conformance: tighten up exception specifications

tests/conformance/README.md

@@ -2,7 +2,7 @@
 
 # Buildah/Docker Conformance Test Suite
 
-The conformance test for buildah is used to verify the images built with Buildah are equivalent to those built by Docker.  It does this by building an image using the version of buildah library that's being tested, building what should be the same image using `docker build`, and comparing them.
+The conformance test for buildah is used to verify the images built with Buildah are equivalent to those built by Docker.  It does this by building an image using the version of buildah library that's being tested, building what should be the same image using the docker engine's build API, and comparing them.
 
 ## Installing dependencies
 
@@ -15,17 +15,17 @@ Conformance tests use Docker CE to build images to be compared with images built
 
 ## Run conformance tests
 
-You can run the test with go test:
+You can run all of the tests with go test:
 ```
 go test -v -tags "$(./btrfs_tag.sh) $(./btrfs_installed_tag.sh) $(./libdm_tag.sh)" ./tests/conformance
 ```
 
-If you want to run one of the test cases you can use flag "-run":
+If you want to run one of the test cases you can use the "-run" flag:
 ```
 go test -v -tags "$(./btrfs_tag.sh) $(./btrfs_installed_tag.sh) $(./libdm_tag.sh)" -run TestConformance/shell ./tests/conformance
 ```
 
 If you also want to build and compare on a line-by-line basis, run:
 ```
-go test -v -tags "$(./btrfs_tag.sh) $(./btrfs_installed_tag.sh) $(./libdm_tag.sh)" ./tests/conformance -compare-layers
+go test -v -timeout=60m -tags "$(./btrfs_tag.sh) $(./btrfs_installed_tag.sh) $(./libdm_tag.sh)" ./tests/conformance -compare-layers
 ```

tests/conformance/conformance_test.go

@@ -55,7 +55,8 @@ var (
 		"created",
 		"container",
 		"docker_version",
-		"hostname",
+		"container_config:hostname",
+		"config:hostname",
 		"config:image",
 		"container_config:cmd",
 		"container_config:image",
@@ -477,6 +478,11 @@ func testConformanceInternalBuild(ctx context.Context, t *testing.T, cwd string,
 
 	// the report on the imagebuilder image should be there if we expected the build to succeed
 	if compareImagebuilder && !test.withoutImagebuilder {
+		originalDockerConfig, ociDockerConfig, fsDocker = readReport(t, filepath.Join(dockerDir, t.Name()))
+		if t.Failed() {
+			t.FailNow()
+		}
+
 		originalImagebuilderConfig, ociImagebuilderConfig, fsImagebuilder := readReport(t, filepath.Join(imagebuilderDir, t.Name()))
 		if t.Failed() {
 			t.FailNow()
@@ -1063,13 +1069,11 @@ func compareJSON(a, b map[string]interface{}, skip []string) (missKeys, leftKeys
 			// this field in the object is itself an object (e.g.
 			// "config" or "container_config"), so recursively
 			// compare them
-			nextSkip := skip
+			var nextSkip []string
+			prefix := k + ":"
 			for _, s := range skip {
-				if strings.Contains(s, ":") {
-					tmp := strings.Split(s, ":")
-					if tmp[0] == k {
-						nextSkip = append(nextSkip, strings.Join(tmp[1:], ":"))
-					}
+				if strings.HasPrefix(s, prefix) {
+					nextSkip = append(nextSkip, strings.TrimPrefix(s, prefix))
 				}
 			}
 			submiss, subleft, subdiff, ok := compareJSON(v.(map[string]interface{}), vb.(map[string]interface{}), nextSkip)
@@ -1405,8 +1409,9 @@ var internalTestCases = []testCase{
 		name:       "edgecases",
 		dockerfile: "Dockerfile.edgecases",
 		fsSkip: []string{
-			"(dir):test:mtime", "(dir):test2:mtime", "(dir):test3:mtime", "(dir):test3:(dir):copy:mtime",
-			"(dir):tmp:mtime", "(dir):tmp:passwd:mtime",
+			"(dir):test:mtime", "(dir):test:(dir):copy:mtime", "(dir):test2:mtime", "(dir):test3:mtime",
+			"(dir):test3:(dir):copy:mtime",
+			"(dir):test3:(dir):test:mtime", "(dir):tmp:mtime", "(dir):tmp:(dir):passwd:mtime",
 		},
 	},
 
@@ -1480,7 +1485,7 @@ var internalTestCases = []testCase{
 			"(dir):go:(dir):src:(dir):github.com:mtime",
 			"(dir):go:(dir):src:(dir):github.com:(dir):openshift:mtime",
 			"(dir):go:(dir):src:(dir):github.com:(dir):openshift:(dir):ocp-release-operator-sdk:mtime",
-			"env-contents.txt:mtime",
+			"(dir):env-contents.txt:mtime",
 		},
 	},
 
@@ -1657,7 +1662,7 @@ var internalTestCases = []testCase{
 		dockerfileContents: strings.Join([]string{
 			"FROM busybox@sha256:9ddee63a712cea977267342e8750ecbc60d3aab25f04ceacfa795e6fce341793 AS build",
 			"RUN mkdir -p /target/subdir",
-			"RUN cp /etc/passwd /target/",
+			"RUN cp -p /etc/passwd /target/",
 			"RUN ln /target/passwd /target/subdir/passwd",
 			"RUN ln /target/subdir/passwd /target/subdir/passwd2",
 			"FROM scratch",
@@ -1968,7 +1973,7 @@ var internalTestCases = []testCase{
 			"COPY --from=base . .", // --from=otherstage ignores that stage's WORKDIR
 		}, "\n"),
 		contextDir: "dockerignore/populated",
-		fsSkip:     []string{"(dir):subdir:mtime"},
+		fsSkip:     []string{"(dir):subdir:mtime", "(dir):subdir:(dir):subdir:mtime"},
 	},
 
 	{
@@ -2029,7 +2034,7 @@ var internalTestCases = []testCase{
 		name: "multi-stage-through-base",
 		dockerfileContents: strings.Join([]string{
 			"FROM alpine AS base",
-			"RUN touch /1",
+			"RUN touch -t @1485449953 /1",
 			"ENV LOCAL=/1",
 			"RUN find $LOCAL",
 			"FROM base",
@@ -2044,7 +2049,7 @@ var internalTestCases = []testCase{
 			"FROM busybox as layer",
 			"RUN touch /root/layer",
 			"FROM layer as derived",
-			"RUN touch /root/derived ; rm /root/layer",
+			"RUN touch -t @1485449953 /root/derived ; rm /root/layer",
 			"FROM busybox AS output",
 			"COPY --from=layer /root /root",
 		}, "\n"),
@@ -2642,7 +2647,7 @@ var internalTestCases = []testCase{
 		name: "copy-from-owner", // from issue #2518
 		dockerfileContents: strings.Join([]string{
 			`FROM alpine`,
-			`RUN set -ex; touch /test; chown 65:65 /test`,
+			`RUN set -ex; touch -t @1485449953 /test; chown 65:65 /test`,
 			`FROM scratch`,
 			`USER 66:66`,
 			`COPY --from=0 /test /test`,
@@ -2654,7 +2659,7 @@ var internalTestCases = []testCase{
 		name: "copy-from-owner-with-chown", // issue #2518, but with chown to override
 		dockerfileContents: strings.Join([]string{
 			`FROM alpine`,
-			`RUN set -ex; touch /test; chown 65:65 /test`,
+			`RUN set -ex; touch -t @1485449953 /test; chown 65:65 /test`,
 			`FROM scratch`,
 			`USER 66:66`,
 			`COPY --from=0 --chown=1:1 /test /test`,
@@ -2685,13 +2690,13 @@ var internalTestCases = []testCase{
 	{
 		name:       "add-parent-symlink",
 		contextDir: "add/parent-symlink",
-		fsSkip:     []string{"(dir):testsubdir:mtime"},
+		fsSkip:     []string{"(dir):testsubdir:mtime", "(dir):testsubdir:(dir):etc:mtime"},
 	},
 
 	{
 		name:       "add-parent-dangling",
 		contextDir: "add/parent-dangling",
-		fsSkip:     []string{"(dir):symlink:mtime", "(dir):symlink-target:mtime"},
+		fsSkip:     []string{"(dir):symlink:mtime", "(dir):symlink-target:mtime", "(dir):symlink-target:(dir):subdirectory:mtime"},
 	},
 
 	{

tests/conformance/testdata/Dockerfile.copyfrom_1

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN touch /a /b
+RUN touch -t @1485449953 /a /b
 FROM busybox
 COPY --from=base /a /
 RUN ls -al /a

tests/conformance/testdata/Dockerfile.copyfrom_10

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/1
+RUN mkdir -p /a && touch -t @1485449953 /a/1
 FROM busybox
 COPY --from=base /a/1 /a/b/c
 RUN ls -al /a/b/c && ! ls -al /a/b/1

tests/conformance/testdata/Dockerfile.copyfrom_11

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/1
+RUN mkdir -p /a && touch -t @1485449953 /a/1
 FROM busybox
 COPY --from=base /a /a/b/c
 RUN ls -al /a/b/c/1 && ! ls -al /a/b/1

tests/conformance/testdata/Dockerfile.copyfrom_12

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/1
+RUN mkdir -p /a && touch -t @1485449953 /a/1
 FROM busybox
 COPY --from=base a /a
 RUN ls -al /a/1 && ! ls -al /a/a

tests/conformance/testdata/Dockerfile.copyfrom_13

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/1
+RUN mkdir -p /a && touch -t @1485449953 /a/1
 FROM busybox
 COPY --from=base a/. /a
 RUN ls -al /a/1 && ! ls -al /a/a

tests/conformance/testdata/Dockerfile.copyfrom_2

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN touch /a
+RUN touch -t @1485449953 /a
 FROM busybox
 COPY --from=base /a /a
 RUN ls -al /a

tests/conformance/testdata/Dockerfile.copyfrom_3

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN touch /a
+RUN touch -t @1485449953 /a
 FROM busybox
 WORKDIR /b
 COPY --from=base /a .

tests/conformance/testdata/Dockerfile.copyfrom_3_1

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN touch /a
+RUN touch -t @1485449953 /a
 FROM busybox
 WORKDIR /b
 COPY --from=base /a ./b

tests/conformance/testdata/Dockerfile.copyfrom_4

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a/b && touch /a/b/1 /a/b/2
+RUN mkdir -p /a/b && touch -t @1485449953 /a/b/1 /a/b/2
 FROM busybox
 COPY --from=base /a/b/ /b/
 RUN ls -al /b/1 /b/2 /b && ! ls -al /a

tests/conformance/testdata/Dockerfile.copyfrom_5

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a/b && touch /a/b/1 /a/b/2
+RUN mkdir -p /a/b && touch -t @1485449953 /a/b/1 /a/b/2
 FROM busybox
 COPY --from=base /a/b/* /b/
 RUN ls -al /b/1 /b/2 /b && ! ls -al /a /b/a /b/b

tests/conformance/testdata/Dockerfile.copyfrom_6

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a/b && touch /a/b/1 /a/b/2
+RUN mkdir -p /a/b && touch -t @1485449953 /a/b/1 /a/b/2
 FROM busybox
 COPY --from=base /a/b/. /b/
 RUN ls -al /b/1 /b/2 /b && ! ls -al /a /b/a /b/b

tests/conformance/testdata/Dockerfile.copyfrom_7

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN touch /b
+RUN touch -t @1485449953 /b
 FROM busybox
 COPY --from=base /b /a
 RUN ls -al /a && ! ls -al /b

tests/conformance/testdata/Dockerfile.copyfrom_8

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/b
+RUN mkdir -p /a && touch -t @1485449953 /a/b
 FROM busybox
 COPY --from=base /a/b /a
 RUN ls -al /a && ! ls -al /b

tests/conformance/testdata/Dockerfile.copyfrom_9

@@ -1,5 +1,5 @@
 FROM busybox as base
-RUN mkdir -p /a && touch /a/1
+RUN mkdir -p /a && touch -t @1485449953 /a/1
 FROM busybox
 COPY --from=base /a/1 /a/b/c/
 RUN ls -al /a/b/c/1 && ! ls -al /a/b/1

tests/conformance/testdata/Dockerfile.reusebase

@@ -1,5 +1,5 @@
 FROM registry.centos.org/centos/centos:centos7 AS base
-RUN touch /1
+RUN touch -t 201701261659.13 /1
 ENV LOCAL=/1
 
 FROM base

tests/conformance/testdata/dockerignore/allowlist/nothing-dot/Dockerfile

@@ -1,5 +1,5 @@
 FROM busybox
-RUN touch /file
+RUN touch -t @1485449953 /file
 FROM scratch
 COPY --from=0 /file /
 ADD . .

tests/conformance/testdata/dockerignore/allowlist/nothing-slash/Dockerfile

@@ -1,5 +1,5 @@
 FROM busybox
-RUN touch /file
+RUN touch -t @1485449953 /file
 FROM scratch
 COPY --from=0 /file /
 ADD / /

tests/conformance/testdata/dockerignore/allowlist/subsubdir-file/Dockerfile

@@ -1,5 +1,5 @@
 FROM busybox
-RUN touch /file
+RUN touch -t @1485449953 /file
 FROM scratch
 COPY --from=0 /file /
 ADD . .

tests/conformance/testdata/dockerignore/allowlist/subsubdir-nofile/Dockerfile

@@ -1,5 +1,5 @@
 FROM busybox
-RUN touch /file
+RUN touch -t @1485449953 /file
 FROM scratch
 COPY --from=0 /file /
 ADD . .

tests/conformance/testdata/dockerignore/allowlist/subsubdir-nosubdir/Dockerfile

@@ -1,5 +1,5 @@
 FROM busybox
-RUN touch /file
+RUN touch -t @1485449953 /file
 FROM scratch
 COPY --from=0 /file /
 ADD . .