Add build test for .containerignore tar file

Ensure a directory added to .containerignore on client is not included
in tar sent to remote podman API service

* Clean up podman invocations to not include duplicate --remote and
  --url flags
* Use pkill vs. pgrep when cleaning up podman API service in tests
* Add exit code when logging error when testing

Closes containers#13535

Signed-off-by: Jhon Honce <[email protected]>

pkg/bindings/images/build.go

@@ -367,20 +367,20 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 			return nil, err
 		}
 
-		// Check if Containerfile is in the context directory, if so truncate the contextdirectory off path
+		// Check if Containerfile is in the context directory, if so truncate the context directory off path
 		// Do NOT add to tarfile
 		if strings.HasPrefix(containerfile, contextDir+string(filepath.Separator)) {
 			containerfile = strings.TrimPrefix(containerfile, contextDir+string(filepath.Separator))
 			dontexcludes = append(dontexcludes, "!"+containerfile)
 		} else {
-			// If Containerfile does not exists assume it is in context directory, do Not add to tarfile
+			// If Containerfile does not exist, assume it is in context directory and do Not add to tarfile
 			if _, err := os.Lstat(containerfile); err != nil {
 				if !os.IsNotExist(err) {
 					return nil, err
 				}
 				containerfile = c
 			} else {
-				// If Containerfile does exists but is not in context directory add it to the tarfile
+				// If Containerfile does exist and not in the context directory, add it to the tarfile
 				tarContent = append(tarContent, containerfile)
 			}
 		}
@@ -586,6 +586,9 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
 					return errors.Wrapf(err, "error checking if %q is excluded", name)
 				}
 				if excluded {
+					// Note: filepath.SkipDir is not possible to use given .dockerignore semantics.
+					// An exception to exclusions may include an excluded directory, therefore we
+					// are required to visit all files. :(
 					return nil
 				}
 

test/e2e/build_test.go

@@ -1,9 +1,11 @@
 package integration
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -493,14 +495,64 @@ subdir**`
 		Expect(output).NotTo(ContainSubstring("/testfilter/subdir"))
 	})
 
+	// See https://github.com/containers/podman/issues/13535
+	It("Remote build .containerignore filtering embedded directory (#13535)", func() {
+		SkipIfNotRemote("Testing remote .containerignore file filtering")
+		podmanTest.RestartRemoteService()
+
+		// Switch to temp dir and restore it afterwards
+		cwd, err := os.Getwd()
+		Expect(err).ToNot(HaveOccurred())
+
+		podmanTest.AddImageToRWStore(ALPINE)
+
+		contents := bytes.Buffer{}
+		contents.WriteString("FROM " + ALPINE + "\n")
+		contents.WriteString("ADD . /testfilter/\n")
+		contents.WriteString("RUN find /testfilter/ -print\n")
+
+		containerfile := filepath.Join(tempdir, "Containerfile")
+		Expect(ioutil.WriteFile(containerfile, contents.Bytes(), 0644)).ToNot(HaveOccurred())
+
+		contextDir, err := CreateTempDirInTempDir()
+		Expect(err).ToNot(HaveOccurred())
+		defer os.RemoveAll(contextDir)
+
+		Expect(ioutil.WriteFile(filepath.Join(contextDir, "expected"), contents.Bytes(), 0644)).
+			ToNot(HaveOccurred())
+
+		subdirPath := filepath.Join(contextDir, "subdir")
+		Expect(os.MkdirAll(subdirPath, 0755)).ToNot(HaveOccurred())
+		Expect(ioutil.WriteFile(filepath.Join(subdirPath, "extra"), contents.Bytes(), 0644)).
+			ToNot(HaveOccurred())
+		randomFile := filepath.Join(subdirPath, "randomFile")
+		dd := exec.Command("dd", "if=/dev/random", "of="+randomFile, "bs=1G", "count=1")
+		ddSession, err := Start(dd, GinkgoWriter, GinkgoWriter)
+		Expect(err).ToNot(HaveOccurred())
+		Eventually(ddSession).Should(Exit(0))
+
+		// make cwd as context root path
+		Expect(os.Chdir(contextDir)).ToNot(HaveOccurred())
+		defer os.Chdir(cwd)
+
+		By("Test .containerignore filtering subdirectory")
+		err = ioutil.WriteFile(filepath.Join(contextDir, ".containerignore"), []byte(`subdir/`), 0644)
+		Expect(err).ToNot(HaveOccurred())
+
+		session := podmanTest.Podman([]string{"build", "-f", containerfile, contextDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+
+		output := session.OutputToString()
+		Expect(output).To(ContainSubstring("Containerfile"))
+		Expect(output).To(ContainSubstring("/testfilter/expected"))
+		Expect(output).NotTo(ContainSubstring("subdir"))
+	})
+
 	It("podman remote test context dir contains empty dirs and symlinks", func() {
-		if IsRemote() {
-			podmanTest.StopRemoteService()
-			podmanTest.StartRemoteService()
-		} else {
-			Skip("Only valid at remote test")
-		}
-		// Given
+		SkipIfNotRemote("Testing remote contextDir empty")
+		podmanTest.RestartRemoteService()
+
 		// Switch to temp dir and restore it afterwards
 		cwd, err := os.Getwd()
 		Expect(err).To(BeNil())

test/e2e/common_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/podman/v4/pkg/inspect"
 	"github.com/containers/podman/v4/pkg/rootless"
+	"github.com/containers/podman/v4/pkg/util"
 	. "github.com/containers/podman/v4/test/utils"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/reexec"
@@ -500,14 +501,12 @@ func (p *PodmanTestIntegration) BuildImageWithLabel(dockerfile, imageName string
 // PodmanPID execs podman and returns its PID
 func (p *PodmanTestIntegration) PodmanPID(args []string) (*PodmanSessionIntegration, int) {
 	podmanOptions := p.MakeOptions(args, false, false)
-	if p.RemoteTest {
-		podmanOptions = append([]string{"--remote", "--url", p.RemoteSocket}, podmanOptions...)
-	}
 	fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " "))
+
 	command := exec.Command(p.PodmanBinary, podmanOptions...)
 	session, err := Start(command, GinkgoWriter, GinkgoWriter)
 	if err != nil {
-		Fail(fmt.Sprintf("unable to run podman command: %s", strings.Join(podmanOptions, " ")))
+		Fail("unable to run podman command: " + strings.Join(podmanOptions, " "))
 	}
 	podmanSession := &PodmanSession{Session: session}
 	return &PodmanSessionIntegration{podmanSession}, command.Process.Pid
@@ -843,11 +842,13 @@ func (p *PodmanTestIntegration) PodmanNoEvents(args []string) *PodmanSessionInte
 // MakeOptions assembles all the podman main options
 func (p *PodmanTestIntegration) makeOptions(args []string, noEvents, noCache bool) []string {
 	if p.RemoteTest {
+		if !util.StringInSlice("--remote", args) {
+			return append([]string{"--remote", "--url", p.RemoteSocket}, args...)
+		}
 		return args
 	}
-	var (
-		debug string
-	)
+
+	var debug string
 	if _, ok := os.LookupEnv("DEBUG"); ok {
 		debug = "--log-level=debug --syslog=true "
 	}

test/e2e/libpod_suite_remote_test.go

@@ -4,7 +4,6 @@
 package integration
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -25,31 +24,28 @@ func IsRemote() bool {
 
 // Podman is the exec call to podman on the filesystem
 func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration {
-	var remoteArgs = []string{"--remote", "--url", p.RemoteSocket}
-	remoteArgs = append(remoteArgs, args...)
-	podmanSession := p.PodmanBase(remoteArgs, false, false)
+	args = p.makeOptions(args, false, false)
+	podmanSession := p.PodmanBase(args, false, false)
 	return &PodmanSessionIntegration{podmanSession}
 }
 
 // PodmanSystemdScope runs the podman command in a new systemd scope
 func (p *PodmanTestIntegration) PodmanSystemdScope(args []string) *PodmanSessionIntegration {
-	var remoteArgs = []string{"--remote", "--url", p.RemoteSocket}
-	remoteArgs = append(remoteArgs, args...)
+	args = p.makeOptions(args, false, false)
 
 	wrapper := []string{"systemd-run", "--scope"}
 	if rootless.IsRootless() {
 		wrapper = []string{"systemd-run", "--scope", "--user"}
 	}
 
-	podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, wrapper, nil)
+	podmanSession := p.PodmanAsUserBase(args, 0, 0, "", nil, false, false, wrapper, nil)
 	return &PodmanSessionIntegration{podmanSession}
 }
 
 // PodmanExtraFiles is the exec call to podman on the filesystem and passes down extra files
 func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os.File) *PodmanSessionIntegration {
-	var remoteArgs = []string{"--remote", "--url", p.RemoteSocket}
-	remoteArgs = append(remoteArgs, args...)
-	podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, nil, extraFiles)
+	args = p.makeOptions(args, false, false)
+	podmanSession := p.PodmanAsUserBase(args, 0, 0, "", nil, false, false, nil, extraFiles)
 	return &PodmanSessionIntegration{podmanSession}
 }
 
@@ -96,57 +92,39 @@ func (p *PodmanTestIntegration) StartRemoteService() {
 	command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	p.RemoteCommand = command
 	p.RemoteSession = command.Process
-	err := p.DelayForService()
-	p.RemoteStartErr = err
+	p.RemoteStartErr = p.DelayForService()
 }
 
 func (p *PodmanTestIntegration) StopRemoteService() {
-	var out bytes.Buffer
-	var pids []int
-	remoteSession := p.RemoteSession
-
 	if !rootless.IsRootless() {
-		if err := remoteSession.Kill(); err != nil {
+		if err := p.RemoteSession.Kill(); err != nil {
 			fmt.Fprintf(os.Stderr, "error on remote stop-kill %q", err)
 		}
-		if _, err := remoteSession.Wait(); err != nil {
+		if _, err := p.RemoteSession.Wait(); err != nil {
 			fmt.Fprintf(os.Stderr, "error on remote stop-wait %q", err)
 		}
 	} else {
-		parentPid := fmt.Sprintf("%d", p.RemoteSession.Pid)
-		pgrep := exec.Command("pgrep", "-P", parentPid)
-		fmt.Printf("running: pgrep %s\n", parentPid)
-		pgrep.Stdout = &out
-		err := pgrep.Run()
-		if err != nil {
-			fmt.Fprint(os.Stderr, "unable to find remote pid")
-		}
-
-		for _, s := range strings.Split(out.String(), "\n") {
-			if len(s) == 0 {
-				continue
-			}
-			p, err := strconv.Atoi(s)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "unable to convert %s to int", s)
-			}
-			if p != 0 {
-				pids = append(pids, p)
+		// Stop any children of `podman system service`
+		pkill := exec.Command("pkill", "-P", strconv.Itoa(p.RemoteSession.Pid), "-15")
+		if err := pkill.Run(); err != nil {
+			exitErr := err.(*exec.ExitError)
+			if exitErr.ExitCode() != 1 {
+				fmt.Fprintf(os.Stderr, "pkill unable to clean up service %d children, exit code %d\n",
+					p.RemoteSession.Pid, exitErr.ExitCode())
 			}
 		}
-
-		pids = append(pids, p.RemoteSession.Pid)
-		for _, pid := range pids {
-			syscall.Kill(pid, syscall.SIGKILL)
+		if err := p.RemoteSession.Kill(); err != nil {
+			fmt.Fprintf(os.Stderr, "unable to clean up service %d, %v\n", p.RemoteSession.Pid, err)
 		}
 	}
+
 	socket := strings.Split(p.RemoteSocket, ":")[1]
-	if err := os.Remove(socket); err != nil {
-		fmt.Println(err)
+	if err := os.Remove(socket); err != nil && !errors.Is(err, os.ErrNotExist) {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
 	}
 	if p.RemoteSocketLock != "" {
-		if err := os.Remove(p.RemoteSocketLock); err != nil {
-			fmt.Println(err)
+		if err := os.Remove(p.RemoteSocketLock); err != nil && !errors.Is(err, os.ErrNotExist) {
+			fmt.Fprintf(os.Stderr, "%v\n", err)
 		}
 	}
 }
@@ -185,8 +163,9 @@ func (p *PodmanTestIntegration) RestoreArtifact(image string) error {
 }
 
 func (p *PodmanTestIntegration) DelayForService() error {
+	var session *PodmanSessionIntegration
 	for i := 0; i < 5; i++ {
-		session := p.Podman([]string{"info"})
+		session = p.Podman([]string{"info"})
 		session.WaitWithDefaultTimeout()
 		if session.ExitCode() == 0 {
 			return nil
@@ -195,5 +174,5 @@ func (p *PodmanTestIntegration) DelayForService() error {
 		}
 		time.Sleep(2 * time.Second)
 	}
-	return errors.New("Service not detected")
+	return fmt.Errorf("service not detected, exit code(%d)", session.ExitCode())
 }

test/utils/utils.go

@@ -100,9 +100,7 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string
 	if p.NetworkBackend == Netavark {
 		runCmd = append(runCmd, []string{"--network-backend", "netavark"}...)
 	}
-	if p.RemoteTest {
-		podmanOptions = append([]string{"--remote", "--url", p.RemoteSocket}, podmanOptions...)
-	}
+
 	if env == nil {
 		fmt.Printf("Running: %s %s\n", strings.Join(runCmd, " "), strings.Join(podmanOptions, " "))
 	} else {