split runtimevalidate into inside and outside validaiton

Signed-off-by: Ma Shimiao <[email protected]>

Makefile

@@ -39,7 +39,6 @@ $(RUNTIME_TOOLS_LINK):
 localvalidation:
 	RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
 
-
 .PHONY: test .gofmt .govet .golint
 
 test: .gofmt .govet .golint

validation/validation_test.go

@@ -1,14 +1,19 @@
 package validation
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/hashicorp/go-multierror"
+	"github.com/mndrix/tap-go"
 	"github.com/mrunalp/fileutils"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/satori/go.uuid"
 )
@@ -17,54 +22,90 @@ var (
 	runtime = "runc"
 )
 
+type validation struct {
+	test        func(string, string, *rspec.Spec) error
+	description string
+}
+
 func init() {
 	runtime = os.Getenv("RUNTIME")
 }
 
-func runtimeValidate(runtime string, g *generate.Generator) error {
+func TestValidateRuntimeInside(t *testing.T) {
+	g, err := getDefaultGenerator()
+	if err != nil {
+		t.Errorf("%s failed validation: %v", runtime, err)
+	}
+	g.SetProcessArgs([]string{"/runtimetest"})
+
+	if err := runtimeInsideValidate(runtime, g); err != nil {
+		t.Errorf("%s failed validation: %v", runtime, err)
+	}
+}
+
+func TestValidateRuntimeOutside(t *testing.T) {
+	g, err := getDefaultGenerator()
+	if err != nil {
+		t.Errorf("%s failed validation: %v", runtime, err)
+	}
+	g.SetProcessArgs([]string{"sleep", "50"})
+
+	if err := runtimeOutsideValidate(runtime, g); err != nil {
+		t.Errorf("%s failed validation: %v", runtime, err)
+	}
+}
+
+func runtimeInsideValidate(runtime string, g *generate.Generator) error {
 	// Find the runtime binary in the PATH
 	runtimePath, err := exec.LookPath(runtime)
 	if err != nil {
 		return err
 	}
 
-	// Setup a temporary test directory
-	tmpDir, err := ioutil.TempDir("", "ocitest")
+	bundleDir, rootfsDir, err := prepareBundle(g)
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tmpDir)
+	defer os.RemoveAll(bundleDir)
 
-	// Create bundle directory for the test container
-	bundleDir := tmpDir + "/busybox"
-	if err := os.MkdirAll(bundleDir, 0755); err != nil {
+	// Copy the runtimetest binary to the rootfs
+	err = fileutils.CopyFile("../runtimetest", filepath.Join(rootfsDir, "runtimetest"))
+	if err != nil {
 		return err
 	}
 
-	// Untar the root fs
-	untarCmd := exec.Command("tar", "-xf", "../rootfs.tar.gz", "-C", bundleDir)
-	output, err := untarCmd.CombinedOutput()
-	if err != nil {
-		fmt.Println(string(output))
+	// TODO: Use a library to split run into create/start
+	// Launch the OCI runtime
+	containerID := uuid.NewV4()
+	runtimeCmd := exec.Command(runtimePath, "run", containerID.String())
+	runtimeCmd.Dir = bundleDir
+	runtimeCmd.Stdin = os.Stdin
+	runtimeCmd.Stdout = os.Stdout
+	runtimeCmd.Stderr = os.Stderr
+	if err = runtimeCmd.Run(); err != nil {
 		return err
 	}
 
-	// Copy the runtimetest binary to the rootfs
-	err = fileutils.CopyFile("../runtimetest", filepath.Join(bundleDir, "runtimetest"))
+	return nil
+}
+
+func runtimeOutsideValidate(runtime string, g *generate.Generator) error {
+	// Find the runtime binary in the PATH
+	runtimePath, err := exec.LookPath(runtime)
 	if err != nil {
 		return err
 	}
 
-	// Generate test configuration
-	err = g.SaveToFile(filepath.Join(bundleDir, "config.json"), generate.ExportOptions{})
+	bundleDir, _, err := prepareBundle(g)
 	if err != nil {
 		return err
 	}
+	//	defer os.RemoveAll(bundleDir)
 
 	// TODO: Use a library to split run into create/start
 	// Launch the OCI runtime
 	containerID := uuid.NewV4()
-	runtimeCmd := exec.Command(runtimePath, "run", containerID.String())
+	runtimeCmd := exec.Command(runtimePath, "run", "-d", containerID.String())
 	runtimeCmd.Dir = bundleDir
 	runtimeCmd.Stdin = os.Stdin
 	runtimeCmd.Stdout = os.Stdout
@@ -73,29 +114,108 @@ func runtimeValidate(runtime string, g *generate.Generator) error {
 		return err
 	}
 
-	return nil
+	outsideValidations := []validation{
+		{
+			test:        validateLabels,
+			description: "labels",
+		},
+		// Add more container outside validation
+	}
+
+	t := tap.New()
+	t.Header(0)
+
+	var validationErrors error
+	for _, v := range outsideValidations {
+		err := v.test(runtimePath, containerID.String(), g.Spec())
+		t.Ok(err == nil, v.description)
+		if err != nil {
+			validationErrors = multierror.Append(validationErrors, err)
+		}
+	}
+	t.AutoPlan()
+
+	return validationErrors
 }
 
-func getDefaultGenerator() *generate.Generator {
-	g := generate.New()
-	g.SetRootPath(".")
-	g.SetProcessArgs([]string{"/runtimetest"})
-	return &g
+func validateLabels(runtimePath, id string, spec *rspec.Spec) error {
+	runtimeCmd := exec.Command(runtimePath, "state", id)
+	output, err := runtimeCmd.Output()
+	if err != nil {
+		return err
+	}
+
+	var state rspec.State
+	if err := json.NewDecoder(strings.NewReader(string(output))).Decode(&state); err != nil {
+		return err
+	}
+	for key, value := range spec.Annotations {
+		if state.Annotations[key] == value {
+			continue
+		}
+		return fmt.Errorf("Expected annotation %s:%s not set", key, value)
+	}
+	return nil
 }
 
-func TestValidateBasic(t *testing.T) {
-	g := getDefaultGenerator()
+func prepareBundle(g *generate.Generator) (string, string, error) {
+	// Setup a temporary test directory
+	tmpDir, err := ioutil.TempDir("", "ocitest")
+	if err != nil {
+		return "", "", err
+	}
 
-	if err := runtimeValidate(runtime, g); err != nil {
-		t.Errorf("%s failed validation: %v", runtime, err)
+	// Create bundle directory for the test container
+	bundleDir := tmpDir
+	if err := os.MkdirAll(bundleDir, 0755); err != nil {
+		return "", "", err
+	}
+
+	// Create rootfs directory for the test container
+	rootfsDir := bundleDir + "/rootfs"
+	if err := os.MkdirAll(rootfsDir, 0755); err != nil {
+		return "", "", err
+	}
+
+	// Untar the root fs
+	untarCmd := exec.Command("tar", "-xf", "../rootfs.tar.gz", "-C", rootfsDir)
+	output, err := untarCmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(string(output))
+		return "", "", err
+	}
+
+	// Generate test configuration
+	err = g.SaveToFile(filepath.Join(bundleDir, "config.json"), generate.ExportOptions{})
+	if err != nil {
+		return "", "", err
 	}
+
+	// Copy the configuration file to the rootfs
+	err = fileutils.CopyFile(filepath.Join(bundleDir, "config.json"), filepath.Join(rootfsDir, "config.json"))
+	if err != nil {
+		return "", "", err
+	}
+
+	return bundleDir, rootfsDir, nil
 }
 
-func TestValidateSysctls(t *testing.T) {
-	g := getDefaultGenerator()
-	g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
+func getDefaultGenerator() (*generate.Generator, error) {
+	// Generate testcase template
+	generateCmd := exec.Command("oci-runtime-tool", "generate", "--bind=/tmp:/volume/testing:rw", "--cgroups-path=/tmp/testcgroup", "--device-add=c:80:500:/dev/test:fileMode=438", "--disable-oom-kill=true", "--env=testvar=vartest", "--hostname=localvalidation", "--label=testlabel=nonevar", "--linux-cpu-shares=1024", "--output", "/tmp/config.json")
+	output, err := generateCmd.CombinedOutput()
+	if err != nil {
+		fmt.Println(string(output))
+		return nil, err
+	}
 
-	if err := runtimeValidate(runtime, g); err != nil {
-		t.Errorf("%s failed validation: %v", runtime, err)
+	// Get testcase configuration
+	g, err := generate.NewFromFile("/tmp/config.json")
+	if err != nil {
+		return nil, err
 	}
+
+	g.SetRootPath("rootfs")
+
+	return &g, nil
 }