Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split runtime validate #391

Closed
wants to merge 1 commit into from
Closed

Split runtime validate #391

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 168 additions & 32 deletions validation/validation_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
package validation

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/hashicorp/go-multierror"
"github.com/mndrix/tap-go"
"github.com/mrunalp/fileutils"
rspec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"github.com/satori/go.uuid"
)
Expand All @@ -17,54 +22,88 @@ var (
runtime = "runc"
)

type validation struct {
test func(string, string, *rspec.Spec) error
description string
}

func init() {
runtime = os.Getenv("RUNTIME")
}

func runtimeValidate(runtime string, g *generate.Generator) error {
func TestValidateRuntimeInside(t *testing.T) {
g, err := getDefaultGenerator()
if err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
}
g.SetProcessArgs([]string{"/runtimetest"})

if err := runtimeInsideValidate(runtime, g); err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
}
}

func TestValidateRuntimeOutside(t *testing.T) {
g, err := getDefaultGenerator()
if err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
}

if err := runtimeOutsideValidate(runtime, g); err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
}
}

func runtimeInsideValidate(runtime string, g *generate.Generator) error {
// Find the runtime binary in the PATH
runtimePath, err := exec.LookPath(runtime)
if err != nil {
return err
}

// Setup a temporary test directory
tmpDir, err := ioutil.TempDir("", "ocitest")
bundleDir, rootfsDir, err := prepareBundle(g)
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)
defer os.RemoveAll(bundleDir)

// Create bundle directory for the test container
bundleDir := tmpDir + "/busybox"
if err := os.MkdirAll(bundleDir, 0755); err != nil {
// Copy the runtimetest binary to the rootfs
err = fileutils.CopyFile("../runtimetest", filepath.Join(rootfsDir, "runtimetest"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this actually work? My local build of runtimetest is dynamically linked:

$ lddtree runtimetest 
runtimetest (interpreter => /lib64/ld-linux-x86-64.so.2)
    libpthread.so.0 => /lib64/libpthread.so.0
    libc.so.6 => /lib64/libc.so.6

and neither that linker nor those libraries aren't part of our Busybox rootfs. For example, here is the statically-linked Busybox working fine in a chroot based on that rootfs:

$ unshare -rUmfp --mount-proc sh -c 'mount --rbind /proc proc && chroot . sh -c "pwd && ls"'
/
bin          etc          proc         runtimetest

And here is runtimetest failing to launch in the same situation:

$ unshare -rUmfp --mount-proc sh -c 'mount --rbind /proc proc && chroot . /runtimetest'
chroot: failed to run command ‘/runtimetest’: No such file or directory

I expect we're going to need to either compile runtimetest as a static binary (#442), copy the linker and required libraries into rootfs, or use busybox for our internal tests.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need #442 merged first

if err != nil {
return err
}

// Untar the root fs
untarCmd := exec.Command("tar", "-xf", "../rootfs.tar.gz", "-C", bundleDir)
output, err := untarCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
// TODO: Use a library to split run into create/start
// Launch the OCI runtime
containerID := uuid.NewV4()
runtimeCmd := exec.Command(runtimePath, "run", containerID.String())
runtimeCmd.Dir = bundleDir
runtimeCmd.Stdin = os.Stdin
runtimeCmd.Stdout = os.Stdout
runtimeCmd.Stderr = os.Stderr
if err = runtimeCmd.Run(); err != nil {
return err
}

// Copy the runtimetest binary to the rootfs
err = fileutils.CopyFile("../runtimetest", filepath.Join(bundleDir, "runtimetest"))
return nil
}

func runtimeOutsideValidate(runtime string, g *generate.Generator) error {
// Find the runtime binary in the PATH
runtimePath, err := exec.LookPath(runtime)
if err != nil {
return err
}

// Generate test configuration
err = g.SaveToFile(filepath.Join(bundleDir, "config.json"), generate.ExportOptions{})
bundleDir, _, err := prepareBundle(g)
if err != nil {
return err
}
defer os.RemoveAll(bundleDir)

// TODO: Use a library to split run into create/start
// Launch the OCI runtime
containerID := uuid.NewV4()
runtimeCmd := exec.Command(runtimePath, "run", containerID.String())
runtimeCmd := exec.Command(runtimePath, "create", containerID.String())
runtimeCmd.Dir = bundleDir
runtimeCmd.Stdin = os.Stdin
runtimeCmd.Stdout = os.Stdout
Expand All @@ -73,29 +112,126 @@ func runtimeValidate(runtime string, g *generate.Generator) error {
return err
}

outsideValidations := []validation{
{
test: validateLabels,
description: "labels",
},
// Add more container outside validation
}

t := tap.New()
t.Header(0)

var validationErrors error
for _, v := range outsideValidations {
err := v.test(runtimePath, containerID.String(), g.Spec())
t.Ok(err == nil, v.description)
if err != nil {
validationErrors = multierror.Append(validationErrors, err)
}
}
t.AutoPlan()

if err = cleanup(runtimePath, containerID.String()); err != nil {
validationErrors = multierror.Append(validationErrors, err)
}

return validationErrors
}

func validateLabels(runtimePath, id string, spec *rspec.Spec) error {
runtimeCmd := exec.Command(runtimePath, "state", id)
output, err := runtimeCmd.Output()
if err != nil {
return err
}

var state rspec.State
if err := json.NewDecoder(strings.NewReader(string(output))).Decode(&state); err != nil {
return err
}
for key, value := range spec.Annotations {
if state.Annotations[key] == value {
continue
}
return fmt.Errorf("Expected annotation %s:%s not set", key, value)
}
return nil
}

func getDefaultGenerator() *generate.Generator {
g := generate.New()
g.SetRootPath(".")
g.SetProcessArgs([]string{"/runtimetest"})
return &g
func cleanup(runtimePath, id string) error {
runtimeCmd := exec.Command(runtimePath, "kill", id, "KILL")
if err := runtimeCmd.Run(); err != nil {
return fmt.Errorf("Failed to kill container %s: %v", id, err)
}

runtimeCmd = exec.Command(runtimePath, "delete", id)
if err := runtimeCmd.Run(); err != nil {
return fmt.Errorf("Failed to kill container %s: %v", id, err)
}

return nil
}

func TestValidateBasic(t *testing.T) {
g := getDefaultGenerator()
func prepareBundle(g *generate.Generator) (string, string, error) {
// Setup a temporary test directory
tmpDir, err := ioutil.TempDir("", "ocitest")
if err != nil {
return "", "", err
}

if err := runtimeValidate(runtime, g); err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
// Create bundle directory for the test container
bundleDir := tmpDir
if err := os.MkdirAll(bundleDir, 0755); err != nil {
return "", "", err
}

// Create rootfs directory for the test container
rootfsDir := bundleDir + "/rootfs"
if err := os.MkdirAll(rootfsDir, 0755); err != nil {
return "", "", err
}

// Untar the root fs
untarCmd := exec.Command("tar", "-xf", "../rootfs.tar.gz", "-C", rootfsDir)
output, err := untarCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
return "", "", err
}

// Generate test configuration
err = g.SaveToFile(filepath.Join(bundleDir, "config.json"), generate.ExportOptions{})
if err != nil {
return "", "", err
}

// Copy the configuration file to the rootfs
err = fileutils.CopyFile(filepath.Join(bundleDir, "config.json"), filepath.Join(rootfsDir, "config.json"))
if err != nil {
return "", "", err
}

return bundleDir, rootfsDir, nil
}

func TestValidateSysctls(t *testing.T) {
g := getDefaultGenerator()
g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
func getDefaultGenerator() (*generate.Generator, error) {
// Generate testcase template
generateCmd := exec.Command("oci-runtime-tool", "generate", "--mount-bind=/tmp:/volume/testing:rw", "--linux-cgroups-path=/tmp/testcgroup", "--linux-device-add=c:80:500:/dev/test:fileMode=438", "--linux-disable-oom-kill=true", "--env=testvar=vartest", "--hostname=localvalidation", "--label=testlabel=nonevar", "--linux-cpu-shares=1024", "--output", "/tmp/config.json")
output, err := generateCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
return nil, err
}

if err := runtimeValidate(runtime, g); err != nil {
t.Errorf("%s failed validation: %v", runtime, err)
// Get testcase configuration
g, err := generate.NewFromFile("/tmp/config.json")
if err != nil {
return nil, err
}

g.SetRootPath("rootfs")

return &g, nil
}