diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json new file mode 100644 index 0000000..44f6900 --- /dev/null +++ b/Godeps/Godeps.json @@ -0,0 +1,73 @@ +{ + "ImportPath": "github.com/appc/acpush", + "GoVersion": "go1.5.1", + "Packages": [ + "./..." + ], + "Deps": [ + { + "ImportPath": "github.com/appc/spec/aci", + "Comment": "v0.7.1-3-g89715a6", + "Rev": "89715a66b8f8ac3750a4986022747c2da73fdcf1" + }, + { + "ImportPath": "github.com/appc/spec/discovery", + "Comment": "v0.7.1-3-g89715a6", + "Rev": "89715a66b8f8ac3750a4986022747c2da73fdcf1" + }, + { + "ImportPath": "github.com/appc/spec/pkg/device", + "Comment": "v0.7.1-3-g89715a6", + "Rev": "89715a66b8f8ac3750a4986022747c2da73fdcf1" + }, + { + "ImportPath": "github.com/appc/spec/pkg/tarheader", + "Comment": "v0.7.1-3-g89715a6", + "Rev": "89715a66b8f8ac3750a4986022747c2da73fdcf1" + }, + { + "ImportPath": "github.com/appc/spec/schema", + "Comment": "v0.7.1-3-g89715a6", + "Rev": "89715a66b8f8ac3750a4986022747c2da73fdcf1" + }, + { + "ImportPath": "github.com/coreos/go-semver/semver", + "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa" + }, + { + "ImportPath": "github.com/coreos/ioprogress", + "Rev": "a9d1979d9f4e84fe8d6ab9f4760757dd67ab6be1" + }, + { + "ImportPath": "github.com/coreos/rkt/common", + "Comment": "v0.8.1-191-g0d4bbae", + "Rev": "0d4bbae500b8f72674c455b5e8c9264cc2be1a6f" + }, + { + "ImportPath": "github.com/coreos/rkt/rkt/config", + "Comment": "v0.8.1-191-g0d4bbae", + "Rev": "0d4bbae500b8f72674c455b5e8c9264cc2be1a6f" + }, + { + "ImportPath": "github.com/spf13/pflag", + "Rev": "67cbc198fd11dab704b214c1e629a97af392c085" + }, + { + "ImportPath": "golang.org/x/crypto/ssh/terminal", + "Rev": "a7ead6ddf06233883deca151dffaef2effbf498f" + }, + { + "ImportPath": "golang.org/x/net/html", + "Rev": "c2528b2dd8352441850638a8bb678c2ad056fd3e" + }, + { + "ImportPath": "k8s.io/kubernetes/pkg/api/resource", + "Comment": "v0.12.0-270-g53ec66c", + "Rev": "53ec66caf4e952a1384ec93b9f0cde37616e4caf" + }, + { + "ImportPath": "speter.net/go/exp/math/dec/inf", + "Rev": "42ca6cd68aa922bc3f32f1e056e61b65945d9ad7" + } + ] +} diff --git a/Godeps/Readme b/Godeps/Readme new file mode 100644 index 0000000..4cdaa53 --- /dev/null +++ b/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/.gitignore b/Godeps/_workspace/.gitignore new file mode 100644 index 0000000..f037d68 --- /dev/null +++ b/Godeps/_workspace/.gitignore @@ -0,0 +1,2 @@ +/pkg +/bin diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/build.go b/Godeps/_workspace/src/github.com/appc/spec/aci/build.go new file mode 100644 index 0000000..a594eb9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/build.go @@ -0,0 +1,110 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "archive/tar" + "io" + "os" + "path/filepath" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/pkg/tarheader" +) + +// TarHeaderWalkFunc is the type of the function which allows setting tar +// headers or filtering out tar entries when building an ACI. It will be +// applied to every entry in the tar file. +// +// If true is returned, the entry will be included in the final ACI; if false, +// the entry will not be included. +type TarHeaderWalkFunc func(hdr *tar.Header) bool + +// BuildWalker creates a filepath.WalkFunc that walks over the given root +// (which should represent an ACI layout on disk) and adds the files in the +// rootfs/ subdirectory to the given ArchiveWriter +func BuildWalker(root string, aw ArchiveWriter, cb TarHeaderWalkFunc) filepath.WalkFunc { + // cache of inode -> filepath, used to leverage hard links in the archive + inos := map[uint64]string{} + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relpath, err := filepath.Rel(root, path) + if err != nil { + return err + } + if relpath == "." { + return nil + } + if relpath == ManifestFile { + // ignore; this will be written by the archive writer + // TODO(jonboulle): does this make sense? maybe just remove from archivewriter? + return nil + } + + link := "" + var r io.Reader + switch info.Mode() & os.ModeType { + case os.ModeSocket: + return nil + case os.ModeNamedPipe: + case os.ModeCharDevice: + case os.ModeDevice: + case os.ModeDir: + case os.ModeSymlink: + target, err := os.Readlink(path) + if err != nil { + return err + } + link = target + default: + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + r = file + } + + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + panic(err) + } + // Because os.FileInfo's Name method returns only the base + // name of the file it describes, it may be necessary to + // modify the Name field of the returned header to provide the + // full path name of the file. + hdr.Name = relpath + tarheader.Populate(hdr, info, inos) + // If the file is a hard link to a file we've already seen, we + // don't need the contents + if hdr.Typeflag == tar.TypeLink { + hdr.Size = 0 + r = nil + } + + if cb != nil { + if !cb(hdr) { + return nil + } + } + + if err := aw.AddFile(hdr, r); err != nil { + return err + } + + return nil + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go b/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go new file mode 100644 index 0000000..624d431 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/doc.go @@ -0,0 +1,16 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package aci contains various functions for working with App Container Images. +package aci diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/file.go b/Godeps/_workspace/src/github.com/appc/spec/aci/file.go new file mode 100644 index 0000000..4ec6826 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/file.go @@ -0,0 +1,246 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "archive/tar" + "bytes" + "compress/bzip2" + "compress/gzip" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os/exec" + "path/filepath" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema" +) + +type FileType string + +const ( + TypeGzip = FileType("gz") + TypeBzip2 = FileType("bz2") + TypeXz = FileType("xz") + TypeTar = FileType("tar") + TypeText = FileType("text") + TypeUnknown = FileType("unknown") + + readLen = 512 // max bytes to sniff + + hexHdrGzip = "1f8b" + hexHdrBzip2 = "425a68" + hexHdrXz = "fd377a585a00" + hexSigTar = "7573746172" + + tarOffset = 257 + + textMime = "text/plain; charset=utf-8" +) + +var ( + hdrGzip []byte + hdrBzip2 []byte + hdrXz []byte + sigTar []byte + tarEnd int +) + +func mustDecodeHex(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func init() { + hdrGzip = mustDecodeHex(hexHdrGzip) + hdrBzip2 = mustDecodeHex(hexHdrBzip2) + hdrXz = mustDecodeHex(hexHdrXz) + sigTar = mustDecodeHex(hexSigTar) + tarEnd = tarOffset + len(sigTar) +} + +// DetectFileType attempts to detect the type of file that the given reader +// represents by comparing it against known file signatures (magic numbers) +func DetectFileType(r io.Reader) (FileType, error) { + var b bytes.Buffer + n, err := io.CopyN(&b, r, readLen) + if err != nil && err != io.EOF { + return TypeUnknown, err + } + bs := b.Bytes() + switch { + case bytes.HasPrefix(bs, hdrGzip): + return TypeGzip, nil + case bytes.HasPrefix(bs, hdrBzip2): + return TypeBzip2, nil + case bytes.HasPrefix(bs, hdrXz): + return TypeXz, nil + case n > int64(tarEnd) && bytes.Equal(bs[tarOffset:tarEnd], sigTar): + return TypeTar, nil + case http.DetectContentType(bs) == textMime: + return TypeText, nil + default: + return TypeUnknown, nil + } +} + +// XzReader is an io.ReadCloser which decompresses xz compressed data. +type XzReader struct { + io.ReadCloser + cmd *exec.Cmd + closech chan error +} + +// NewXzReader shells out to a command line xz executable (if +// available) to decompress the given io.Reader using the xz +// compression format and returns an *XzReader. +// It is the caller's responsibility to call Close on the XzReader when done. +func NewXzReader(r io.Reader) (*XzReader, error) { + rpipe, wpipe := io.Pipe() + ex, err := exec.LookPath("xz") + if err != nil { + log.Fatalf("couldn't find xz executable: %v", err) + } + cmd := exec.Command(ex, "--decompress", "--stdout") + + closech := make(chan error) + + cmd.Stdin = r + cmd.Stdout = wpipe + + go func() { + err := cmd.Run() + wpipe.CloseWithError(err) + closech <- err + }() + + return &XzReader{rpipe, cmd, closech}, nil +} + +func (r *XzReader) Close() error { + r.ReadCloser.Close() + r.cmd.Process.Kill() + return <-r.closech +} + +// ManifestFromImage extracts a new schema.ImageManifest from the given ACI image. +func ManifestFromImage(rs io.ReadSeeker) (*schema.ImageManifest, error) { + var im schema.ImageManifest + + tr, err := NewCompressedTarReader(rs) + if err != nil { + return nil, err + } + defer tr.Close() + + for { + hdr, err := tr.Next() + switch err { + case io.EOF: + return nil, errors.New("missing manifest") + case nil: + if filepath.Clean(hdr.Name) == ManifestFile { + data, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + if err := im.UnmarshalJSON(data); err != nil { + return nil, err + } + return &im, nil + } + default: + return nil, fmt.Errorf("error extracting tarball: %v", err) + } + } +} + +// TarReadCloser embeds a *tar.Reader and the related io.Closer +// It is the caller's responsibility to call Close on TarReadCloser when +// done. +type TarReadCloser struct { + *tar.Reader + io.Closer +} + +func (r *TarReadCloser) Close() error { + return r.Closer.Close() +} + +// NewCompressedTarReader creates a new TarReadCloser reading from the +// given ACI image. +// It is the caller's responsibility to call Close on the TarReadCloser +// when done. +func NewCompressedTarReader(rs io.ReadSeeker) (*TarReadCloser, error) { + cr, err := NewCompressedReader(rs) + if err != nil { + return nil, err + } + return &TarReadCloser{tar.NewReader(cr), cr}, nil +} + +// NewCompressedReader creates a new io.ReaderCloser from the given ACI image. +// It is the caller's responsibility to call Close on the Reader when done. +func NewCompressedReader(rs io.ReadSeeker) (io.ReadCloser, error) { + + var ( + dr io.ReadCloser + err error + ) + + _, err = rs.Seek(0, 0) + if err != nil { + return nil, err + } + + ftype, err := DetectFileType(rs) + if err != nil { + return nil, err + } + + _, err = rs.Seek(0, 0) + if err != nil { + return nil, err + } + + switch ftype { + case TypeGzip: + dr, err = gzip.NewReader(rs) + if err != nil { + return nil, err + } + case TypeBzip2: + dr = ioutil.NopCloser(bzip2.NewReader(rs)) + case TypeXz: + dr, err = NewXzReader(rs) + if err != nil { + return nil, err + } + case TypeTar: + dr = ioutil.NopCloser(rs) + case TypeUnknown: + return nil, errors.New("error: unknown image filetype") + default: + return nil, errors.New("no type returned from DetectFileType?") + } + return dr, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go b/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go new file mode 100644 index 0000000..ea4532b --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/file_test.go @@ -0,0 +1,150 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "archive/tar" + "compress/gzip" + "io/ioutil" + "os" + "testing" +) + +func newTestACI(usedotslash bool) (*os.File, error) { + tf, err := ioutil.TempFile("", "") + if err != nil { + return nil, err + } + + manifestBody := `{"acKind":"ImageManifest","acVersion":"0.7.1","name":"example.com/app"}` + + gw := gzip.NewWriter(tf) + tw := tar.NewWriter(gw) + + manifestPath := "manifest" + if usedotslash { + manifestPath = "./" + manifestPath + } + hdr := &tar.Header{ + Name: manifestPath, + Size: int64(len(manifestBody)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(manifestBody)); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + return tf, nil +} + +func newEmptyTestACI() (*os.File, error) { + tf, err := ioutil.TempFile("", "") + if err != nil { + return nil, err + } + gw := gzip.NewWriter(tf) + tw := tar.NewWriter(gw) + if err := tw.Close(); err != nil { + return nil, err + } + if err := gw.Close(); err != nil { + return nil, err + } + return tf, nil +} + +func TestManifestFromImage(t *testing.T) { + for _, usedotslash := range []bool{false, true} { + img, err := newTestACI(usedotslash) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + im, err := ManifestFromImage(img) + if err != nil { + t.Fatalf("ManifestFromImage: unexpected error: %v", err) + } + if im.Name.String() != "example.com/app" { + t.Errorf("expected %s, got %s", "example.com/app", im.Name.String()) + } + + emptyImg, err := newEmptyTestACI() + if err != nil { + t.Fatalf("newEmptyTestACI: unexpected error: %v", err) + } + defer emptyImg.Close() + defer os.Remove(emptyImg.Name()) + + im, err = ManifestFromImage(emptyImg) + if err == nil { + t.Fatalf("ManifestFromImage: expected error") + } + } +} + +func TestNewCompressedTarReader(t *testing.T) { + img, err := newTestACI(false) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + cr, err := NewCompressedTarReader(img) + if err != nil { + t.Fatalf("NewCompressedTarReader: unexpected error: %v", err) + } + + ftype, err := DetectFileType(cr) + if err != nil { + t.Fatalf("DetectFileType: unexpected error: %v", err) + } + + if ftype != TypeText { + t.Errorf("expected %v, got %v", TypeText, ftype) + } +} + +func TestNewCompressedReader(t *testing.T) { + img, err := newTestACI(false) + if err != nil { + t.Fatalf("newTestACI: unexpected error: %v", err) + } + defer img.Close() + defer os.Remove(img.Name()) + + cr, err := NewCompressedReader(img) + if err != nil { + t.Fatalf("NewCompressedReader: unexpected error: %v", err) + } + + ftype, err := DetectFileType(cr) + if err != nil { + t.Fatalf("DetectFileType: unexpected error: %v", err) + } + + if ftype != TypeTar { + t.Errorf("expected %v, got %v", TypeTar, ftype) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go b/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go new file mode 100644 index 0000000..ce4ac7f --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/layout.go @@ -0,0 +1,187 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +/* + +Image Layout + +The on-disk layout of an app container is straightforward. +It includes a rootfs with all of the files that will exist in the root of the app and a manifest describing the image. +The layout MUST contain an image manifest. + +/manifest +/rootfs/ +/rootfs/usr/bin/mysql + +*/ + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema" + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema/types" +) + +const ( + // Path to manifest file inside the layout + ManifestFile = "manifest" + // Path to rootfs directory inside the layout + RootfsDir = "rootfs" +) + +type ErrOldVersion struct { + version types.SemVer +} + +func (e ErrOldVersion) Error() string { + return fmt.Sprintf("ACVersion too old. Found major version %v, expected %v", e.version.Major, schema.AppContainerVersion.Major) +} + +var ( + ErrNoRootFS = errors.New("no rootfs found in layout") + ErrNoManifest = errors.New("no image manifest found in layout") +) + +// ValidateLayout takes a directory and validates that the layout of the directory +// matches that expected by the Application Container Image format. +// If any errors are encountered during the validation, it will abort and +// return the first one. +func ValidateLayout(dir string) error { + fi, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("error accessing layout: %v", err) + } + if !fi.IsDir() { + return fmt.Errorf("given path %q is not a directory", dir) + } + var flist []string + var imOK, rfsOK bool + var im io.Reader + walkLayout := func(fpath string, fi os.FileInfo, err error) error { + rpath, err := filepath.Rel(dir, fpath) + if err != nil { + return err + } + switch rpath { + case ".": + case ManifestFile: + im, err = os.Open(fpath) + if err != nil { + return err + } + imOK = true + case RootfsDir: + if !fi.IsDir() { + return errors.New("rootfs is not a directory") + } + rfsOK = true + default: + flist = append(flist, rpath) + } + return nil + } + if err := filepath.Walk(dir, walkLayout); err != nil { + return err + } + return validate(imOK, im, rfsOK, flist) +} + +// ValidateArchive takes a *tar.Reader and validates that the layout of the +// filesystem the reader encapsulates matches that expected by the +// Application Container Image format. If any errors are encountered during +// the validation, it will abort and return the first one. +func ValidateArchive(tr *tar.Reader) error { + var fseen map[string]bool = make(map[string]bool) + var imOK, rfsOK bool + var im bytes.Buffer +Tar: + for { + hdr, err := tr.Next() + switch { + case err == nil: + case err == io.EOF: + break Tar + default: + return err + } + name := filepath.Clean(hdr.Name) + switch name { + case ".": + case ManifestFile: + _, err := io.Copy(&im, tr) + if err != nil { + return err + } + imOK = true + case RootfsDir: + if !hdr.FileInfo().IsDir() { + return fmt.Errorf("rootfs is not a directory") + } + rfsOK = true + default: + if _, seen := fseen[name]; seen { + return fmt.Errorf("duplicate file entry in archive: %s", name) + } + fseen[name] = true + } + } + var flist []string + for key := range fseen { + flist = append(flist, key) + } + return validate(imOK, &im, rfsOK, flist) +} + +func validate(imOK bool, im io.Reader, rfsOK bool, files []string) error { + defer func() { + if rc, ok := im.(io.Closer); ok { + rc.Close() + } + }() + if !imOK { + return ErrNoManifest + } + if !rfsOK { + return ErrNoRootFS + } + b, err := ioutil.ReadAll(im) + if err != nil { + return fmt.Errorf("error reading image manifest: %v", err) + } + var a schema.ImageManifest + if err := a.UnmarshalJSON(b); err != nil { + return fmt.Errorf("image manifest validation failed: %v", err) + } + if a.ACVersion.LessThanMajor(schema.AppContainerVersion) { + return ErrOldVersion{ + version: a.ACVersion, + } + } + for _, f := range files { + if !strings.HasPrefix(f, "rootfs") { + return fmt.Errorf("unrecognized file path in layout: %q", f) + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go b/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go new file mode 100644 index 0000000..44e0856 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/layout_test.go @@ -0,0 +1,79 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema" +) + +func newValidateLayoutTest() (string, error) { + td, err := ioutil.TempDir("", "") + if err != nil { + return "", err + } + + if err := os.MkdirAll(path.Join(td, "rootfs"), 0755); err != nil { + return "", err + } + + if err := os.MkdirAll(path.Join(td, "rootfs", "dir", "rootfs"), 0755); err != nil { + return "", err + } + + evilManifestBody := "malformedManifest" + manifestBody := fmt.Sprintf(`{"acKind":"ImageManifest","acVersion":"%s","name":"example.com/app"}`, schema.AppContainerVersion) + + evilManifestPath := "rootfs/manifest" + evilManifestPath = path.Join(td, evilManifestPath) + + em, err := os.Create(evilManifestPath) + if err != nil { + return "", err + } + + em.WriteString(evilManifestBody) + em.Close() + + manifestPath := path.Join(td, "manifest") + + m, err := os.Create(manifestPath) + if err != nil { + return "", err + } + + m.WriteString(manifestBody) + m.Close() + + return td, nil +} + +func TestValidateLayout(t *testing.T) { + layoutPath, err := newValidateLayoutTest() + if err != nil { + t.Fatalf("newValidateLayoutTest: unexpected error: %v", err) + } + defer os.RemoveAll(layoutPath) + + err = ValidateLayout(layoutPath) + if err != nil { + t.Fatalf("ValidateLayout: unexpected error: %v", err) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go b/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go new file mode 100644 index 0000000..328bbc6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/aci/writer.go @@ -0,0 +1,98 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aci + +import ( + "archive/tar" + "bytes" + "encoding/json" + "io" + "time" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema" +) + +// ArchiveWriter writes App Container Images. Users wanting to create an ACI or +// should create an ArchiveWriter and add files to it; the ACI will be written +// to the underlying tar.Writer +type ArchiveWriter interface { + AddFile(hdr *tar.Header, r io.Reader) error + Close() error +} + +type imageArchiveWriter struct { + *tar.Writer + am *schema.ImageManifest +} + +// NewImageWriter creates a new ArchiveWriter which will generate an App +// Container Image based on the given manifest and write it to the given +// tar.Writer +func NewImageWriter(am schema.ImageManifest, w *tar.Writer) ArchiveWriter { + aw := &imageArchiveWriter{ + w, + &am, + } + return aw +} + +func (aw *imageArchiveWriter) AddFile(hdr *tar.Header, r io.Reader) error { + err := aw.Writer.WriteHeader(hdr) + if err != nil { + return err + } + + if r != nil { + _, err := io.Copy(aw.Writer, r) + if err != nil { + return err + } + } + + return nil +} + +func (aw *imageArchiveWriter) addFileNow(path string, contents []byte) error { + buf := bytes.NewBuffer(contents) + now := time.Now() + hdr := tar.Header{ + Name: path, + Mode: 0644, + Uid: 0, + Gid: 0, + Size: int64(buf.Len()), + ModTime: now, + Typeflag: tar.TypeReg, + Uname: "root", + Gname: "root", + ChangeTime: now, + } + return aw.AddFile(&hdr, buf) +} + +func (aw *imageArchiveWriter) addManifest(name string, m json.Marshaler) error { + out, err := m.MarshalJSON() + if err != nil { + return err + } + return aw.addFileNow(name, out) +} + +func (aw *imageArchiveWriter) Close() error { + if err := aw.addManifest(ManifestFile, aw.am); err != nil { + return err + } + return aw.Writer.Close() +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery.go b/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery.go new file mode 100644 index 0000000..ba95c2a --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery.go @@ -0,0 +1,260 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/appc/acpush/Godeps/_workspace/src/golang.org/x/net/html" + "github.com/appc/acpush/Godeps/_workspace/src/golang.org/x/net/html/atom" +) + +type acMeta struct { + name string + prefix string + uri string +} + +type ACIEndpoint struct { + ACI string + ASC string +} + +type Endpoints struct { + ACIEndpoints []ACIEndpoint + Keys []string + ACIPushEndpoints []string +} + +func (e *Endpoints) Append(ep Endpoints) { + e.ACIEndpoints = append(e.ACIEndpoints, ep.ACIEndpoints...) + e.Keys = append(e.Keys, ep.Keys...) + e.ACIPushEndpoints = append(e.ACIPushEndpoints, ep.ACIPushEndpoints...) +} + +const ( + defaultVersion = "latest" +) + +var ( + templateExpression = regexp.MustCompile(`{.*?}`) + errEnough = errors.New("enough discovery information found") +) + +func appendMeta(meta []acMeta, attrs []html.Attribute) []acMeta { + m := acMeta{} + + for _, a := range attrs { + if a.Namespace != "" { + continue + } + + switch a.Key { + case "name": + m.name = a.Val + + case "content": + parts := strings.SplitN(strings.TrimSpace(a.Val), " ", 2) + if len(parts) < 2 { + break + } + m.prefix = parts[0] + m.uri = strings.TrimSpace(parts[1]) + } + } + + // TODO(eyakubovich): should prefix be optional? + if !strings.HasPrefix(m.name, "ac-") || m.prefix == "" || m.uri == "" { + return meta + } + + return append(meta, m) +} + +func extractACMeta(r io.Reader) []acMeta { + var meta []acMeta + + z := html.NewTokenizer(r) + + for { + switch z.Next() { + case html.ErrorToken: + return meta + + case html.StartTagToken, html.SelfClosingTagToken: + tok := z.Token() + if tok.DataAtom == atom.Meta { + meta = appendMeta(meta, tok.Attr) + } + } + } +} + +func renderTemplate(tpl string, kvs ...string) (string, bool) { + for i := 0; i < len(kvs); i += 2 { + k := kvs[i] + v := kvs[i+1] + tpl = strings.Replace(tpl, k, v, -1) + } + return tpl, !templateExpression.MatchString(tpl) +} + +func createTemplateVars(app App) []string { + tplVars := []string{"{name}", app.Name.String()} + // If a label is called "name", it will be ignored as it appears after + // in the slice + for n, v := range app.Labels { + tplVars = append(tplVars, fmt.Sprintf("{%s}", n), v) + } + return tplVars +} + +func doDiscover(pre string, app App, insecure bool) (*Endpoints, error) { + app = *app.Copy() + if app.Labels["version"] == "" { + app.Labels["version"] = defaultVersion + } + + _, body, err := httpsOrHTTP(pre, insecure) + if err != nil { + return nil, err + } + defer body.Close() + + meta := extractACMeta(body) + + tplVars := createTemplateVars(app) + + de := &Endpoints{} + + for _, m := range meta { + if !strings.HasPrefix(app.Name.String(), m.prefix) { + continue + } + + switch m.name { + case "ac-discovery": + // Ignore not handled variables as {ext} isn't already rendered. + uri, _ := renderTemplate(m.uri, tplVars...) + asc, ok := renderTemplate(uri, "{ext}", "aci.asc") + if !ok { + continue + } + aci, ok := renderTemplate(uri, "{ext}", "aci") + if !ok { + continue + } + de.ACIEndpoints = append(de.ACIEndpoints, ACIEndpoint{ACI: aci, ASC: asc}) + + case "ac-discovery-pubkeys": + de.Keys = append(de.Keys, m.uri) + case "ac-push-discovery": + uri, _ := renderTemplate(m.uri, tplVars...) + de.ACIPushEndpoints = append(de.ACIPushEndpoints, uri) + } + } + + return de, nil +} + +// DiscoverWalk will make HTTPS requests to find discovery meta tags and +// optionally will use HTTP if insecure is set. Based on the response of the +// discoverFn it will continue to recurse up the tree. +func DiscoverWalk(app App, insecure bool, discoverFn DiscoverWalkFunc) (err error) { + var ( + eps *Endpoints + ) + + parts := strings.Split(string(app.Name), "/") + for i := range parts { + end := len(parts) - i + pre := strings.Join(parts[:end], "/") + + eps, err = doDiscover(pre, app, insecure) + if derr := discoverFn(pre, eps, err); derr != nil { + return derr + } + } + + return +} + +// DiscoverWalkFunc can stop a DiscoverWalk by returning non-nil error. +type DiscoverWalkFunc func(prefix string, eps *Endpoints, err error) error + +// FailedAttempt represents a failed discovery attempt. This is for debugging +// and user feedback. +type FailedAttempt struct { + Prefix string + Error error +} + +func walker(out *Endpoints, attempts *[]FailedAttempt, testFn DiscoverWalkFunc) DiscoverWalkFunc { + return func(pre string, eps *Endpoints, err error) error { + if err != nil { + *attempts = append(*attempts, FailedAttempt{pre, err}) + return nil + } + out.Append(*eps) + if err := testFn(pre, eps, err); err != nil { + return err + } + return nil + } +} + +// DiscoverEndpoints will make HTTPS requests to find the ac-discovery meta +// tags and optionally will use HTTP if insecure is set. It will not give up +// until it has exhausted the path or found an image discovery. +func DiscoverEndpoints(app App, insecure bool) (out *Endpoints, attempts []FailedAttempt, err error) { + out = &Endpoints{} + testFn := func(pre string, eps *Endpoints, err error) error { + if len(out.ACIEndpoints) != 0 || len(out.Keys) != 0 || len(out.ACIPushEndpoints) != 0 { + return errEnough + } + return nil + } + + err = DiscoverWalk(app, insecure, walker(out, &attempts, testFn)) + if err != nil && err != errEnough { + return nil, attempts, err + } + + return out, attempts, nil +} + +// DiscoverPublicKey will make HTTPS requests to find the ac-public-keys meta +// tags and optionally will use HTTP if insecure is set. It will not give up +// until it has exhausted the path or found an public key. +func DiscoverPublicKeys(app App, insecure bool) (out *Endpoints, attempts []FailedAttempt, err error) { + out = &Endpoints{} + testFn := func(pre string, eps *Endpoints, err error) error { + if len(out.Keys) != 0 { + return errEnough + } + return nil + } + + err = DiscoverWalk(app, insecure, walker(out, &attempts, testFn)) + if err != nil && err != errEnough { + return nil, attempts, err + } + + return out, attempts, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery_test.go b/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery_test.go new file mode 100644 index 0000000..4cbd7ed --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/discovery_test.go @@ -0,0 +1,227 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/appc/acpush/Godeps/_workspace/src/github.com/appc/spec/schema/types" +) + +func fakeHTTPGet(filename string, failures int) func(uri string) (*http.Response, error) { + attempts := 0 + return func(uri string) (*http.Response, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + + var resp *http.Response + + switch { + case attempts < failures: + resp = &http.Response{ + Status: "404 Not Found", + StatusCode: http.StatusNotFound, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Content-Type": []string{"text/html"}, + }, + Body: ioutil.NopCloser(bytes.NewBufferString("")), + } + default: + resp = &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Content-Type": []string{"text/html"}, + }, + Body: f, + } + } + + attempts = attempts + 1 + return resp, nil + } +} + +type httpgetter func(uri string) (*http.Response, error) + +func TestDiscoverEndpoints(t *testing.T) { + tests := []struct { + get httpgetter + expectDiscoverySuccess bool + app App + expectedACIEndpoints []ACIEndpoint + expectedKeys []string + }{ + { + fakeHTTPGet("myapp.html", 0), + true, + App{ + Name: "example.com/myapp", + Labels: map[types.ACIdentifier]string{ + "version": "1.0.0", + "os": "linux", + "arch": "amd64", + }, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci?torrent", + ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc?torrent", + }, + ACIEndpoint{ + ACI: "hdfs://storage.example.com/example.com/myapp-1.0.0.aci", + ASC: "hdfs://storage.example.com/example.com/myapp-1.0.0.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + }, + { + fakeHTTPGet("myapp.html", 1), + true, + App{ + Name: "example.com/myapp/foobar", + Labels: map[types.ACIdentifier]string{ + "version": "1.0.0", + "os": "linux", + "arch": "amd64", + }, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp/foobar-1.0.0.aci?torrent", + ASC: "https://storage.example.com/example.com/myapp/foobar-1.0.0.aci.asc?torrent", + }, + ACIEndpoint{ + ACI: "hdfs://storage.example.com/example.com/myapp/foobar-1.0.0.aci", + ASC: "hdfs://storage.example.com/example.com/myapp/foobar-1.0.0.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + }, + { + fakeHTTPGet("myapp.html", 20), + false, + App{ + Name: "example.com/myapp/foobar/bazzer", + Labels: map[types.ACIdentifier]string{ + "version": "1.0.0", + "os": "linux", + "arch": "amd64", + }, + }, + []ACIEndpoint{}, + []string{}, + }, + // Test missing label. Only one ac-discovery template should be + // returned as the other one cannot be completely rendered due to + // missing labels. + { + fakeHTTPGet("myapp2.html", 0), + true, + App{ + Name: "example.com/myapp", + Labels: map[types.ACIdentifier]string{ + "version": "1.0.0", + }, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", + ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + }, + // Test missing labels. version label should default to + // "latest" and the first template should be rendered + { + fakeHTTPGet("myapp2.html", 0), + false, + App{ + Name: "example.com/myapp", + Labels: map[types.ACIdentifier]string{}, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-latest.aci", + ASC: "https://storage.example.com/example.com/myapp-latest.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + }, + // Test with a label called "name". It should be ignored. + { + fakeHTTPGet("myapp2.html", 0), + false, + App{ + Name: "example.com/myapp", + Labels: map[types.ACIdentifier]string{ + "name": "labelcalledname", + "version": "1.0.0", + }, + }, + []ACIEndpoint{ + ACIEndpoint{ + ACI: "https://storage.example.com/example.com/myapp-1.0.0.aci", + ASC: "https://storage.example.com/example.com/myapp-1.0.0.aci.asc", + }, + }, + []string{"https://example.com/pubkeys.gpg"}, + }, + } + + for i, tt := range tests { + httpGet = &mockHttpGetter{getter: tt.get} + de, _, err := DiscoverEndpoints(tt.app, true) + if err != nil && !tt.expectDiscoverySuccess { + continue + } + if err != nil { + t.Fatalf("#%d DiscoverEndpoints failed: %v", i, err) + } + + if len(de.ACIEndpoints) != len(tt.expectedACIEndpoints) { + t.Errorf("ACIEndpoints array is wrong length want %d got %d", len(tt.expectedACIEndpoints), len(de.ACIEndpoints)) + } else { + for n, _ := range de.ACIEndpoints { + if de.ACIEndpoints[n] != tt.expectedACIEndpoints[n] { + t.Errorf("#%d ACIEndpoints[%d] mismatch: want %v got %v", i, n, tt.expectedACIEndpoints[n], de.ACIEndpoints[n]) + } + } + } + + if len(de.Keys) != len(tt.expectedKeys) { + t.Errorf("Keys array is wrong length want %d got %d", len(tt.expectedKeys), len(de.Keys)) + } else { + for n, _ := range de.Keys { + if de.Keys[n] != tt.expectedKeys[n] { + t.Errorf("#%d sig[%d] mismatch: want %v got %v", i, n, tt.expectedKeys[n], de.Keys[n]) + } + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/doc.go b/Godeps/_workspace/src/github.com/appc/spec/discovery/doc.go new file mode 100644 index 0000000..55bfc3a --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/doc.go @@ -0,0 +1,17 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package discovery contains an experimental implementation of the Image +// Discovery section of the appc specification. +package discovery diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/http.go b/Godeps/_workspace/src/github.com/appc/spec/discovery/http.go new file mode 100644 index 0000000..4198201 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/http.go @@ -0,0 +1,91 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" +) + +const ( + defaultDialTimeout = 5 * time.Second +) + +var ( + // Client is the default http.Client used for discovery requests. + Client *http.Client + + // httpGet is the internal object used by discovery to retrieve URLs; it is + // defined here so it can be overridden for testing + httpGet httpGetter +) + +// httpGetter is an interface used to wrap http.Client for real requests and +// allow easy mocking in local tests. +type httpGetter interface { + Get(url string) (resp *http.Response, err error) +} + +func init() { + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: func(n, a string) (net.Conn, error) { + return net.DialTimeout(n, a, defaultDialTimeout) + }, + } + Client = &http.Client{ + Transport: t, + } + httpGet = Client +} + +func httpsOrHTTP(name string, insecure bool) (urlStr string, body io.ReadCloser, err error) { + fetch := func(scheme string) (urlStr string, res *http.Response, err error) { + u, err := url.Parse(scheme + "://" + name) + if err != nil { + return "", nil, err + } + u.RawQuery = "ac-discovery=1" + urlStr = u.String() + res, err = httpGet.Get(urlStr) + return + } + closeBody := func(res *http.Response) { + if res != nil { + res.Body.Close() + } + } + urlStr, res, err := fetch("https") + if err != nil || res.StatusCode != http.StatusOK { + if insecure { + closeBody(res) + urlStr, res, err = fetch("http") + } + } + + if res != nil && res.StatusCode != http.StatusOK { + err = fmt.Errorf("expected a 200 OK got %d", res.StatusCode) + } + + if err != nil { + closeBody(res) + return "", nil, err + } + return urlStr, res.Body, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/http_test.go b/Godeps/_workspace/src/github.com/appc/spec/discovery/http_test.go new file mode 100644 index 0000000..4aa76a8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/http_test.go @@ -0,0 +1,162 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "bytes" + "errors" + "io/ioutil" + "net/http" + "os" + "strings" + "testing" +) + +// mockHttpGetter defines a wrapper that allows returning a mocked response. +type mockHttpGetter struct { + getter func(url string) (resp *http.Response, err error) +} + +func (m *mockHttpGetter) Get(url string) (resp *http.Response, err error) { + return m.getter(url) +} + +func fakeHttpOrHttpsGet(filename string, httpSuccess bool, httpsSuccess bool, httpErrorCode int) func(uri string) (*http.Response, error) { + return func(uri string) (*http.Response, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + + var resp *http.Response + + switch { + case strings.HasPrefix(uri, "https://") && httpsSuccess: + fallthrough + case strings.HasPrefix(uri, "http://") && httpSuccess: + resp = &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Content-Type": []string{"text/html"}, + }, + Body: f, + } + case httpErrorCode > 0: + resp = &http.Response{ + Status: "Error", + StatusCode: httpErrorCode, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "Content-Type": []string{"text/html"}, + }, + Body: ioutil.NopCloser(bytes.NewBufferString("")), + } + default: + err = errors.New("fakeHttpOrHttpsGet failed as requested") + return nil, err + } + + return resp, nil + } +} + +func TestHttpsOrHTTP(t *testing.T) { + tests := []struct { + name string + insecure bool + get httpgetter + expectUrlStr string + expectSuccess bool + }{ + { + "good-server", + false, + fakeHttpOrHttpsGet("myapp.html", true, true, 0), + "https://good-server?ac-discovery=1", + true, + }, + { + "file-not-found", + false, + fakeHttpOrHttpsGet("myapp.html", false, false, 404), + "", + false, + }, + { + "completely-broken-server", + false, + fakeHttpOrHttpsGet("myapp.html", false, false, 0), + "", + false, + }, + { + "file-only-on-http", + false, // do not accept fallback on http + fakeHttpOrHttpsGet("myapp.html", true, false, 404), + "", + false, + }, + { + "file-only-on-http", + true, // accept fallback on http + fakeHttpOrHttpsGet("myapp.html", true, false, 404), + "http://file-only-on-http?ac-discovery=1", + true, + }, + { + "https-server-is-down", + true, // accept fallback on http + fakeHttpOrHttpsGet("myapp.html", true, false, 0), + "http://https-server-is-down?ac-discovery=1", + true, + }, + } + + for i, tt := range tests { + httpGet = &mockHttpGetter{getter: tt.get} + urlStr, body, err := httpsOrHTTP(tt.name, tt.insecure) + if tt.expectSuccess { + if err != nil { + t.Fatalf("#%d httpsOrHTTP failed: %v", i, err) + } + if urlStr == "" { + t.Fatalf("#%d httpsOrHTTP didn't return a urlStr", i) + } + if urlStr != tt.expectUrlStr { + t.Fatalf("#%d httpsOrHTTP urlStr mismatch: want %s got %s", + i, tt.expectUrlStr, urlStr) + } + if body == nil { + t.Fatalf("#%d httpsOrHTTP didn't return a body", i) + } + } else { + if err == nil { + t.Fatalf("#%d httpsOrHTTP should have failed", i) + } + if urlStr != "" { + t.Fatalf("#%d httpsOrHTTP should not have returned a urlStr", i) + } + if body != nil { + t.Fatalf("#%d httpsOrHTTP should not have returned a body", i) + } + } + } +} diff --git a/Godeps/_workspace/src/github.com/appc/spec/discovery/myapp.html b/Godeps/_workspace/src/github.com/appc/spec/discovery/myapp.html new file mode 100644 index 0000000..10c80eb --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/spec/discovery/myapp.html @@ -0,0 +1,15 @@ + + + +
+HTTP charset
+ + + + + +The character encoding of a page can be set using the HTTP header charset declaration.
+The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ
. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
The only character encoding declaration for this HTML file is in the HTTP header, which sets the encoding to ISO 8859-15.
+the-input-byte-stream-001
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs UTF-8 BOM
+ + + + + +A character encoding set in the HTTP header has lower precedence than the UTF-8 signature.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè
. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
If the test is unsuccessful, the characters  should appear at the top of the page. These represent the bytes that make up the UTF-8 signature when encountered in the ISO 8859-15 encoding.
+the-input-byte-stream-034
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs meta charset
+ + + + + +The HTTP header has a higher precedence than an encoding declaration in a meta charset attribute.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-1.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ
. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-018
Result summary & related tests
Detailed results for this test
Link to spec
HTTP vs meta content
+ + + + + +The HTTP header has a higher precedence than an encoding declaration in a meta content attribute.
+The HTTP header attempts to set the character encoding to ISO 8859-15. The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-1.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ
. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-016
Result summary & related tests
Detailed results for this test
Link to spec
No encoding declaration
+ + + + + +A page with no encoding information in HTTP, BOM, XML declaration or meta element will be treated as UTF-8.
+The test on this page contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè
. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-015
Result summary & related tests
Detailed results for this test
Link to spec
UTF-8 BOM vs meta charset
+ + + + + +A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta charset attribute declares a different encoding.
+The page contains an encoding declaration in a meta charset attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè
. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-038
Result summary & related tests
Detailed results for this test
Link to spec
UTF-8 BOM vs meta content
+ + + + + +A page with a UTF-8 BOM will be recognized as UTF-8 even if the meta content attribute declares a different encoding.
+The page contains an encoding declaration in a meta content attribute that attempts to set the character encoding to ISO 8859-15, but the file starts with a UTF-8 signature.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ýäè
. This matches the sequence of bytes above when they are interpreted as UTF-8. If the class name matches the selector then the test will pass.
the-input-byte-stream-037
Result summary & related tests
Detailed results for this test
Link to spec
meta charset attribute
+ + + + + +The character encoding of the page can be set by a meta element with charset attribute.
+The only character encoding declaration for this HTML file is in the charset attribute of the meta element, which declares the encoding to be ISO 8859-15.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ
. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-009
Result summary & related tests
Detailed results for this test
Link to spec
meta content attribute
+ + + + + +The character encoding of the page can be set by a meta element with http-equiv and content attributes.
+The only character encoding declaration for this HTML file is in the content attribute of the meta element, which declares the encoding to be ISO 8859-15.
The test contains a div with a class name that contains the following sequence of bytes: 0xC3 0xBD 0xC3 0xA4 0xC3 0xA8. These represent different sequences of characters in ISO 8859-15, ISO 8859-1 and UTF-8. The external, UTF-8-encoded stylesheet contains a selector .test div.ÜÀÚ
. This matches the sequence of bytes above when they are interpreted as ISO 8859-15. If the class name matches the selector then the test will pass.
the-input-byte-stream-007
Result summary & related tests
Detailed results for this test
Link to spec
Links:
` + doc, err := html.Parse(strings.NewReader(s)) + if err != nil { + log.Fatal(err) + } + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for _, a := range n.Attr { + if a.Key == "href" { + fmt.Println(a.Val) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + // Output: + // foo + // /bar/baz +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/foreign.go b/Godeps/_workspace/src/golang.org/x/net/html/foreign.go new file mode 100644 index 0000000..d3b3844 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/foreign.go @@ -0,0 +1,226 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "strings" +) + +func adjustAttributeNames(aa []Attribute, nameMap map[string]string) { + for i := range aa { + if newName, ok := nameMap[aa[i].Key]; ok { + aa[i].Key = newName + } + } +} + +func adjustForeignAttributes(aa []Attribute) { + for i, a := range aa { + if a.Key == "" || a.Key[0] != 'x' { + continue + } + switch a.Key { + case "xlink:actuate", "xlink:arcrole", "xlink:href", "xlink:role", "xlink:show", + "xlink:title", "xlink:type", "xml:base", "xml:lang", "xml:space", "xmlns:xlink": + j := strings.Index(a.Key, ":") + aa[i].Namespace = a.Key[:j] + aa[i].Key = a.Key[j+1:] + } + } +} + +func htmlIntegrationPoint(n *Node) bool { + if n.Type != ElementNode { + return false + } + switch n.Namespace { + case "math": + if n.Data == "annotation-xml" { + for _, a := range n.Attr { + if a.Key == "encoding" { + val := strings.ToLower(a.Val) + if val == "text/html" || val == "application/xhtml+xml" { + return true + } + } + } + } + case "svg": + switch n.Data { + case "desc", "foreignObject", "title": + return true + } + } + return false +} + +func mathMLTextIntegrationPoint(n *Node) bool { + if n.Namespace != "math" { + return false + } + switch n.Data { + case "mi", "mo", "mn", "ms", "mtext": + return true + } + return false +} + +// Section 12.2.5.5. +var breakout = map[string]bool{ + "b": true, + "big": true, + "blockquote": true, + "body": true, + "br": true, + "center": true, + "code": true, + "dd": true, + "div": true, + "dl": true, + "dt": true, + "em": true, + "embed": true, + "h1": true, + "h2": true, + "h3": true, + "h4": true, + "h5": true, + "h6": true, + "head": true, + "hr": true, + "i": true, + "img": true, + "li": true, + "listing": true, + "menu": true, + "meta": true, + "nobr": true, + "ol": true, + "p": true, + "pre": true, + "ruby": true, + "s": true, + "small": true, + "span": true, + "strong": true, + "strike": true, + "sub": true, + "sup": true, + "table": true, + "tt": true, + "u": true, + "ul": true, + "var": true, +} + +// Section 12.2.5.5. +var svgTagNameAdjustments = map[string]string{ + "altglyph": "altGlyph", + "altglyphdef": "altGlyphDef", + "altglyphitem": "altGlyphItem", + "animatecolor": "animateColor", + "animatemotion": "animateMotion", + "animatetransform": "animateTransform", + "clippath": "clipPath", + "feblend": "feBlend", + "fecolormatrix": "feColorMatrix", + "fecomponenttransfer": "feComponentTransfer", + "fecomposite": "feComposite", + "feconvolvematrix": "feConvolveMatrix", + "fediffuselighting": "feDiffuseLighting", + "fedisplacementmap": "feDisplacementMap", + "fedistantlight": "feDistantLight", + "feflood": "feFlood", + "fefunca": "feFuncA", + "fefuncb": "feFuncB", + "fefuncg": "feFuncG", + "fefuncr": "feFuncR", + "fegaussianblur": "feGaussianBlur", + "feimage": "feImage", + "femerge": "feMerge", + "femergenode": "feMergeNode", + "femorphology": "feMorphology", + "feoffset": "feOffset", + "fepointlight": "fePointLight", + "fespecularlighting": "feSpecularLighting", + "fespotlight": "feSpotLight", + "fetile": "feTile", + "feturbulence": "feTurbulence", + "foreignobject": "foreignObject", + "glyphref": "glyphRef", + "lineargradient": "linearGradient", + "radialgradient": "radialGradient", + "textpath": "textPath", +} + +// Section 12.2.5.1 +var mathMLAttributeAdjustments = map[string]string{ + "definitionurl": "definitionURL", +} + +var svgAttributeAdjustments = map[string]string{ + "attributename": "attributeName", + "attributetype": "attributeType", + "basefrequency": "baseFrequency", + "baseprofile": "baseProfile", + "calcmode": "calcMode", + "clippathunits": "clipPathUnits", + "contentscripttype": "contentScriptType", + "contentstyletype": "contentStyleType", + "diffuseconstant": "diffuseConstant", + "edgemode": "edgeMode", + "externalresourcesrequired": "externalResourcesRequired", + "filterres": "filterRes", + "filterunits": "filterUnits", + "glyphref": "glyphRef", + "gradienttransform": "gradientTransform", + "gradientunits": "gradientUnits", + "kernelmatrix": "kernelMatrix", + "kernelunitlength": "kernelUnitLength", + "keypoints": "keyPoints", + "keysplines": "keySplines", + "keytimes": "keyTimes", + "lengthadjust": "lengthAdjust", + "limitingconeangle": "limitingConeAngle", + "markerheight": "markerHeight", + "markerunits": "markerUnits", + "markerwidth": "markerWidth", + "maskcontentunits": "maskContentUnits", + "maskunits": "maskUnits", + "numoctaves": "numOctaves", + "pathlength": "pathLength", + "patterncontentunits": "patternContentUnits", + "patterntransform": "patternTransform", + "patternunits": "patternUnits", + "pointsatx": "pointsAtX", + "pointsaty": "pointsAtY", + "pointsatz": "pointsAtZ", + "preservealpha": "preserveAlpha", + "preserveaspectratio": "preserveAspectRatio", + "primitiveunits": "primitiveUnits", + "refx": "refX", + "refy": "refY", + "repeatcount": "repeatCount", + "repeatdur": "repeatDur", + "requiredextensions": "requiredExtensions", + "requiredfeatures": "requiredFeatures", + "specularconstant": "specularConstant", + "specularexponent": "specularExponent", + "spreadmethod": "spreadMethod", + "startoffset": "startOffset", + "stddeviation": "stdDeviation", + "stitchtiles": "stitchTiles", + "surfacescale": "surfaceScale", + "systemlanguage": "systemLanguage", + "tablevalues": "tableValues", + "targetx": "targetX", + "targety": "targetY", + "textlength": "textLength", + "viewbox": "viewBox", + "viewtarget": "viewTarget", + "xchannelselector": "xChannelSelector", + "ychannelselector": "yChannelSelector", + "zoomandpan": "zoomAndPan", +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/node.go b/Godeps/_workspace/src/golang.org/x/net/html/node.go new file mode 100644 index 0000000..6d2eec5 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/node.go @@ -0,0 +1,193 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "github.com/appc/acpush/Godeps/_workspace/src/golang.org/x/net/html/atom" +) + +// A NodeType is the type of a Node. +type NodeType uint32 + +const ( + ErrorNode NodeType = iota + TextNode + DocumentNode + ElementNode + CommentNode + DoctypeNode + scopeMarkerNode +) + +// Section 12.2.3.3 says "scope markers are inserted when entering applet +// elements, buttons, object elements, marquees, table cells, and table +// captions, and are used to prevent formatting from 'leaking'". +var scopeMarker = Node{Type: scopeMarkerNode} + +// A Node consists of a NodeType and some Data (tag name for element nodes, +// content for text) and are part of a tree of Nodes. Element nodes may also +// have a Namespace and contain a slice of Attributes. Data is unescaped, so +// that it looks like "a 0 { + return (*s)[i-1] + } + return nil +} + +// index returns the index of the top-most occurrence of n in the stack, or -1 +// if n is not present. +func (s *nodeStack) index(n *Node) int { + for i := len(*s) - 1; i >= 0; i-- { + if (*s)[i] == n { + return i + } + } + return -1 +} + +// insert inserts a node at the given index. +func (s *nodeStack) insert(i int, n *Node) { + (*s) = append(*s, nil) + copy((*s)[i+1:], (*s)[i:]) + (*s)[i] = n +} + +// remove removes a node from the stack. It is a no-op if n is not present. +func (s *nodeStack) remove(n *Node) { + i := s.index(n) + if i == -1 { + return + } + copy((*s)[i:], (*s)[i+1:]) + j := len(*s) - 1 + (*s)[j] = nil + *s = (*s)[:j] +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/node_test.go b/Godeps/_workspace/src/golang.org/x/net/html/node_test.go new file mode 100644 index 0000000..471102f --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/node_test.go @@ -0,0 +1,146 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "fmt" +) + +// checkTreeConsistency checks that a node and its descendants are all +// consistent in their parent/child/sibling relationships. +func checkTreeConsistency(n *Node) error { + return checkTreeConsistency1(n, 0) +} + +func checkTreeConsistency1(n *Node, depth int) error { + if depth == 1e4 { + return fmt.Errorf("html: tree looks like it contains a cycle") + } + if err := checkNodeConsistency(n); err != nil { + return err + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if err := checkTreeConsistency1(c, depth+1); err != nil { + return err + } + } + return nil +} + +// checkNodeConsistency checks that a node's parent/child/sibling relationships +// are consistent. +func checkNodeConsistency(n *Node) error { + if n == nil { + return nil + } + + nParent := 0 + for p := n.Parent; p != nil; p = p.Parent { + nParent++ + if nParent == 1e4 { + return fmt.Errorf("html: parent list looks like an infinite loop") + } + } + + nForward := 0 + for c := n.FirstChild; c != nil; c = c.NextSibling { + nForward++ + if nForward == 1e6 { + return fmt.Errorf("html: forward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + nBackward := 0 + for c := n.LastChild; c != nil; c = c.PrevSibling { + nBackward++ + if nBackward == 1e6 { + return fmt.Errorf("html: backward list of children looks like an infinite loop") + } + if c.Parent != n { + return fmt.Errorf("html: inconsistent child/parent relationship") + } + } + + if n.Parent != nil { + if n.Parent == n { + return fmt.Errorf("html: inconsistent parent relationship") + } + if n.Parent == n.FirstChild { + return fmt.Errorf("html: inconsistent parent/first relationship") + } + if n.Parent == n.LastChild { + return fmt.Errorf("html: inconsistent parent/last relationship") + } + if n.Parent == n.PrevSibling { + return fmt.Errorf("html: inconsistent parent/prev relationship") + } + if n.Parent == n.NextSibling { + return fmt.Errorf("html: inconsistent parent/next relationship") + } + + parentHasNAsAChild := false + for c := n.Parent.FirstChild; c != nil; c = c.NextSibling { + if c == n { + parentHasNAsAChild = true + break + } + } + if !parentHasNAsAChild { + return fmt.Errorf("html: inconsistent parent/child relationship") + } + } + + if n.PrevSibling != nil && n.PrevSibling.NextSibling != n { + return fmt.Errorf("html: inconsistent prev/next relationship") + } + if n.NextSibling != nil && n.NextSibling.PrevSibling != n { + return fmt.Errorf("html: inconsistent next/prev relationship") + } + + if (n.FirstChild == nil) != (n.LastChild == nil) { + return fmt.Errorf("html: inconsistent first/last relationship") + } + if n.FirstChild != nil && n.FirstChild == n.LastChild { + // We have a sole child. + if n.FirstChild.PrevSibling != nil || n.FirstChild.NextSibling != nil { + return fmt.Errorf("html: inconsistent sole child's sibling relationship") + } + } + + seen := map[*Node]bool{} + + var last *Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if seen[c] { + return fmt.Errorf("html: inconsistent repeated child") + } + seen[c] = true + last = c + } + if last != n.LastChild { + return fmt.Errorf("html: inconsistent last relationship") + } + + var first *Node + for c := n.LastChild; c != nil; c = c.PrevSibling { + if !seen[c] { + return fmt.Errorf("html: inconsistent missing child") + } + delete(seen, c) + first = c + } + if first != n.FirstChild { + return fmt.Errorf("html: inconsistent first relationship") + } + + if len(seen) != 0 { + return fmt.Errorf("html: inconsistent forwards/backwards child list") + } + + return nil +} diff --git a/Godeps/_workspace/src/golang.org/x/net/html/parse.go b/Godeps/_workspace/src/golang.org/x/net/html/parse.go new file mode 100644 index 0000000..5555df3 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/html/parse.go @@ -0,0 +1,2094 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package html + +import ( + "errors" + "fmt" + "io" + "strings" + + a "github.com/appc/acpush/Godeps/_workspace/src/golang.org/x/net/html/atom" +) + +// A parser implements the HTML5 parsing algorithm: +// https://html.spec.whatwg.org/multipage/syntax.html#tree-construction +type parser struct { + // tokenizer provides the tokens for the parser. + tokenizer *Tokenizer + // tok is the most recently read token. + tok Token + // Self-closing tags likeblock. + if d != "" && d[0] == '\r' { + d = d[1:] + } + if d != "" && d[0] == '\n' { + d = d[1:] + } + } + } + d = strings.Replace(d, "\x00", "", -1) + if d == "" { + return true + } + p.reconstructActiveFormattingElements() + p.addText(d) + if p.framesetOK && strings.TrimLeft(d, whitespace) != "" { + // There were non-whitespace characters inserted. + p.framesetOK = false + } + case StartTagToken: + switch p.tok.DataAtom { + case a.Html: + copyAttributes(p.oe[0], p.tok) + case a.Base, a.Basefont, a.Bgsound, a.Command, a.Link, a.Meta, a.Noframes, a.Script, a.Style, a.Title: + return inHeadIM(p) + case a.Body: + if len(p.oe) >= 2 { + body := p.oe[1] + if body.Type == ElementNode && body.DataAtom == a.Body { + p.framesetOK = false + copyAttributes(body, p.tok) + } + } + case a.Frameset: + if !p.framesetOK || len(p.oe) < 2 || p.oe[1].DataAtom != a.Body { + // Ignore the token. + return true + } + body := p.oe[1] + if body.Parent != nil { + body.Parent.RemoveChild(body) + } + p.oe = p.oe[:1] + p.addElement() + p.im = inFramesetIM + return true + case a.Address, a.Article, a.Aside, a.Blockquote, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Menu, a.Nav, a.Ol, a.P, a.Section, a.Summary, a.Ul: + p.popUntil(buttonScope, a.P) + p.addElement() + case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6: + p.popUntil(buttonScope, a.P) + switch n := p.top(); n.DataAtom { + case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6: + p.oe.pop() + } + p.addElement() + case a.Pre, a.Listing: + p.popUntil(buttonScope, a.P) + p.addElement() + // The newline, if any, will be dealt with by the TextToken case. + p.framesetOK = false + case a.Form: + if p.form == nil { + p.popUntil(buttonScope, a.P) + p.addElement() + p.form = p.top() + } + case a.Li: + p.framesetOK = false + for i := len(p.oe) - 1; i >= 0; i-- { + node := p.oe[i] + switch node.DataAtom { + case a.Li: + p.oe = p.oe[:i] + case a.Address, a.Div, a.P: + continue + default: + if !isSpecialElement(node) { + continue + } + } + break + } + p.popUntil(buttonScope, a.P) + p.addElement() + case a.Dd, a.Dt: + p.framesetOK = false + for i := len(p.oe) - 1; i >= 0; i-- { + node := p.oe[i] + switch node.DataAtom { + case a.Dd, a.Dt: + p.oe = p.oe[:i] + case a.Address, a.Div, a.P: + continue + default: + if !isSpecialElement(node) { + continue + } + } + break + } + p.popUntil(buttonScope, a.P) + p.addElement() + case a.Plaintext: + p.popUntil(buttonScope, a.P) + p.addElement() + case a.Button: + p.popUntil(defaultScope, a.Button) + p.reconstructActiveFormattingElements() + p.addElement() + p.framesetOK = false + case a.A: + for i := len(p.afe) - 1; i >= 0 && p.afe[i].Type != scopeMarkerNode; i-- { + if n := p.afe[i]; n.Type == ElementNode && n.DataAtom == a.A { + p.inBodyEndTagFormatting(a.A) + p.oe.remove(n) + p.afe.remove(n) + break + } + } + p.reconstructActiveFormattingElements() + p.addFormattingElement() + case a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U: + p.reconstructActiveFormattingElements() + p.addFormattingElement() + case a.Nobr: + p.reconstructActiveFormattingElements() + if p.elementInScope(defaultScope, a.Nobr) { + p.inBodyEndTagFormatting(a.Nobr) + p.reconstructActiveFormattingElements() + } + p.addFormattingElement() + case a.Applet, a.Marquee, a.Object: + p.reconstructActiveFormattingElements() + p.addElement() + p.afe = append(p.afe, &scopeMarker) + p.framesetOK = false + case a.Table: + if !p.quirks { + p.popUntil(buttonScope, a.P) + } + p.addElement() + p.framesetOK = false + p.im = inTableIM + return true + case a.Area, a.Br, a.Embed, a.Img, a.Input, a.Keygen, a.Wbr: + p.reconstructActiveFormattingElements() + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + if p.tok.DataAtom == a.Input { + for _, t := range p.tok.Attr { + if t.Key == "type" { + if strings.ToLower(t.Val) == "hidden" { + // Skip setting framesetOK = false + return true + } + } + } + } + p.framesetOK = false + case a.Param, a.Source, a.Track: + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + case a.Hr: + p.popUntil(buttonScope, a.P) + p.addElement() + p.oe.pop() + p.acknowledgeSelfClosingTag() + p.framesetOK = false + case a.Image: + p.tok.DataAtom = a.Img + p.tok.Data = a.Img.String() + return false + case a.Isindex: + if p.form != nil { + // Ignore the token. + return true + } + action := "" + prompt := "This is a searchable index. Enter search keywords: " + attr := []Attribute{{Key: "name", Val: "isindex"}} + for _, t := range p.tok.Attr { + switch t.Key { + case "action": + action = t.Val + case "name": + // Ignore the attribute. + case "prompt": + prompt = t.Val + default: + attr = append(attr, t) + } + } + p.acknowledgeSelfClosingTag() + p.popUntil(buttonScope, a.P) + p.parseImpliedToken(StartTagToken, a.Form, a.Form.String()) + if action != "" { + p.form.Attr = []Attribute{{Key: "action", Val: action}} + } + p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String()) + p.parseImpliedToken(StartTagToken, a.Label, a.Label.String()) + p.addText(prompt) + p.addChild(&Node{ + Type: ElementNode, + DataAtom: a.Input, + Data: a.Input.String(), + Attr: attr, + }) + p.oe.pop() + p.parseImpliedToken(EndTagToken, a.Label, a.Label.String()) + p.parseImpliedToken(StartTagToken, a.Hr, a.Hr.String()) + p.parseImpliedToken(EndTagToken, a.Form, a.Form.String()) + case a.Textarea: + p.addElement() + p.setOriginalIM() + p.framesetOK = false + p.im = textIM + case a.Xmp: + p.popUntil(buttonScope, a.P) + p.reconstructActiveFormattingElements() + p.framesetOK = false + p.addElement() + p.setOriginalIM() + p.im = textIM + case a.Iframe: + p.framesetOK = false + p.addElement() + p.setOriginalIM() + p.im = textIM + case a.Noembed, a.Noscript: + p.addElement() + p.setOriginalIM() + p.im = textIM + case a.Select: + p.reconstructActiveFormattingElements() + p.addElement() + p.framesetOK = false + p.im = inSelectIM + return true + case a.Optgroup, a.Option: + if p.top().DataAtom == a.Option { + p.oe.pop() + } + p.reconstructActiveFormattingElements() + p.addElement() + case a.Rp, a.Rt: + if p.elementInScope(defaultScope, a.Ruby) { + p.generateImpliedEndTags() + } + p.addElement() + case a.Math, a.Svg: + p.reconstructActiveFormattingElements() + if p.tok.DataAtom == a.Math { + adjustAttributeNames(p.tok.Attr, mathMLAttributeAdjustments) + } else { + adjustAttributeNames(p.tok.Attr, svgAttributeAdjustments) + } + adjustForeignAttributes(p.tok.Attr) + p.addElement() + p.top().Namespace = p.tok.Data + if p.hasSelfClosingToken { + p.oe.pop() + p.acknowledgeSelfClosingTag() + } + return true + case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: + // Ignore the token. + default: + p.reconstructActiveFormattingElements() + p.addElement() + } + case EndTagToken: + switch p.tok.DataAtom { + case a.Body: + if p.elementInScope(defaultScope, a.Body) { + p.im = afterBodyIM + } + case a.Html: + if p.elementInScope(defaultScope, a.Body) { + p.parseImpliedToken(EndTagToken, a.Body, a.Body.String()) + return false + } + return true + case a.Address, a.Article, a.Aside, a.Blockquote, a.Button, a.Center, a.Details, a.Dir, a.Div, a.Dl, a.Fieldset, a.Figcaption, a.Figure, a.Footer, a.Header, a.Hgroup, a.Listing, a.Menu, a.Nav, a.Ol, a.Pre, a.Section, a.Summary, a.Ul: + p.popUntil(defaultScope, p.tok.DataAtom) + case a.Form: + node := p.form + p.form = nil + i := p.indexOfElementInScope(defaultScope, a.Form) + if node == nil || i == -1 || p.oe[i] != node { + // Ignore the token. + return true + } + p.generateImpliedEndTags() + p.oe.remove(node) + case a.P: + if !p.elementInScope(buttonScope, a.P) { + p.parseImpliedToken(StartTagToken, a.P, a.P.String()) + } + p.popUntil(buttonScope, a.P) + case a.Li: + p.popUntil(listItemScope, a.Li) + case a.Dd, a.Dt: + p.popUntil(defaultScope, p.tok.DataAtom) + case a.H1, a.H2, a.H3, a.H4, a.H5, a.H6: + p.popUntil(defaultScope, a.H1, a.H2, a.H3, a.H4, a.H5, a.H6) + case a.A, a.B, a.Big, a.Code, a.Em, a.Font, a.I, a.Nobr, a.S, a.Small, a.Strike, a.Strong, a.Tt, a.U: + p.inBodyEndTagFormatting(p.tok.DataAtom) + case a.Applet, a.Marquee, a.Object: + if p.popUntil(defaultScope, p.tok.DataAtom) { + p.clearActiveFormattingElements() + } + case a.Br: + p.tok.Type = StartTagToken + return false + default: + p.inBodyEndTagOther(p.tok.DataAtom) + } + case CommentToken: + p.addChild(&Node{ + Type: CommentNode, + Data: p.tok.Data, + }) + } + + return true +} + +func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) { + // This is the "adoption agency" algorithm, described at + // https://html.spec.whatwg.org/multipage/syntax.html#adoptionAgency + + // TODO: this is a fairly literal line-by-line translation of that algorithm. + // Once the code successfully parses the comprehensive test suite, we should + // refactor this code to be more idiomatic. + + // Steps 1-4. The outer loop. + for i := 0; i < 8; i++ { + // Step 5. Find the formatting element. + var formattingElement *Node + for j := len(p.afe) - 1; j >= 0; j-- { + if p.afe[j].Type == scopeMarkerNode { + break + } + if p.afe[j].DataAtom == tagAtom { + formattingElement = p.afe[j] + break + } + } + if formattingElement == nil { + p.inBodyEndTagOther(tagAtom) + return + } + feIndex := p.oe.index(formattingElement) + if feIndex == -1 { + p.afe.remove(formattingElement) + return + } + if !p.elementInScope(defaultScope, tagAtom) { + // Ignore the tag. + return + } + + // Steps 9-10. Find the furthest block. + var furthestBlock *Node + for _, e := range p.oe[feIndex:] { + if isSpecialElement(e) { + furthestBlock = e + break + } + } + if furthestBlock == nil { + e := p.oe.pop() + for e != formattingElement { + e = p.oe.pop() + } + p.afe.remove(e) + return + } + + // Steps 11-12. Find the common ancestor and bookmark node. + commonAncestor := p.oe[feIndex-1] + bookmark := p.afe.index(formattingElement) + + // Step 13. The inner loop. Find the lastNode to reparent. + lastNode := furthestBlock + node := furthestBlock + x := p.oe.index(node) + // Steps 13.1-13.2 + for j := 0; j < 3; j++ { + // Step 13.3. + x-- + node = p.oe[x] + // Step 13.4 - 13.5. + if p.afe.index(node) == -1 { + p.oe.remove(node) + continue + } + // Step 13.6. + if node == formattingElement { + break + } + // Step 13.7. + clone := node.clone() + p.afe[p.afe.index(node)] = clone + p.oe[p.oe.index(node)] = clone + node = clone + // Step 13.8. + if lastNode == furthestBlock { + bookmark = p.afe.index(node) + 1 + } + // Step 13.9. + if lastNode.Parent != nil { + lastNode.Parent.RemoveChild(lastNode) + } + node.AppendChild(lastNode) + // Step 13.10. + lastNode = node + } + + // Step 14. Reparent lastNode to the common ancestor, + // or for misnested table nodes, to the foster parent. + if lastNode.Parent != nil { + lastNode.Parent.RemoveChild(lastNode) + } + switch commonAncestor.DataAtom { + case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: + p.fosterParent(lastNode) + default: + commonAncestor.AppendChild(lastNode) + } + + // Steps 15-17. Reparent nodes from the furthest block's children + // to a clone of the formatting element. + clone := formattingElement.clone() + reparentChildren(clone, furthestBlock) + furthestBlock.AppendChild(clone) + + // Step 18. Fix up the list of active formatting elements. + if oldLoc := p.afe.index(formattingElement); oldLoc != -1 && oldLoc < bookmark { + // Move the bookmark with the rest of the list. + bookmark-- + } + p.afe.remove(formattingElement) + p.afe.insert(bookmark, clone) + + // Step 19. Fix up the stack of open elements. + p.oe.remove(formattingElement) + p.oe.insert(p.oe.index(furthestBlock)+1, clone) + } +} + +// inBodyEndTagOther performs the "any other end tag" algorithm for inBodyIM. +// "Any other end tag" handling from 12.2.5.5 The rules for parsing tokens in foreign content +// https://html.spec.whatwg.org/multipage/syntax.html#parsing-main-inforeign +func (p *parser) inBodyEndTagOther(tagAtom a.Atom) { + for i := len(p.oe) - 1; i >= 0; i-- { + if p.oe[i].DataAtom == tagAtom { + p.oe = p.oe[:i] + break + } + if isSpecialElement(p.oe[i]) { + break + } + } +} + +// Section 12.2.5.4.8. +func textIM(p *parser) bool { + switch p.tok.Type { + case ErrorToken: + p.oe.pop() + case TextToken: + d := p.tok.Data + if n := p.oe.top(); n.DataAtom == a.Textarea && n.FirstChild == nil { + // Ignore a newline at the start of a