From 94665892c08441e10cebe97f091cfab44fe1ed58 Mon Sep 17 00:00:00 2001 From: Alexey Miroshkin Date: Fri, 1 Dec 2017 23:54:30 +0100 Subject: [PATCH] Initial version of Clair API V3 support --- clair/api.go | 198 ++++++++++++++++++++++++++++++++++++++++++++ clair/clair.go | 118 +++++--------------------- clair/clair_test.go | 7 +- main.go | 35 +++++--- 4 files changed, 248 insertions(+), 110 deletions(-) create mode 100644 clair/api.go diff --git a/clair/api.go b/clair/api.go new file mode 100644 index 0000000..94791b0 --- /dev/null +++ b/clair/api.go @@ -0,0 +1,198 @@ +package clair + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/coreos/clair/api/v3/clairpb" + "github.com/optiopay/klar/docker" + "github.com/optiopay/klar/utils" + "google.golang.org/grpc" +) + +type apiV1 struct { + url string + client http.Client +} + +type apiV3 struct { + client clairpb.AncestryServiceClient +} + +func newAPI(url string, version int) (API, error) { + if version < 3 { + return newAPIV1(url), nil + } + return newAPIV3(url) +} + +func newAPIV1(url string) *apiV1 { + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = fmt.Sprintf("http://%s", url) + } + if strings.LastIndex(url, ":") < 6 { + url = fmt.Sprintf("%s:6060", url) + } + return &apiV1{ + url: url, + client: http.Client{ + Timeout: time.Minute, + }, + } +} + +func newAPIV3(url string) (*apiV3, error) { + if i := strings.Index(url, "://"); i != -1 { + runes := []rune(url) + url = string(runes[i+3:]) + } + fmt.Printf("URL: %q\n", url) + conn, err := grpc.Dial(url, grpc.WithInsecure()) + if err != nil { + return nil, fmt.Errorf("did not connect to %s: %v", url, err) + } + fmt.Println("Created GRPC client") + return &apiV3{clairpb.NewAncestryServiceClient(conn)}, nil +} + +func (a *apiV1) Push(image *docker.Image) error { + for i := 0; i < len(image.FsLayers); i++ { + layer := newLayer(image, i) + if err := a.pushLayer(layer); err != nil { + return err + } + } + return nil +} + +func (a *apiV1) pushLayer(layer *layer) error { + envelope := layerEnvelope{Layer: layer} + reqBody, err := json.Marshal(envelope) + if err != nil { + return fmt.Errorf("can't serialze push request: %s", err) + } + url := fmt.Sprintf("%s/v1/layers", a.url) + request, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody)) + if err != nil { + return fmt.Errorf("can't create a push request: %s", err) + } + request.Header.Set("Content-Type", "application/json") + utils.DumpRequest(request) + response, err := a.client.Do(request) + if err != nil { + return fmt.Errorf("can't push layer to Clair: %s", err) + } + utils.DumpResponse(response) + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return fmt.Errorf("can't read clair response : %s", err) + } + if response.StatusCode != http.StatusCreated { + var lerr layerError + err = json.Unmarshal(body, &lerr) + if err != nil { + return fmt.Errorf("can't even read an error message: %s", err) + } + return fmt.Errorf("push error %d: %s", response.StatusCode, string(body)) + } + return nil +} + +func (a *apiV1) Analyze(image *docker.Image) ([]*Vulnerability, error) { + url := fmt.Sprintf("%s/v1/layers/%s?vulnerabilities", a.url, image.AnalyzedLayerName()) + request, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("can't create an analyze request: %s", err) + } + utils.DumpRequest(request) + response, err := a.client.Do(request) + if err != nil { + return nil, err + } + utils.DumpResponse(response) + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + body, _ := ioutil.ReadAll(response.Body) + return nil, fmt.Errorf("analyze error %d: %s", response.StatusCode, string(body)) + } + var envelope layerEnvelope + if err = json.NewDecoder(response.Body).Decode(&envelope); err != nil { + return nil, err + } + var vs []*Vulnerability + for _, f := range envelope.Layer.Features { + for _, v := range f.Vulnerabilities { + v.FeatureName = f.Name + vs = append(vs, &v) + } + } + return vs, nil +} + +func (a *apiV3) Push(image *docker.Image) error { + fmt.Println("grpc push image") + req := &clairpb.PostAncestryRequest{ + Format: "Docker", + AncestryName: image.Name, + } + + ls := make([]*clairpb.PostAncestryRequest_PostLayer, len(image.FsLayers)) + for i := 0; i < len(image.FsLayers); i++ { + ls[i] = newLayerV3(image, i) + } + req.Layers = ls + resp, err := a.client.PostAncestry(context.Background(), req) + if err != nil { + return err + } + fmt.Printf("grpc resp: %v", resp) + return nil +} + +func newLayerV3(image *docker.Image, index int) *clairpb.PostAncestryRequest_PostLayer { + return &clairpb.PostAncestryRequest_PostLayer{ + Hash: image.LayerName(index), + Path: strings.Join([]string{image.Registry, image.Name, "blobs", image.FsLayers[index].BlobSum}, "/"), + Headers: map[string]string{"Authorization": image.Token}, + } +} + +func (a *apiV3) Analyze(image *docker.Image) ([]*Vulnerability, error) { + req := &clairpb.GetAncestryRequest{ + AncestryName: image.Name, + WithFeatures: true, + WithVulnerabilities: true, + } + + resp, err := a.client.GetAncestry(context.Background(), req) + if err != nil { + return nil, err + } + var vs []*Vulnerability + for _, f := range resp.Ancestry.Features { + for _, v := range f.Vulnerabilities { + cv := convertVulnerability(v) + cv.FeatureName = f.Name + vs = append(vs, cv) + } + } + return vs, nil +} + +func convertVulnerability(cv *clairpb.Vulnerability) *Vulnerability { + return &Vulnerability{ + Name: cv.Name, + NamespaceName: cv.NamespaceName, + Description: cv.Description, + Severity: cv.Severity, + Link: cv.Link, + FixedBy: cv.FixedBy, + } +} diff --git a/clair/clair.go b/clair/clair.go index 7b7d793..467f8a4 100644 --- a/clair/clair.go +++ b/clair/clair.go @@ -1,25 +1,24 @@ package clair import ( - "bytes" - "encoding/json" "fmt" - "io/ioutil" - "net/http" "os" "strings" - "time" "github.com/optiopay/klar/docker" - "github.com/optiopay/klar/utils" ) const EMPTY_LAYER_BLOB_SUM = "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4" // Clair is representation of Clair server type Clair struct { - url string - client http.Client + url string + api API +} + +type API interface { + Analyze(image *docker.Image) ([]*Vulnerability, error) + Push(image *docker.Image) error } type layer struct { @@ -53,6 +52,7 @@ type Vulnerability struct { Metadata map[string]interface{} `json:"Metadata,omitempty"` FixedBy string `json:"FixedBy,omitempty"` FixedIn []feature `json:"FixedIn,omitempty"` + FeatureName string `json:"featureName",omitempty` } type layerError struct { @@ -70,18 +70,13 @@ type layerEnvelope struct { // NewClair construct Clair entity using potentially incomplete server URL // If protocol is missing HTTP will be used. If port is missing 6060 will be used -func NewClair(url string) Clair { - if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { - url = fmt.Sprintf("http://%s", url) - } - if strings.LastIndex(url, ":") < 5 { - url = fmt.Sprintf("%s:6060", url) - } - client := http.Client{ - Timeout: time.Minute, +func NewClair(url string, version int) Clair { + api, err := newAPI(url, version) + if err != nil { + panic(fmt.Sprintf("cant't create API client version %d %s: %s", version, url, err)) } - return Clair{url, client} + return Clair{url, api} } func newLayer(image *docker.Image, index int) *layer { @@ -110,96 +105,23 @@ func filterEmptyLayers(fsLayers []docker.FsLayer) (filteredLayers []docker.FsLay // Analyse sent each layer from Docker image to Clair and returns // a list of found vulnerabilities -func (c *Clair) Analyse(image *docker.Image) []Vulnerability { +func (c *Clair) Analyse(image *docker.Image) ([]*Vulnerability, error) { // Filter the empty layers in image image.FsLayers = filterEmptyLayers(image.FsLayers) layerLength := len(image.FsLayers) if layerLength == 0 { - fmt.Fprintf(os.Stderr, "No need to analyse image %s/%s:%s as there is no non-emtpy layer\n", + fmt.Fprintf(os.Stderr, "no need to analyse image %s/%s:%s as there is no non-emtpy layer\n", image.Registry, image.Name, image.Tag) - return nil + return nil, nil } - var vs []Vulnerability - for i := 0; i < layerLength; i++ { - layer := newLayer(image, i) - err := c.pushLayer(layer) - if err != nil { - fmt.Fprintf(os.Stderr, "Push layer %d failed: %s\n", i, err.Error()) - continue - } + if err := c.api.Push(image); err != nil { + return nil, fmt.Errorf("push image %s/%s:%s to Clair failed: %s\n", image.Registry, image.Name, image.Tag, err.Error()) } - - vs, err := c.analyzeLayer(image.AnalyzedLayerName()) + vs, err := c.api.Analyze(image) if err != nil { - fmt.Fprintf(os.Stderr, "Analyse image %s/%s:%s failed: %s\n", image.Registry, image.Name, image.Tag, err.Error()) - return nil + return nil, fmt.Errorf("analyse image %s/%s:%s failed: %s\n", image.Registry, image.Name, image.Tag, err.Error()) } - return vs -} - -func (c *Clair) analyzeLayer(layerName string) ([]Vulnerability, error) { - url := fmt.Sprintf("%s/v1/layers/%s?vulnerabilities", c.url, layerName) - request, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, fmt.Errorf("Can't create an analyze request: %s", err) - } - utils.DumpRequest(request) - response, err := c.client.Do(request) - if err != nil { - return nil, err - } - utils.DumpResponse(response) - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - body, _ := ioutil.ReadAll(response.Body) - return nil, fmt.Errorf("Analyze error %d: %s", response.StatusCode, string(body)) - } - var envelope layerEnvelope - if err = json.NewDecoder(response.Body).Decode(&envelope); err != nil { - return nil, err - } - var vs []Vulnerability - for _, f := range envelope.Layer.Features { - for _, v := range f.Vulnerabilities { - vs = append(vs, v) - } - } return vs, nil } - -func (c *Clair) pushLayer(layer *layer) error { - envelope := layerEnvelope{Layer: layer} - reqBody, err := json.Marshal(envelope) - if err != nil { - return fmt.Errorf("can't serialze push request: %s", err) - } - url := fmt.Sprintf("%s/v1/layers", c.url) - request, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody)) - if err != nil { - return fmt.Errorf("Can't create a push request: %s", err) - } - request.Header.Set("Content-Type", "application/json") - utils.DumpRequest(request) - response, err := c.client.Do(request) - if err != nil { - return fmt.Errorf("Can't push layer to Clair: %s", err) - } - utils.DumpResponse(response) - defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) - if err != nil { - return fmt.Errorf("Can't read clair response : %s", err) - } - if response.StatusCode != http.StatusCreated { - var lerr layerError - err = json.Unmarshal(body, &lerr) - if err != nil { - return fmt.Errorf("Can't even read an error message: %s", err) - } - return fmt.Errorf("Push error %d: %s", response.StatusCode, string(body)) - } - return nil - -} diff --git a/clair/clair_test.go b/clair/clair_test.go index 690392b..610f19c 100644 --- a/clair/clair_test.go +++ b/clair/clair_test.go @@ -83,8 +83,11 @@ func TestAnalyse(t *testing.T) { Token: imageToken, } - c := NewClair(ts.URL) - vs := c.Analyse(dockerImage) + c := NewClair(ts.URL, 1) + vs, err := c.Analyse(dockerImage) + if err != nil { + t.Fatal(err) + } if len(vs) != 1 { t.Fatalf("Expected 1 vulnerability, got %d", len(vs)) } diff --git a/main.go b/main.go index 7632fa4..6c2337f 100644 --- a/main.go +++ b/main.go @@ -14,11 +14,11 @@ import ( type jsonOutput struct { LayerCount int - Vulnerabilities map[string][]clair.Vulnerability + Vulnerabilities map[string][]*clair.Vulnerability } var priorities = []string{"Unknown", "Negligible", "Low", "Medium", "High", "Critical", "Defcon1"} -var store = make(map[string][]clair.Vulnerability) +var store = make(map[string][]*clair.Vulnerability) func main() { if len(os.Args) != 2 { @@ -28,6 +28,9 @@ func main() { if os.Getenv("KLAR_TRACE") != "" { utils.Trace = true + os.Setenv("GRPC_TRACE", "all") + os.Setenv("GRPC_VERBOSITY", "DEBUG") + os.Setenv("GODEBUG", "http2debug=2") } clairAddr := os.Getenv("CLAIR_ADDR") @@ -92,7 +95,7 @@ func main() { } output := jsonOutput{ - Vulnerabilities: make(map[string][]clair.Vulnerability), + Vulnerabilities: make(map[string][]*clair.Vulnerability), } if len(image.FsLayers) == 0 { @@ -106,8 +109,21 @@ func main() { } } - c := clair.NewClair(clairAddr) - vs := c.Analyse(image) + var vs []*clair.Vulnerability + for _, ver := range []int{1, 3} { + c := clair.NewClair(clairAddr, ver) + vs, err = c.Analyse(image) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to analyze using API V%d: %s", ver, err) + } else { + break + } + } + if err != nil { + fmt.Fprintln(os.Stderr, "failed to analyze, exiting") + os.Exit(2) + } + groupBySeverity(vs) vsNumber := 0 @@ -123,7 +139,7 @@ func main() { iteratePriorities(clairOutput, func(sev string) { vsNumber += len(store[sev]) for _, v := range store[sev] { - fmt.Printf("%s: [%s] \n%s\n%s\n", v.Name, v.Severity, v.Description, v.Link) + fmt.Printf("%s: [%s] \nFound in: %s\n%s\n%s\n", v.Name, v.Severity, v.FeatureName, v.Description, v.Link) fmt.Println("-----------------------------------------") } }) @@ -150,20 +166,19 @@ func iteratePriorities(output string, f func(sev string)) { f(sev) } } - } -func groupBySeverity(vs []clair.Vulnerability) { +func groupBySeverity(vs []*clair.Vulnerability) { for _, v := range vs { sevRow := vulnsBy(v.Severity, store) store[v.Severity] = append(sevRow, v) } } -func vulnsBy(sev string, store map[string][]clair.Vulnerability) []clair.Vulnerability { +func vulnsBy(sev string, store map[string][]*clair.Vulnerability) []*clair.Vulnerability { items, found := store[sev] if !found { - items = make([]clair.Vulnerability, 0) + items = make([]*clair.Vulnerability, 0) store[sev] = items } return items