From 22c27fab5f136bdfbcbc3fdf93897c1c9967e2fa Mon Sep 17 00:00:00 2001 From: Matt Oswalt Date: Sat, 2 Feb 2019 01:12:37 -0800 Subject: [PATCH 1/2] WIP work for advisor changes Signed-off-by: Matt Oswalt --- api/exp/definitions/lessondef.proto | 34 ++++++--- api/exp/definitions/lessondef.swagger.json | 83 ++++++++++++++++++---- api/exp/lessondefs.go | 75 ++++++++++++++++--- api/exp/server.go | 30 +++++++- api/exp/swagger/swagger.pb.go | 83 ++++++++++++++++++---- cmd/syringed/main.go | 23 +++--- 6 files changed, 270 insertions(+), 58 deletions(-) diff --git a/api/exp/definitions/lessondef.proto b/api/exp/definitions/lessondef.proto index 8c1236b5..c8e6b8e6 100644 --- a/api/exp/definitions/lessondef.proto +++ b/api/exp/definitions/lessondef.proto @@ -2,15 +2,21 @@ syntax = "proto3"; package syringe.api.exp; import "google/api/annotations.proto"; -import "google/protobuf/empty.proto"; +// import "google/protobuf/empty.proto"; import "validate/validate.proto"; service LessonDefService { // Retrieve all LessonDefs with filter - rpc ListLessonDefs(google.protobuf.Empty) returns (LessonCategoryMap) { + rpc ListLessonDefs(LessonDefFilter) returns (LessonDefs) { option (google.api.http) = { - get: "/exp/lessondef/all" + get: "/exp/lessondef" + }; + } + + rpc GetAllLessonPrereqs(LessonID) returns (LessonPrereqs) { + option (google.api.http) = { + get: "/exp/lessondef/{id}/prereqs" }; } @@ -22,14 +28,19 @@ service LessonDefService { } -message LessonCategoryMap { - map lessonCategories = 1; -} +// message LessonCategoryMap { +// map lessonCategories = 1; +// } + message LessonDefs { repeated LessonDef lessonDefs = 1; } +message LessonPrereqs { + repeated int32 prereqs = 1; +} + message LessonID { int32 id = 1; } @@ -43,14 +54,21 @@ message LessonDef { repeated Utility Utilities = 6; repeated Blackbox Blackboxes = 7; repeated Connection Connections = 8; - string Category = 9 [(validate.rules).string = {in: ["introductory", "troubleshooting", "verification", "configuration"]}]; + string Category = 9 [(validate.rules).string = {in: ["fundamentals", "tools", "workflows"]}]; string LessonDiagram = 10; string LessonVideo = 11; string Tier = 12 [(validate.rules).string = {in: ["local", "ptr", "prod"]}]; + repeated int32 Prereqs = 13; + repeated string Tags = 14; + string Collection = 15; + string Description = 16 [(validate.rules).string.min_len = 10]; + + // This is meant to fill: "How well do you know ?" + string Slug = 17 [(validate.rules).string.min_len = 1]; } message LessonDefFilter { - string Category = 1; + string Category = 2; } message LessonStage { diff --git a/api/exp/definitions/lessondef.swagger.json b/api/exp/definitions/lessondef.swagger.json index def88972..4adfe896 100644 --- a/api/exp/definitions/lessondef.swagger.json +++ b/api/exp/definitions/lessondef.swagger.json @@ -15,7 +15,7 @@ "application/json" ], "paths": { - "/exp/lessondef/all": { + "/exp/lessondef": { "get": { "summary": "Retrieve all LessonDefs with filter", "operationId": "ListLessonDefs", @@ -23,10 +23,18 @@ "200": { "description": "", "schema": { - "$ref": "#/definitions/expLessonCategoryMap" + "$ref": "#/definitions/expLessonDefs" } } }, + "parameters": [ + { + "name": "Category", + "in": "query", + "required": false, + "type": "string" + } + ], "tags": [ "LessonDefService" ] @@ -56,6 +64,31 @@ "LessonDefService" ] } + }, + "/exp/lessondef/{id}/prereqs": { + "get": { + "operationId": "GetAllLessonPrereqs", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/expLessonPrereqs" + } + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "type": "integer", + "format": "int32" + } + ], + "tags": [ + "LessonDefService" + ] + } } }, "definitions": { @@ -127,17 +160,6 @@ } } }, - "expLessonCategoryMap": { - "type": "object", - "properties": { - "lessonCategories": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/expLessonDefs" - } - } - } - }, "expLessonDef": { "type": "object", "properties": { @@ -195,6 +217,29 @@ }, "Tier": { "type": "string" + }, + "Prereqs": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + "Tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "Collection": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Slug": { + "type": "string", + "title": "This is meant to fill: \"How well do you know \u003cslug\u003e?\"" } } }, @@ -209,6 +254,18 @@ } } }, + "expLessonPrereqs": { + "type": "object", + "properties": { + "prereqs": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + }, "expLessonStage": { "type": "object", "properties": { diff --git a/api/exp/lessondefs.go b/api/exp/lessondefs.go index 05476002..8f97d39b 100644 --- a/api/exp/lessondefs.go +++ b/api/exp/lessondefs.go @@ -9,35 +9,88 @@ import ( "path/filepath" "reflect" - "github.com/golang/protobuf/ptypes/empty" pb "github.com/nre-learning/syringe/api/exp/generated" "github.com/nre-learning/syringe/config" log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" ) -func (s *server) ListLessonDefs(ctx context.Context, _ *empty.Empty) (*pb.LessonCategoryMap, error) { +func (s *server) ListLessonDefs(ctx context.Context, filter *pb.LessonDefFilter) (*pb.LessonDefs, error) { - retMap := map[string]*pb.LessonDefs{} + defs := []*pb.LessonDef{} // TODO(mierdin): Okay for now, but not super efficient. Should store in category keys when loaded. for _, lessonDef := range s.scheduler.LessonDefs { - // Initialize category - if _, ok := retMap[lessonDef.Category]; !ok { - retMap[lessonDef.Category] = &pb.LessonDefs{ - LessonDefs: []*pb.LessonDef{}, - } + for s := range lessonDef.Stages { + lessonDef.Stages[s].LabGuide = "" + } + + if filter.Category == "" { + defs = append(defs, lessonDef) + continue } - retMap[lessonDef.Category].LessonDefs = append(retMap[lessonDef.Category].LessonDefs, lessonDef) + if lessonDef.Category == filter.Category { + defs = append(defs, lessonDef) + } } - return &pb.LessonCategoryMap{ - LessonCategories: retMap, + return &pb.LessonDefs{ + LessonDefs: defs, + }, nil +} + +// var preReqs []int32 + +func (s *server) GetAllLessonPrereqs(ctx context.Context, lid *pb.LessonID) (*pb.LessonPrereqs, error) { + + // Preload the requested lesson ID so we can strip it before returning + pr := s.getPrereqs(lid.Id, []int32{lid.Id}) + log.Debugf("Getting prerequisites for Lesson %d: %d", lid.Id, pr) + + return &pb.LessonPrereqs{ + // Remove first item from slice - this is the lesson ID being requested + Prereqs: pr[1:], }, nil } +func (s *server) getPrereqs(lessonID int32, currentPrereqs []int32) []int32 { + + // Return if lesson ID doesn't exist + if _, ok := s.scheduler.LessonDefs[lessonID]; !ok { + return currentPrereqs + } + + // Add this lessonID to prereqs if doesn't already exist + if !isAlreadyInSlice(lessonID, currentPrereqs) { + currentPrereqs = append(currentPrereqs, lessonID) + } + + // Return if lesson doesn't have prerequisites + lesson := s.scheduler.LessonDefs[lessonID] + if len(lesson.Prereqs) == 0 { + return currentPrereqs + } + + // Call recursion for lesson IDs that need it + for i := range lesson.Prereqs { + pid := lesson.Prereqs[i] + currentPrereqs = s.getPrereqs(pid, currentPrereqs) + } + + return currentPrereqs +} + +func isAlreadyInSlice(lessonID int32, currentPrereqs []int32) bool { + for i := range currentPrereqs { + if currentPrereqs[i] == lessonID { + return true + } + } + return false +} + func (s *server) GetLessonDef(ctx context.Context, lid *pb.LessonID) (*pb.LessonDef, error) { if _, ok := s.scheduler.LessonDefs[lid.Id]; !ok { diff --git a/api/exp/server.go b/api/exp/server.go index 93c0dc47..21073b74 100644 --- a/api/exp/server.go +++ b/api/exp/server.go @@ -11,6 +11,7 @@ import ( "strings" "sync" + "github.com/golang/glog" swag "github.com/nre-learning/syringe/api/exp/swagger" "github.com/nre-learning/syringe/pkg/ui/data/swagger" @@ -191,8 +192,35 @@ func grpcHandlerFunc(grpcServer *grpc.Server, otherHandler http.Handler) http.Ha } }) + // Allow CORS (ONLY IN PREPROD) // Add gorilla's logging handler for standards-based access logging - return ghandlers.LoggingHandler(os.Stdout, handlerFunc) + return ghandlers.LoggingHandler(os.Stdout, allowCORS(handlerFunc)) +} + +// allowCORS allows Cross Origin Resoruce Sharing from any origin. +// Don't do this without consideration in production systems. +func allowCORS(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if origin := r.Header.Get("Origin"); origin != "" { + w.Header().Set("Access-Control-Allow-Origin", origin) + if r.Method == "OPTIONS" && r.Header.Get("Access-Control-Request-Method") != "" { + preflightHandler(w, r) + return + } + } + h.ServeHTTP(w, r) + }) +} + +// preflightHandler adds the necessary headers in order to serve +// CORS from any origin using the methods "GET", "HEAD", "POST", "PUT", "DELETE" +// We insist, don't do this without consideration in production systems. +func preflightHandler(w http.ResponseWriter, r *http.Request) { + headers := []string{"Content-Type", "Accept"} + w.Header().Set("Access-Control-Allow-Headers", strings.Join(headers, ",")) + methods := []string{"GET", "HEAD", "POST", "PUT", "DELETE"} + w.Header().Set("Access-Control-Allow-Methods", strings.Join(methods, ",")) + glog.Infof("preflight request for %s", r.URL.Path) } func serveSwagger(mux *http.ServeMux) { diff --git a/api/exp/swagger/swagger.pb.go b/api/exp/swagger/swagger.pb.go index bfd8fec9..f7d75a17 100644 --- a/api/exp/swagger/swagger.pb.go +++ b/api/exp/swagger/swagger.pb.go @@ -18,7 +18,7 @@ Lessondef = `{ "application/json" ], "paths": { - "/exp/lessondef/all": { + "/exp/lessondef": { "get": { "summary": "Retrieve all LessonDefs with filter", "operationId": "ListLessonDefs", @@ -26,10 +26,18 @@ Lessondef = `{ "200": { "description": "", "schema": { - "$ref": "#/definitions/expLessonCategoryMap" + "$ref": "#/definitions/expLessonDefs" } } }, + "parameters": [ + { + "name": "Category", + "in": "query", + "required": false, + "type": "string" + } + ], "tags": [ "LessonDefService" ] @@ -59,6 +67,31 @@ Lessondef = `{ "LessonDefService" ] } + }, + "/exp/lessondef/{id}/prereqs": { + "get": { + "operationId": "GetAllLessonPrereqs", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/expLessonPrereqs" + } + } + }, + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "type": "integer", + "format": "int32" + } + ], + "tags": [ + "LessonDefService" + ] + } } }, "definitions": { @@ -130,17 +163,6 @@ Lessondef = `{ } } }, - "expLessonCategoryMap": { - "type": "object", - "properties": { - "lessonCategories": { - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/expLessonDefs" - } - } - } - }, "expLessonDef": { "type": "object", "properties": { @@ -198,6 +220,29 @@ Lessondef = `{ }, "Tier": { "type": "string" + }, + "Prereqs": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + }, + "Tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "Collection": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Slug": { + "type": "string", + "title": "This is meant to fill: \"How well do you know \u003cslug\u003e?\"" } } }, @@ -212,6 +257,18 @@ Lessondef = `{ } } }, + "expLessonPrereqs": { + "type": "object", + "properties": { + "prereqs": { + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + }, "expLessonStage": { "type": "object", "properties": { diff --git a/cmd/syringed/main.go b/cmd/syringed/main.go index fcf036ef..a38bc8a1 100644 --- a/cmd/syringed/main.go +++ b/cmd/syringed/main.go @@ -13,7 +13,6 @@ import ( config "github.com/nre-learning/syringe/config" "github.com/nre-learning/syringe/scheduler" log "github.com/sirupsen/logrus" - "k8s.io/client-go/rest" ) func init() { @@ -29,10 +28,10 @@ func main() { log.Fatalf("Invalid configuration. Please re-run Syringe with appropriate env variables") } - kubeConfig, err := rest.InClusterConfig() - if err != nil { - log.Fatal(err) - } + // kubeConfig, err := rest.InClusterConfig() + // if err != nil { + // log.Fatal(err) + // } lessonDefs, err := api.ImportLessonDefs(syringeConfig, syringeConfig.LessonsDir) if err != nil { @@ -41,7 +40,7 @@ func main() { // Start lesson scheduler lessonScheduler := scheduler.LessonScheduler{ - KubeConfig: kubeConfig, + // KubeConfig: kubeConfig, Requests: make(chan *scheduler.LessonScheduleRequest), Results: make(chan *scheduler.LessonScheduleResult), LessonDefs: lessonDefs, @@ -49,12 +48,12 @@ func main() { GcWhiteList: make(map[string]*pb.Session), GcWhiteListMu: &sync.Mutex{}, } - go func() { - err = lessonScheduler.Start() - if err != nil { - log.Fatalf("Problem starting lesson scheduler: %s", err) - } - }() + // go func() { + // err = lessonScheduler.Start() + // if err != nil { + // log.Fatalf("Problem starting lesson scheduler: %s", err) + // } + // }() antidoteSha, err := ioutil.ReadFile(fmt.Sprintf("%s/.git/refs/heads/%s", syringeConfig.LessonRepoDir, syringeConfig.LessonRepoBranch)) if err != nil { From e3bb82e323306274f056ba99f8ebfbc659d0df2b Mon Sep 17 00:00:00 2001 From: Matt Oswalt Date: Sun, 3 Feb 2019 11:30:54 -0800 Subject: [PATCH 2/2] Re-enable scheduler Signed-off-by: Matt Oswalt --- cmd/syringed/main.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/syringed/main.go b/cmd/syringed/main.go index a38bc8a1..fcf036ef 100644 --- a/cmd/syringed/main.go +++ b/cmd/syringed/main.go @@ -13,6 +13,7 @@ import ( config "github.com/nre-learning/syringe/config" "github.com/nre-learning/syringe/scheduler" log "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" ) func init() { @@ -28,10 +29,10 @@ func main() { log.Fatalf("Invalid configuration. Please re-run Syringe with appropriate env variables") } - // kubeConfig, err := rest.InClusterConfig() - // if err != nil { - // log.Fatal(err) - // } + kubeConfig, err := rest.InClusterConfig() + if err != nil { + log.Fatal(err) + } lessonDefs, err := api.ImportLessonDefs(syringeConfig, syringeConfig.LessonsDir) if err != nil { @@ -40,7 +41,7 @@ func main() { // Start lesson scheduler lessonScheduler := scheduler.LessonScheduler{ - // KubeConfig: kubeConfig, + KubeConfig: kubeConfig, Requests: make(chan *scheduler.LessonScheduleRequest), Results: make(chan *scheduler.LessonScheduleResult), LessonDefs: lessonDefs, @@ -48,12 +49,12 @@ func main() { GcWhiteList: make(map[string]*pb.Session), GcWhiteListMu: &sync.Mutex{}, } - // go func() { - // err = lessonScheduler.Start() - // if err != nil { - // log.Fatalf("Problem starting lesson scheduler: %s", err) - // } - // }() + go func() { + err = lessonScheduler.Start() + if err != nil { + log.Fatalf("Problem starting lesson scheduler: %s", err) + } + }() antidoteSha, err := ioutil.ReadFile(fmt.Sprintf("%s/.git/refs/heads/%s", syringeConfig.LessonRepoDir, syringeConfig.LessonRepoBranch)) if err != nil {