From f95747b05d87954ca727125f2ca8835bd17a3e09 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Mon, 10 Jul 2023 15:39:02 -0700 Subject: [PATCH 1/4] add schema list api route. add String method to FieldKind type. --- api/http/handlerfuncs.go | 55 ++++++++++++++++++++++++++++++++++++++-- api/http/router.go | 2 ++ client/descriptions.go | 35 +++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/api/http/handlerfuncs.go b/api/http/handlerfuncs.go index d6e90af778..dbb05850b2 100644 --- a/api/http/handlerfuncs.go +++ b/api/http/handlerfuncs.go @@ -155,9 +155,60 @@ func execGQLHandler(rw http.ResponseWriter, req *http.Request) { sendJSON(req.Context(), rw, newGQLResult(result.GQL), http.StatusOK) } -type collectionResponse struct { - Name string `json:"name"` +type fieldResponse struct { ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` +} + +type collectionResponse struct { + Name string `json:"name"` + ID string `json:"id"` + Fields []fieldResponse `json:"fields,omitempty"` +} + +func listSchemaHandler(rw http.ResponseWriter, req *http.Request) { + db, err := dbFromContext(req.Context()) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + cols, err := db.GetAllCollections(req.Context()) + if err != nil { + handleErr(req.Context(), rw, err, http.StatusInternalServerError) + return + } + + colResp := make([]collectionResponse, len(cols)) + for i, col := range cols { + fields := make([]fieldResponse, len(col.Schema().Fields)) + for j, field := range col.Schema().Fields { + fields[j] = fieldResponse{ + ID: field.ID.String(), + Name: field.Name, + } + if field.IsObjectArray() { + fields[j].Kind = fmt.Sprintf("[%s]", field.Schema) + } else if field.IsObject() { + fields[j].Kind = field.Schema + } else { + fields[j].Kind = field.Kind.String() + } + } + colResp[i] = collectionResponse{ + Name: col.Name(), + ID: col.SchemaID(), + Fields: fields, + } + } + + sendJSON( + req.Context(), + rw, + simpleDataResponse("result", "success", "collections", colResp), + http.StatusOK, + ) } func loadSchemaHandler(rw http.ResponseWriter, req *http.Request) { diff --git a/api/http/router.go b/api/http/router.go index aa154a8622..b02eaba40f 100644 --- a/api/http/router.go +++ b/api/http/router.go @@ -30,6 +30,7 @@ const ( DumpPath string = versionedAPIPath + "/debug/dump" BlocksPath string = versionedAPIPath + "/blocks" GraphQLPath string = versionedAPIPath + "/graphql" + SchemaListPath string = versionedAPIPath + "/schema/list" SchemaLoadPath string = versionedAPIPath + "/schema/load" SchemaPatchPath string = versionedAPIPath + "/schema/patch" IndexPath string = versionedAPIPath + "/index" @@ -59,6 +60,7 @@ func setRoutes(h *handler) *handler { h.Get(BlocksPath+"/{cid}", h.handle(getBlockHandler)) h.Get(GraphQLPath, h.handle(execGQLHandler)) h.Post(GraphQLPath, h.handle(execGQLHandler)) + h.Get(SchemaListPath, h.handle(listSchemaHandler)) h.Post(SchemaLoadPath, h.handle(loadSchemaHandler)) h.Post(SchemaPatchPath, h.handle(patchSchemaHandler)) h.Post(IndexPath, h.handle(createIndexHandler)) diff --git a/client/descriptions.go b/client/descriptions.go index 5d6a37bacc..8713ddb152 100644 --- a/client/descriptions.go +++ b/client/descriptions.go @@ -121,6 +121,41 @@ func (sd SchemaDescription) GetField(name string) (FieldDescription, bool) { // FieldKind describes the type of a field. type FieldKind uint8 +func (f FieldKind) String() string { + switch f { + case FieldKind_DocKey: + return "ID" + case FieldKind_BOOL: + return "Boolean" + case FieldKind_NILLABLE_BOOL_ARRAY: + return "[Boolean]" + case FieldKind_BOOL_ARRAY: + return "[Boolean!]" + case FieldKind_INT: + return "Int" + case FieldKind_NILLABLE_INT_ARRAY: + return "[Int]" + case FieldKind_INT_ARRAY: + return "[Int!]" + case FieldKind_DATETIME: + return "DateTime" + case FieldKind_FLOAT: + return "Float" + case FieldKind_NILLABLE_FLOAT_ARRAY: + return "[Float]" + case FieldKind_FLOAT_ARRAY: + return "[Float!]" + case FieldKind_STRING: + return "String" + case FieldKind_NILLABLE_STRING_ARRAY: + return "[String]" + case FieldKind_STRING_ARRAY: + return "[String!]" + default: + return fmt.Sprint(uint8(f)) + } +} + // Note: These values are serialized and persisted in the database, avoid modifying existing values. const ( FieldKind_None FieldKind = 0 From 69b9271090c971d4f680035e95c209a4262edbf1 Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jul 2023 12:15:44 -0700 Subject: [PATCH 2/4] add schema list cli command. ignore generated fields when listing schemas --- api/http/handlerfuncs.go | 17 +++++--- cli/cli.go | 1 + cli/schema_list.go | 84 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 cli/schema_list.go diff --git a/api/http/handlerfuncs.go b/api/http/handlerfuncs.go index dbb05850b2..45a18c6315 100644 --- a/api/http/handlerfuncs.go +++ b/api/http/handlerfuncs.go @@ -26,6 +26,7 @@ import ( "github.com/multiformats/go-multihash" "github.com/pkg/errors" + "github.com/sourcenetwork/defradb/client" corecrdt "github.com/sourcenetwork/defradb/core/crdt" "github.com/sourcenetwork/defradb/events" ) @@ -182,19 +183,23 @@ func listSchemaHandler(rw http.ResponseWriter, req *http.Request) { colResp := make([]collectionResponse, len(cols)) for i, col := range cols { - fields := make([]fieldResponse, len(col.Schema().Fields)) - for j, field := range col.Schema().Fields { - fields[j] = fieldResponse{ + var fields []fieldResponse + for _, field := range col.Schema().Fields { + if field.Name == "_key" || field.RelationType == client.Relation_Type_INTERNAL_ID { + continue // ignore generated fields + } + fieldRes := fieldResponse{ ID: field.ID.String(), Name: field.Name, } if field.IsObjectArray() { - fields[j].Kind = fmt.Sprintf("[%s]", field.Schema) + fieldRes.Kind = fmt.Sprintf("[%s]", field.Schema) } else if field.IsObject() { - fields[j].Kind = field.Schema + fieldRes.Kind = field.Schema } else { - fields[j].Kind = field.Kind.String() + fieldRes.Kind = field.Kind.String() } + fields = append(fields, fieldRes) } colResp[i] = collectionResponse{ Name: col.Name(), diff --git a/cli/cli.go b/cli/cli.go index 2069528296..c439d53015 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -80,6 +80,7 @@ func NewDefraCommand(cfg *config.Config) DefraCommand { ) schemaCmd.AddCommand( MakeSchemaAddCommand(cfg), + MakeSchemaListCommand(cfg), MakeSchemaPatchCommand(cfg), ) indexCmd.AddCommand( diff --git a/cli/schema_list.go b/cli/schema_list.go new file mode 100644 index 0000000000..96c4f8ff11 --- /dev/null +++ b/cli/schema_list.go @@ -0,0 +1,84 @@ +// Copyright 2023 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package cli + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/spf13/cobra" + + httpapi "github.com/sourcenetwork/defradb/api/http" + "github.com/sourcenetwork/defradb/config" + "github.com/sourcenetwork/defradb/errors" +) + +type schemaListResponse struct { + Data struct { + Result string `json:"result"` + Collections []struct { + Name string `json:"name"` + ID string `json:"id"` + Fields []struct { + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"fields"` + } `json:"collections"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +func MakeSchemaListCommand(cfg *config.Config) *cobra.Command { + var cmd = &cobra.Command{ + Use: "list", + Short: "List schema types from DefraDB", + RunE: func(cmd *cobra.Command, args []string) (err error) { + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaListPath) + if err != nil { + return NewErrFailedToJoinEndpoint(err) + } + + res, err := http.Get(endpoint.String()) + if err != nil { + return NewErrFailedToSendRequest(err) + } + defer res.Body.Close() //nolint:errcheck + + data, err := io.ReadAll(res.Body) + if err != nil { + return NewErrFailedToReadResponseBody(err) + } + + var r schemaListResponse + if err := json.Unmarshal(data, &r); err != nil { + return NewErrFailedToUnmarshalResponse(err) + } + if len(r.Errors) > 0 { + return errors.New("failed to list schemas", errors.NewKV("errors", r.Errors)) + } + + for _, c := range r.Data.Collections { + cmd.Printf("type %s {\n", c.Name) + for _, f := range c.Fields { + cmd.Printf("\t%s: %s\n", f.Name, f.Kind) + } + cmd.Printf("}\n") + } + + return nil + }, + } + return cmd +} From fc9e9e1f207a9aab433434e9065851d0b524f5fd Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jul 2023 15:39:33 -0700 Subject: [PATCH 3/4] simplify schema api routes --- api/http/handlerfuncs_test.go | 8 ++++---- api/http/router.go | 24 +++++++++++------------- cli/schema_add.go | 2 +- cli/schema_list.go | 4 ++-- cli/schema_patch.go | 8 ++++++-- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/api/http/handlerfuncs_test.go b/api/http/handlerfuncs_test.go index b21526efc0..888dd401e8 100644 --- a/api/http/handlerfuncs_test.go +++ b/api/http/handlerfuncs_test.go @@ -604,7 +604,7 @@ func TestLoadSchemaHandlerWithReadBodyError(t *testing.T) { Testing: t, DB: nil, Method: "POST", - Path: SchemaLoadPath, + Path: SchemaPath, Body: &mockReadCloser, ExpectedStatus: 500, ResponseData: &errResponse, @@ -634,7 +634,7 @@ type user { Testing: t, DB: nil, Method: "POST", - Path: SchemaLoadPath, + Path: SchemaPath, Body: buf, ExpectedStatus: 500, ResponseData: &errResponse, @@ -669,7 +669,7 @@ types user { Testing: t, DB: defra, Method: "POST", - Path: SchemaLoadPath, + Path: SchemaPath, Body: buf, ExpectedStatus: 500, ResponseData: &errResponse, @@ -705,7 +705,7 @@ type user { Testing: t, DB: defra, Method: "POST", - Path: SchemaLoadPath, + Path: SchemaPath, Body: buf, ExpectedStatus: 200, ResponseData: &resp, diff --git a/api/http/router.go b/api/http/router.go index b02eaba40f..6f74f59535 100644 --- a/api/http/router.go +++ b/api/http/router.go @@ -25,16 +25,14 @@ const ( Version string = "v0" versionedAPIPath string = "/api/" + Version - RootPath string = versionedAPIPath + "" - PingPath string = versionedAPIPath + "/ping" - DumpPath string = versionedAPIPath + "/debug/dump" - BlocksPath string = versionedAPIPath + "/blocks" - GraphQLPath string = versionedAPIPath + "/graphql" - SchemaListPath string = versionedAPIPath + "/schema/list" - SchemaLoadPath string = versionedAPIPath + "/schema/load" - SchemaPatchPath string = versionedAPIPath + "/schema/patch" - IndexPath string = versionedAPIPath + "/index" - PeerIDPath string = versionedAPIPath + "/peerid" + RootPath string = versionedAPIPath + "" + PingPath string = versionedAPIPath + "/ping" + DumpPath string = versionedAPIPath + "/debug/dump" + BlocksPath string = versionedAPIPath + "/blocks" + GraphQLPath string = versionedAPIPath + "/graphql" + SchemaPath string = versionedAPIPath + "/schema" + IndexPath string = versionedAPIPath + "/index" + PeerIDPath string = versionedAPIPath + "/peerid" ) func setRoutes(h *handler) *handler { @@ -60,9 +58,9 @@ func setRoutes(h *handler) *handler { h.Get(BlocksPath+"/{cid}", h.handle(getBlockHandler)) h.Get(GraphQLPath, h.handle(execGQLHandler)) h.Post(GraphQLPath, h.handle(execGQLHandler)) - h.Get(SchemaListPath, h.handle(listSchemaHandler)) - h.Post(SchemaLoadPath, h.handle(loadSchemaHandler)) - h.Post(SchemaPatchPath, h.handle(patchSchemaHandler)) + h.Get(SchemaPath, h.handle(listSchemaHandler)) + h.Post(SchemaPath, h.handle(loadSchemaHandler)) + h.Patch(SchemaPath, h.handle(patchSchemaHandler)) h.Post(IndexPath, h.handle(createIndexHandler)) h.Delete(IndexPath, h.handle(dropIndexHandler)) h.Get(IndexPath, h.handle(listIndexHandler)) diff --git a/cli/schema_add.go b/cli/schema_add.go index 4fc916567f..a9742049e4 100644 --- a/cli/schema_add.go +++ b/cli/schema_add.go @@ -94,7 +94,7 @@ Learn more about the DefraDB GraphQL Schema Language on https://docs.source.netw return errors.New("empty schema provided") } - endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaLoadPath) + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaPath) if err != nil { return errors.Wrap("join paths failed", err) } diff --git a/cli/schema_list.go b/cli/schema_list.go index 96c4f8ff11..94fa8087a2 100644 --- a/cli/schema_list.go +++ b/cli/schema_list.go @@ -43,9 +43,9 @@ type schemaListResponse struct { func MakeSchemaListCommand(cfg *config.Config) *cobra.Command { var cmd = &cobra.Command{ Use: "list", - Short: "List schema types from DefraDB", + Short: "List schema types with their respective fields", RunE: func(cmd *cobra.Command, args []string) (err error) { - endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaListPath) + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaPath) if err != nil { return NewErrFailedToJoinEndpoint(err) } diff --git a/cli/schema_patch.go b/cli/schema_patch.go index 31ac830345..6923a03c9b 100644 --- a/cli/schema_patch.go +++ b/cli/schema_patch.go @@ -95,12 +95,16 @@ To learn more about the DefraDB GraphQL Schema Language, refer to https://docs.s return ErrEmptyFile } - endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaPatchPath) + endpoint, err := httpapi.JoinPaths(cfg.API.AddressToURL(), httpapi.SchemaPath) if err != nil { return err } - res, err := http.Post(endpoint.String(), "text", strings.NewReader(patch)) + req, err := http.NewRequest(http.MethodPatch, endpoint.String(), strings.NewReader(patch)) + if err != nil { + return NewErrFailedToSendRequest(err) + } + res, err := http.DefaultClient.Do(req) if err != nil { return NewErrFailedToSendRequest(err) } From f48888bc62ba8d7584d1ac58ebd72cef1793d8ce Mon Sep 17 00:00:00 2001 From: Keenan Nemetz Date: Tue, 11 Jul 2023 16:03:48 -0700 Subject: [PATCH 4/4] add list schema http api test. --- api/http/handlerfuncs.go | 2 +- api/http/handlerfuncs_test.go | 106 ++++++++++++++++++++++++++++++++++ cli/schema_list.go | 1 - 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/api/http/handlerfuncs.go b/api/http/handlerfuncs.go index 45a18c6315..6d49afbae1 100644 --- a/api/http/handlerfuncs.go +++ b/api/http/handlerfuncs.go @@ -211,7 +211,7 @@ func listSchemaHandler(rw http.ResponseWriter, req *http.Request) { sendJSON( req.Context(), rw, - simpleDataResponse("result", "success", "collections", colResp), + simpleDataResponse("collections", colResp), http.StatusOK, ) } diff --git a/api/http/handlerfuncs_test.go b/api/http/handlerfuncs_test.go index 888dd401e8..0389a43ffd 100644 --- a/api/http/handlerfuncs_test.go +++ b/api/http/handlerfuncs_test.go @@ -592,6 +592,112 @@ mutation { } } +func TestListSchemaHandlerWithoutDB(t *testing.T) { + t.Cleanup(CleanupEnv) + env = "dev" + + errResponse := ErrorResponse{} + testRequest(testOptions{ + Testing: t, + DB: nil, + Method: "GET", + Path: SchemaPath, + ExpectedStatus: 500, + ResponseData: &errResponse, + }) + + assert.Contains(t, errResponse.Errors[0].Extensions.Stack, "no database available") + assert.Equal(t, http.StatusInternalServerError, errResponse.Errors[0].Extensions.Status) + assert.Equal(t, "Internal Server Error", errResponse.Errors[0].Extensions.HTTPError) + assert.Equal(t, "no database available", errResponse.Errors[0].Message) +} + +func TestListSchemaHandlerWitNoError(t *testing.T) { + ctx := context.Background() + defra := testNewInMemoryDB(t, ctx) + defer defra.Close(ctx) + + stmt := ` +type user { + name: String + age: Int + verified: Boolean + points: Float +} +type group { + owner: user + members: [user] +}` + + _, err := defra.AddSchema(ctx, stmt) + if err != nil { + t.Fatal(err) + } + + resp := DataResponse{} + testRequest(testOptions{ + Testing: t, + DB: defra, + Method: "GET", + Path: SchemaPath, + ExpectedStatus: 200, + ResponseData: &resp, + }) + + switch v := resp.Data.(type) { + case map[string]any: + assert.Equal(t, map[string]any{ + "collections": []any{ + map[string]any{ + "name": "group", + "id": "bafkreieunyhcyupkdppyo2g4zcqtdxvj5xi4f422gp2jwene6ohndvcobe", + "fields": []any{ + map[string]any{ + "id": "1", + "kind": "[user]", + "name": "members", + }, + map[string]any{ + "id": "2", + "kind": "user", + "name": "owner", + }, + }, + }, + map[string]any{ + "name": "user", + "id": "bafkreigrucdl7x3lsa4xwgz2bn7lbqmiwkifnspgx7hlkpaal3o55325bq", + "fields": []any{ + map[string]any{ + "id": "1", + "kind": "Int", + "name": "age", + }, + map[string]any{ + "id": "2", + "kind": "String", + "name": "name", + }, + map[string]any{ + "id": "3", + "kind": "Float", + "name": "points", + }, + map[string]any{ + "id": "4", + "kind": "Boolean", + "name": "verified", + }, + }, + }, + }, + }, v) + + default: + t.Fatalf("data should be of type map[string]any but got %T\n%v", resp.Data, v) + } +} + func TestLoadSchemaHandlerWithReadBodyError(t *testing.T) { t.Cleanup(CleanupEnv) env = "dev" diff --git a/cli/schema_list.go b/cli/schema_list.go index 94fa8087a2..921d6ddfb6 100644 --- a/cli/schema_list.go +++ b/cli/schema_list.go @@ -24,7 +24,6 @@ import ( type schemaListResponse struct { Data struct { - Result string `json:"result"` Collections []struct { Name string `json:"name"` ID string `json:"id"`