diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 06384f95be..2f796a11de 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -16,6 +16,7 @@ on: - "consumers/notifiers/api/**" - "http/api/**" - "invitations/api/**" + - "journal/api/**" - "provision/api/**" - "readers/api/**" - "things/api/**" @@ -44,6 +45,7 @@ env: TIMESCALE_READER_URL: http://localhost:9011 SMPP_NOTIFIER_URL: http://localhost:9014 SMTP_NOTIFIER_URL: http://localhost:9015 + JOURNAL_URL: http://localhost:9021 jobs: api-test: @@ -78,6 +80,11 @@ jobs: id: changes with: filters: | + journal: + - ".github/workflows/api-tests.yml" + - "api/openapi/journal.yml" + - "journal/api/**" + auth: - ".github/workflows/api-tests.yml" - "api/openapi/auth.yml" @@ -183,6 +190,17 @@ jobs: report: false args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links' + - name: Run Journal API tests + if: steps.changes.outputs.journal == 'true' + uses: schemathesis/action@v1 + with: + schema: api/openapi/journal.yml + base-url: ${{ env.JOURNAL_URL }} + checks: all + report: false + args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --contrib-openapi-formats-uuid --hypothesis-suppress-health-check=filter_too_much --stateful=links' + + - name: Run Bootstrap API tests if: steps.changes.outputs.bootstrap == 'true' uses: schemathesis/action@v1 diff --git a/.github/workflows/check-generated-files.yml b/.github/workflows/check-generated-files.yml index 1b5f57cb7d..bca413a0c7 100644 --- a/.github/workflows/check-generated-files.yml +++ b/.github/workflows/check-generated-files.yml @@ -75,6 +75,7 @@ jobs: - "twins/twins.go" - "twins/states.go" - "twins/service.go" + - "journal/journal.go" - name: Set up protoc if: steps.changes.outputs.proto == 'true' @@ -157,6 +158,8 @@ jobs: mv ./twins/mocks/states.go ./twins/mocks/states.go.tmp mv ./twins/mocks/repository.go ./twins/mocks/repository.go.tmp mv ./twins/mocks/cache.go ./twins/mocks/cache.go.tmp + mv ./journal/mocks/repository.go ./journal/mocks/repository.go.tmp + mv ./journal/mocks/service.go ./journal/mocks/service.go.tmp make mocks @@ -208,3 +211,5 @@ jobs: check_mock_changes ./twins/mocks/states.go "Twins States ./twins/mocks/states.go" check_mock_changes ./twins/mocks/repository.go "Twins Repository ./twins/mocks/repository.go" check_mock_changes ./twins/mocks/cache.go "Twins Cache ./twins/mocks/cache.go" + check_mock_changes ./journal/mocks/repository.go "Journal Repository ./journal/mocks/repository.go" + check_mock_changes ./journal/mocks/service.go "Journal Service ./journal/mocks/service.go" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4baf4b6fc..c903f53436 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -116,6 +116,14 @@ jobs: - "pkg/uuid/**" - "pkg/messaging/**" + journal: + - "journal/**" + - "cmd/journal/**" + - "auth.pb.go" + - "auth_grpc.pb.go" + - "auth/**" + - "pkg/events/**" + http: - "http/**" - "cmd/http/**" @@ -268,6 +276,11 @@ jobs: run: | mkdir coverage + - name: Run Journal tests + if: steps.changes.outputs.journal == 'true' || steps.changes.outputs.workflow == 'true' + run: | + go test --race -v -count=1 -coverprofile=coverage/journal.out ./journal/... + - name: Run auth tests if: steps.changes.outputs.auth == 'true' || steps.changes.outputs.workflow == 'true' run: | diff --git a/Makefile b/Makefile index b9d3fdcdb5..62c2143e6f 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,8 @@ MG_DOCKER_IMAGE_NAME_PREFIX ?= magistrala BUILD_DIR = build SERVICES = auth users things http coap ws lora influxdb-writer influxdb-reader mongodb-writer \ mongodb-reader cassandra-writer cassandra-reader postgres-writer postgres-reader timescale-writer timescale-reader cli \ - bootstrap opcua twins mqtt provision certs smtp-notifier smpp-notifier invitations -TEST_API_SERVICES = auth bootstrap certs http invitations notifiers provision readers things twins users + bootstrap opcua twins mqtt provision certs smtp-notifier smpp-notifier invitations journal +TEST_API_SERVICES = journal auth bootstrap certs http invitations notifiers provision readers things twins users TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES)) DOCKERS = $(addprefix docker_,$(SERVICES)) DOCKERS_DEV = $(addprefix docker_dev_,$(SERVICES)) @@ -73,7 +73,7 @@ endef ADDON_SERVICES = bootstrap cassandra-reader cassandra-writer certs \ influxdb-reader influxdb-writer lora-adapter mongodb-reader mongodb-writer \ opcua-adapter postgres-reader postgres-writer provision smpp-notifier smtp-notifier \ - timescale-reader timescale-writer twins + timescale-reader timescale-writer twins journal EXTERNAL_SERVICES = vault prometheus @@ -177,6 +177,7 @@ test_api_twins: TEST_API_URL := http://localhost:9018 test_api_provision: TEST_API_URL := http://localhost:9016 test_api_readers: TEST_API_URL := http://localhost:9009 # This can be the URL of any reader service. test_api_notifiers: TEST_API_URL := http://localhost:9014 # This can be the URL of any notifier service. +test_api_journal: TEST_API_URL := http://localhost:9021 $(TEST_API): $(call test_api_service,$(@),$(TEST_API_URL)) diff --git a/api/openapi/journal.yml b/api/openapi/journal.yml new file mode 100644 index 0000000000..ac55dba0db --- /dev/null +++ b/api/openapi/journal.yml @@ -0,0 +1,285 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +openapi: 3.0.3 +info: + title: Magistrala Journal Log Service + description: | + This is the Journal Log Server based on the OpenAPI 3.0 specification. It is the HTTP API for viewing journal log history. You can now help us improve the API whether it's by making changes to the definition itself or to the code. + Some useful links: + - [The Magistrala repository](https://github.com/absmach/magistrala) + contact: + email: info@mainflux.com + license: + name: Apache 2.0 + url: https://github.com/absmach/magistrala/blob/master/LICENSE + version: 0.14.0 + +servers: + - url: http://localhost:9021 + - url: https://localhost:9021 + +tags: + - name: journal-log + description: Everything about your Journal Log + externalDocs: + description: Find out more about Journal Log + url: http://docs.mainflux.io/ + +paths: + /journal/{entity_type}/{id}: + get: + tags: + - journal-log + summary: List journal log + description: | + Retrieves a list of journal. Due to performance concerns, data + is retrieved in subsets. The API must ensure that the entire + dataset is consumed either by making subsequent requests, or by + increasing the subset size of the initial request. + parameters: + - $ref: "#/components/parameters/entity_type" + - $ref: "#/components/parameters/id" + - $ref: "#/components/parameters/offset" + - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/operation" + - $ref: "#/components/parameters/with_attributes" + - $ref: "#/components/parameters/with_metadata" + - $ref: "#/components/parameters/from" + - $ref: "#/components/parameters/to" + - $ref: "#/components/parameters/dir" + security: + - bearerAuth: [] + responses: + "200": + $ref: "#/components/responses/JournalsPageRes" + "400": + description: Failed due to malformed query parameters. + "401": + description: Missing or invalid access token provided. + "403": + description: Failed to perform authorization over the entity. + "404": + description: A non-existent entity request. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + + /health: + get: + summary: Retrieves service health check info. + tags: + - health + responses: + "200": + $ref: "#/components/responses/HealthRes" + "500": + $ref: "#/components/responses/ServiceError" + +components: + schemas: + Journal: + type: object + properties: + operation: + type: string + example: user.create + description: Journal operation. + occurred_at: + type: string + format: date-time + example: "2024-01-11T12:05:07.449053Z" + description: Time when the journal occurred. + attributes: + type: object + description: Journal attributes. + example: + { + "created_at": "2024-06-12T11:34:32.991591Z", + "id": "29d425c8-542b-4614-8a4d-a5951945d720", + "identity": "Gawne-Havlicek@email.com", + "name": "Newgard-Frisina", + "status": "enabled", + "updated_at": "2024-06-12T11:34:33.116795Z", + "updated_by": "ad228f20-4741-47c5-bef7-d871b541c019", + } + metadata: + type: object + description: Journal payload. + example: { "Update": "Calvo-Felkins" } + xml: + name: journal + + JournalPage: + type: object + properties: + journals: + type: array + minItems: 0 + uniqueItems: true + items: + $ref: "#/components/schemas/Journal" + total: + type: integer + example: 1 + description: Total number of items. + offset: + type: integer + description: Number of items to skip during retrieval. + limit: + type: integer + example: 10 + description: Maximum number of items to return in one page. + required: + - journals + - total + - offset + + Error: + type: object + properties: + error: + type: string + description: Error message + example: { "error": "malformed entity specification" } + + parameters: + entity_type: + name: entity_type + description: Type of entity, e.g. user, group, thing, etc. + in: path + schema: + type: string + enum: + - user + - group + - thing + - channel + required: true + example: user + + id: + name: id + description: Unique identifier for an entity, e.g. user, group, domain, etc. Used together with entity_type. + in: path + schema: + type: string + format: uuid + required: true + example: bb7edb32-2eac-4aad-aebe-ed96fe073879 + + offset: + name: offset + description: Number of items to skip during retrieval. + in: query + schema: + type: integer + default: 0 + minimum: 0 + required: false + example: "0" + + limit: + name: limit + description: Size of the subset to retrieve. + in: query + schema: + type: integer + default: 10 + maximum: 10 + minimum: 1 + required: false + example: "10" + + operation: + name: operation + description: Journal operation. + in: query + schema: + type: string + required: false + example: user.create + + with_attributes: + name: with_attributes + description: Include journal attributes. + in: query + schema: + type: boolean + required: false + example: true + + with_metadata: + name: with_metadata + description: Include journal metadata. + in: query + schema: + type: boolean + required: false + example: true + + from: + name: from + description: Start date in unix time. + in: query + schema: + type: string + format: int64 + required: false + example: 1966777289 + + to: + name: to + description: End date in unix time. + in: query + schema: + type: string + format: int64 + required: false + example: 1966777289 + + dir: + name: dir + description: Sort direction. + in: query + schema: + type: string + enum: + - asc + - desc + required: false + example: desc + + responses: + JournalsPageRes: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/JournalPage" + + HealthRes: + description: Service Health Check. + content: + application/health+json: + schema: + $ref: "./schemas/HealthInfo.yml" + + ServiceError: + description: Unexpected server-side error occurred. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + description: | + * User access: "Authorization: Bearer " + +security: + - bearerAuth: [] diff --git a/api/openapi/notifiers.yml b/api/openapi/notifiers.yml index d78b052ef3..6a4c099d65 100644 --- a/api/openapi/notifiers.yml +++ b/api/openapi/notifiers.yml @@ -43,6 +43,8 @@ paths: $ref: "#/components/responses/Create" "400": description: Failed due to malformed JSON. + "401": + description: Missing or invalid access token provided. "403": description: Failed to perform authorization over the entity. "409": @@ -92,7 +94,7 @@ paths: "200": $ref: "#/components/responses/View" "400": - description: Failed due to malformed query parameters. + description: Failed due to malformed ID. "401": description: Missing or invalid access token provided. "403": diff --git a/auth/events/events.go b/auth/events/events.go index f384a99e29..e26981867a 100644 --- a/auth/events/events.go +++ b/auth/events/events.go @@ -4,9 +4,6 @@ package events import ( - "encoding/json" - "fmt" - "strings" "time" "github.com/absmach/magistrala/auth" @@ -59,16 +56,10 @@ func (cde createDomainEvent) Encode() (map[string]interface{}, error) { val["permission"] = cde.Permission } if len(cde.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(cde.Tags, ",")) - val["tags"] = tags + val["tags"] = cde.Tags } if cde.Metadata != nil { - metadata, err := json.Marshal(cde.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = cde.Metadata } return val, nil @@ -91,16 +82,10 @@ func (rde retrieveDomainEvent) Encode() (map[string]interface{}, error) { val["name"] = rde.Name } if len(rde.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(rde.Tags, ",")) - val["tags"] = tags + val["tags"] = rde.Tags } if rde.Metadata != nil { - metadata, err := json.Marshal(rde.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = rde.Metadata } if !rde.UpdatedAt.IsZero() { @@ -124,12 +109,7 @@ func (rpe retrieveDomainPermissionsEvent) Encode() (map[string]interface{}, erro } if rpe.permissions != nil { - permissions, err := json.Marshal(rpe.permissions) - if err != nil { - return map[string]interface{}{}, err - } - - val["permissions"] = permissions + val["permissions"] = rpe.permissions } return val, nil @@ -155,16 +135,10 @@ func (ude updateDomainEvent) Encode() (map[string]interface{}, error) { val["name"] = ude.Name } if len(ude.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(ude.Tags, ",")) - val["tags"] = tags + val["tags"] = ude.Tags } if ude.Metadata != nil { - metadata, err := json.Marshal(ude.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = ude.Metadata } return val, nil @@ -210,12 +184,7 @@ func (lde listDomainsEvent) Encode() (map[string]interface{}, error) { val["dir"] = lde.Dir } if lde.Metadata != nil { - metadata, err := json.Marshal(lde.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lde.Metadata } if lde.Tag != "" { val["tag"] = lde.Tag @@ -298,12 +267,7 @@ func (lde listUserDomainsEvent) Encode() (map[string]interface{}, error) { val["dir"] = lde.Dir } if lde.Metadata != nil { - metadata, err := json.Marshal(lde.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lde.Metadata } if lde.Tag != "" { val["tag"] = lde.Tag diff --git a/bootstrap/events/consumer/streams.go b/bootstrap/events/consumer/streams.go index d3defda655..7c0d5bcbfb 100644 --- a/bootstrap/events/consumer/streams.go +++ b/bootstrap/events/consumer/streams.go @@ -5,7 +5,6 @@ package consumer import ( "context" - "encoding/json" "time" "github.com/absmach/magistrala/bootstrap" @@ -69,6 +68,9 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { if thingID == "" { return svcerr.ErrMalformedEntity } + } + + for _, thingID := range dte.thingIDs { if err = es.svc.DisconnectThingHandler(ctx, dte.channelID, thingID); err != nil { return err } @@ -89,50 +91,47 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { func decodeRemoveThing(event map[string]interface{}) removeEvent { return removeEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } func decodeUpdateChannel(event map[string]interface{}) updateChannelEvent { - strmeta := read(event, "metadata", "{}") - var metadata map[string]interface{} - if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil { - metadata = map[string]interface{}{} - } + metadata := events.Read(event, "metadata", map[string]interface{}{}) return updateChannelEvent{ - id: read(event, "id", ""), - name: read(event, "name", ""), + id: events.Read(event, "id", ""), + name: events.Read(event, "name", ""), metadata: metadata, - updatedAt: readTime(event, "updated_at", time.Now()), - updatedBy: read(event, "updated_by", ""), + updatedAt: events.Read(event, "updated_at", time.Now()), + updatedBy: events.Read(event, "updated_by", ""), } } func decodeRemoveChannel(event map[string]interface{}) removeEvent { return removeEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } func decodeConnectThing(event map[string]interface{}) connectionEvent { - if read(event, "memberKind", "") != memberKind && read(event, "relation", "") != relation { + if events.Read(event, "memberKind", "") != memberKind && events.Read(event, "relation", "") != relation { return connectionEvent{} } return connectionEvent{ - channelID: read(event, "group_id", ""), - thingIDs: ReadStringSlice(event, "member_ids"), + channelID: events.Read(event, "group_id", ""), + thingIDs: events.ReadStringSlice(event, "member_ids"), } } func decodeDisconnectThing(event map[string]interface{}) connectionEvent { - if read(event, "memberKind", "") != memberKind && read(event, "relation", "") != relation { + if events.Read(event, "memberKind", "") != memberKind && events.Read(event, "relation", "") != relation { return connectionEvent{} } + return connectionEvent{ - channelID: read(event, "group_id", ""), - thingIDs: ReadStringSlice(event, "member_ids"), + channelID: events.Read(event, "group_id", ""), + thingIDs: events.ReadStringSlice(event, "member_ids"), } } @@ -144,42 +143,6 @@ func (es *eventHandler) handleUpdateChannel(ctx context.Context, uce updateChann UpdatedAt: uce.updatedAt, UpdatedBy: uce.updatedBy, } - return es.svc.UpdateChannelHandler(ctx, channel) -} - -func read(event map[string]interface{}, key, def string) string { - val, ok := event[key].(string) - if !ok { - return def - } - - return val -} - -// ReadStringSlice reads string slice from event map. -// If value is not a string slice, returns empty slice. -func ReadStringSlice(event map[string]interface{}, key string) []string { - var res []string - - vals, ok := event[key].([]interface{}) - if !ok { - return res - } - - for _, v := range vals { - if s, ok := v.(string); ok { - res = append(res, s) - } - } - return res -} - -func readTime(event map[string]interface{}, key string, def time.Time) time.Time { - val, ok := event[key].(time.Time) - if !ok { - return def - } - - return val + return es.svc.UpdateChannelHandler(ctx, channel) } diff --git a/bootstrap/events/producer/events.go b/bootstrap/events/producer/events.go index beebb7976a..1d4e0964ee 100644 --- a/bootstrap/events/producer/events.go +++ b/bootstrap/events/producer/events.go @@ -4,14 +4,12 @@ package producer import ( - "encoding/json" - "github.com/absmach/magistrala/bootstrap" "github.com/absmach/magistrala/pkg/events" ) const ( - configPrefix = "config." + configPrefix = "bootstrap.config." configCreate = configPrefix + "create" configUpdate = configPrefix + "update" configRemove = configPrefix + "remove" @@ -19,18 +17,18 @@ const ( configList = configPrefix + "list" configHandlerRemove = configPrefix + "remove_handler" - thingPrefix = "thing." + thingPrefix = "bootstrap.thing." thingBootstrap = thingPrefix + "bootstrap" thingStateChange = thingPrefix + "change_state" thingUpdateConnections = thingPrefix + "update_connections" thingConnect = thingPrefix + "connect" thingDisconnect = thingPrefix + "disconnect" - channelPrefix = "group." + channelPrefix = "bootstrap.channel." channelHandlerRemove = channelPrefix + "remove_handler" channelUpdateHandler = channelPrefix + "update_handler" - certUpdate = "cert.update" + certUpdate = "bootstrap.cert.update" ) var ( @@ -74,11 +72,7 @@ func (ce configEvent) Encode() (map[string]interface{}, error) { for i, ch := range ce.Channels { channels[i] = ch.ID } - data, err := json.Marshal(channels) - if err != nil { - return map[string]interface{}{}, err - } - val["channels"] = string(data) + val["channels"] = channels } if ce.ClientCert != "" { val["client_cert"] = ce.ClientCert @@ -121,21 +115,11 @@ func (rce listConfigsEvent) Encode() (map[string]interface{}, error) { "operation": configList, } if len(rce.fullMatch) > 0 { - data, err := json.Marshal(rce.fullMatch) - if err != nil { - return map[string]interface{}{}, err - } - - val["full_match"] = data + val["full_match"] = rce.fullMatch } if len(rce.partialMatch) > 0 { - data, err := json.Marshal(rce.partialMatch) - if err != nil { - return map[string]interface{}{}, err - } - - val["full_match"] = data + val["full_match"] = rce.partialMatch } return val, nil } @@ -173,11 +157,7 @@ func (be bootstrapEvent) Encode() (map[string]interface{}, error) { for i, ch := range be.Channels { channels[i] = ch.ID } - data, err := json.Marshal(channels) - if err != nil { - return map[string]interface{}{}, err - } - val["channels"] = string(data) + val["channels"] = channels } if be.ClientCert != "" { val["client_cert"] = be.ClientCert @@ -213,14 +193,9 @@ type updateConnectionsEvent struct { } func (uce updateConnectionsEvent) Encode() (map[string]interface{}, error) { - data, err := json.Marshal(uce.mgChannels) - if err != nil { - return map[string]interface{}{}, err - } - return map[string]interface{}{ "thing_id": uce.mgThing, - "channels": string(data), + "channels": uce.mgChannels, "operation": thingUpdateConnections, }, nil } @@ -267,12 +242,7 @@ func (uche updateChannelHandlerEvent) Encode() (map[string]interface{}, error) { val["name"] = uche.Name } if uche.Metadata != nil { - metadata, err := json.Marshal(uche.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = uche.Metadata } return val, nil } diff --git a/bootstrap/events/producer/streams_test.go b/bootstrap/events/producer/streams_test.go index 2bcddd0e5c..63e8dfc9a0 100644 --- a/bootstrap/events/producer/streams_test.go +++ b/bootstrap/events/producer/streams_test.go @@ -5,7 +5,6 @@ package producer_test import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -123,7 +122,7 @@ func TestAdd(t *testing.T) { "thing_id": "1", "owner": email, "name": config.Name, - "channels": strings.Join(channels, ", "), + "channels": channels, "external_id": config.ExternalID, "content": config.Content, "timestamp": time.Now().Unix(), @@ -238,8 +237,6 @@ func TestUpdate(t *testing.T) { nonExisting.ThingID = "unknown" channels := []string{modified.Channels[0].ID, modified.Channels[1].ID} - chs, err := json.Marshal(channels) - assert.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) cases := []struct { desc string @@ -258,7 +255,7 @@ func TestUpdate(t *testing.T) { "content": modified.Content, "timestamp": time.Now().UnixNano(), "operation": configUpdate, - "channels": string(chs), + "channels": channels, "external_id": modified.ExternalID, "thing_id": modified.ThingID, "owner": validID, @@ -340,7 +337,7 @@ func TestUpdateConnections(t *testing.T) { err: nil, event: map[string]interface{}{ "thing_id": saved.ThingID, - "channels": "2", + "channels": []string{"2"}, "timestamp": time.Now().Unix(), "operation": thingUpdateConnections, }, @@ -1226,21 +1223,14 @@ func test(t *testing.T, expected, actual map[string]interface{}, description str delete(actual, "occurred_at") } - if expected["channels"] != nil || actual["channels"] != nil { - ech := expected["channels"] - ach := actual["channels"] - - che := []string{} - err = json.Unmarshal([]byte(ech.(string)), &che) - require.Nil(t, err, fmt.Sprintf("%s: expected to get a valid channels, got %s", description, err)) - - cha := []string{} - err = json.Unmarshal([]byte(ach.(string)), &cha) - require.Nil(t, err, fmt.Sprintf("%s: expected to get a valid channels, got %s", description, err)) + exchs := expected["channels"].([]interface{}) + achs := actual["channels"].([]interface{}) - if assert.ElementsMatchf(t, che, cha, "%s: got incorrect channels\n", description) { - delete(expected, "channels") - delete(actual, "channels") + if exchs != nil && achs != nil { + if assert.Len(t, exchs, len(achs), fmt.Sprintf("%s: got incorrect number of channels\n", description)) { + for _, exch := range exchs { + assert.Contains(t, achs, exch, fmt.Sprintf("%s: got incorrect channel\n", description)) + } } } diff --git a/cli/config.go b/cli/config.go index 893bd1dbfa..bf9590d83c 100644 --- a/cli/config.go +++ b/cli/config.go @@ -27,6 +27,7 @@ const ( defCertsURL string = defURL + ":9019" defInvitationsURL string = defURL + ":9020" defHTTPURL string = defURL + ":8008" + defJournalURL string = defURL + ":9021" defTLSVerification bool = false defOffset string = "0" defLimit string = "10" @@ -43,6 +44,7 @@ type remotes struct { BootstrapURL string `toml:"bootstrap_url"` CertsURL string `toml:"certs_url"` InvitationsURL string `toml:"invitations_url"` + JournalURL string `toml:"journal_url"` TLSVerification bool `toml:"tls_verification"` } @@ -112,6 +114,7 @@ func ParseConfig(sdkConf mgxsdk.Config) (mgxsdk.Config, error) { BootstrapURL: defBootstrapURL, CertsURL: defCertsURL, InvitationsURL: defInvitationsURL, + JournalURL: defJournalURL, TLSVerification: defTLSVerification, }, Filter: filter{ @@ -198,6 +201,10 @@ func ParseConfig(sdkConf mgxsdk.Config) (mgxsdk.Config, error) { sdkConf.InvitationsURL = config.Remotes.InvitationsURL } + if sdkConf.JournalURL == "" && config.Remotes.JournalURL != "" { + sdkConf.JournalURL = config.Remotes.JournalURL + } + sdkConf.TLSVerification = config.Remotes.TLSVerification || sdkConf.TLSVerification return sdkConf, nil diff --git a/cli/journal.go b/cli/journal.go new file mode 100644 index 0000000000..fa659ca4c3 --- /dev/null +++ b/cli/journal.go @@ -0,0 +1,50 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + mgxsdk "github.com/absmach/magistrala/pkg/sdk/go" + "github.com/spf13/cobra" +) + +var cmdJournal = cobra.Command{ + Use: "get ", + Short: "Get journal", + Long: "Get journal\n" + + "Usage:\n" + + "\tmagistrala-cli journal get - lists journal logs\n" + + "\tmagistrala-cli journal get --offset --limit - lists journal logs with provided offset and limit\n", + Run: func(cmd *cobra.Command, args []string) { + if len(args) != 3 { + logUsage(cmd.Use) + return + } + + pageMetadata := mgxsdk.PageMetadata{ + Offset: Offset, + Limit: Limit, + } + + journal, err := sdk.Journal(args[0], args[1], pageMetadata, args[2]) + if err != nil { + logError(err) + return + } + + logJSON(journal) + }, +} + +// NewJournalCmd returns journal log command. +func NewJournalCmd() *cobra.Command { + cmd := cobra.Command{ + Use: "journal get", + Short: "journal log", + Long: `journal to read journal log`, + } + + cmd.AddCommand(&cmdJournal) + + return &cmd +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 994cb4f6f8..245c8205bb 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -12,9 +12,7 @@ import ( "github.com/spf13/cobra" ) -const ( - defURL string = "http://localhost" -) +const defURL string = "http://localhost" func main() { msgContentType := string(sdk.CTJSONSenML) @@ -52,6 +50,7 @@ func main() { subscriptionsCmd := cli.NewSubscriptionCmd() configCmd := cli.NewConfigCmd() invitationsCmd := cli.NewInvitationsCmd() + journalCmd := cli.NewJournalCmd() // Root Commands rootCmd.AddCommand(healthCmd) @@ -67,6 +66,7 @@ func main() { rootCmd.AddCommand(subscriptionsCmd) rootCmd.AddCommand(configCmd) rootCmd.AddCommand(invitationsCmd) + rootCmd.AddCommand(journalCmd) // Root Flags rootCmd.PersistentFlags().StringVarP( @@ -133,6 +133,14 @@ func main() { "Inivitations URL", ) + rootCmd.PersistentFlags().StringVarP( + &sdkConf.JournalURL, + "journal-url", + "a", + sdkConf.JournalURL, + "Journal Log URL", + ) + rootCmd.PersistentFlags().StringVarP( &sdkConf.HostURL, "host-url", diff --git a/cmd/journal/main.go b/cmd/journal/main.go new file mode 100644 index 0000000000..1ed0f44d09 --- /dev/null +++ b/cmd/journal/main.go @@ -0,0 +1,181 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main contains journal main function to start the journal service. +package main + +import ( + "context" + "fmt" + "log" + "log/slog" + "net/url" + "os" + + chclient "github.com/absmach/callhome/pkg/client" + "github.com/absmach/magistrala" + "github.com/absmach/magistrala/internal" + jaegerclient "github.com/absmach/magistrala/internal/clients/jaeger" + pgclient "github.com/absmach/magistrala/internal/clients/postgres" + "github.com/absmach/magistrala/internal/postgres" + "github.com/absmach/magistrala/internal/server" + httpserver "github.com/absmach/magistrala/internal/server/http" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/journal/api" + "github.com/absmach/magistrala/journal/events" + "github.com/absmach/magistrala/journal/middleware" + journalpg "github.com/absmach/magistrala/journal/postgres" + mglog "github.com/absmach/magistrala/logger" + "github.com/absmach/magistrala/pkg/auth" + "github.com/absmach/magistrala/pkg/events/store" + "github.com/absmach/magistrala/pkg/uuid" + "github.com/caarlos0/env/v10" + "github.com/jmoiron/sqlx" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" +) + +const ( + svcName = "journal" + envPrefixDB = "MG_JOURNAL_" + envPrefixHTTP = "MG_JOURNAL_HTTP_" + envPrefixAuth = "MG_AUTH_GRPC_" + defDB = "journal" + defSvcHTTPPort = "9021" +) + +type config struct { + LogLevel string `env:"MG_JOURNAL_LOG_LEVEL" envDefault:"info"` + ESURL string `env:"MG_ES_URL" envDefault:"nats://localhost:4222"` + JaegerURL url.URL `env:"MG_JAEGER_URL" envDefault:"http://jaeger:14268/api/traces"` + SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"` + InstanceID string `env:"MG_JOURNAL_INSTANCE_ID" envDefault:""` + TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` +} + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + g, ctx := errgroup.WithContext(ctx) + + cfg := config{} + if err := env.Parse(&cfg); err != nil { + log.Fatalf("failed to load %s configuration : %s", svcName, err) + } + + logger, err := mglog.New(os.Stdout, cfg.LogLevel) + if err != nil { + log.Fatalf("failed to init logger: %s", err) + } + + var exitCode int + defer mglog.ExitWithError(&exitCode) + + if cfg.InstanceID == "" { + if cfg.InstanceID, err = uuid.New().ID(); err != nil { + logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err)) + exitCode = 1 + return + } + } + + dbConfig := pgclient.Config{Name: defDB} + if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil { + logger.Error(err.Error()) + exitCode = 1 + return + } + db, err := pgclient.Setup(dbConfig, *journalpg.Migration()) + if err != nil { + logger.Error(err.Error()) + exitCode = 1 + return + } + defer db.Close() + + authConfig := auth.Config{} + if err := env.ParseWithOptions(&authConfig, env.Options{Prefix: envPrefixAuth}); err != nil { + logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err)) + exitCode = 1 + return + } + + ac, acHandler, err := auth.Setup(ctx, authConfig) + if err != nil { + logger.Error(err.Error()) + exitCode = 1 + return + } + defer acHandler.Close() + + logger.Info("Successfully connected to auth grpc server " + acHandler.Secure()) + + tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio) + if err != nil { + logger.Error(fmt.Sprintf("failed to init Jaeger: %s", err)) + exitCode = 1 + return + } + defer func() { + if err := tp.Shutdown(ctx); err != nil { + logger.Error(fmt.Sprintf("error shutting down tracer provider: %s", err)) + } + }() + tracer := tp.Tracer(svcName) + + svc := newService(db, dbConfig, ac, logger, tracer) + + subscriber, err := store.NewSubscriber(ctx, cfg.ESURL, logger) + if err != nil { + logger.Error(fmt.Sprintf("failed to create subscriber: %s", err)) + exitCode = 1 + return + } + + logger.Info("Subscribed to Event Store") + + if err := events.Start(ctx, svcName, subscriber, svc); err != nil { + logger.Error("failed to start %s service: %s", svcName, err) + exitCode = 1 + return + } + + httpServerConfig := server.Config{Port: defSvcHTTPPort} + if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil { + logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err.Error())) + exitCode = 1 + return + } + + hs := httpserver.New(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger, svcName, cfg.InstanceID), logger) + + if cfg.SendTelemetry { + chc := chclient.New(svcName, magistrala.Version, logger, cancel) + go chc.CallHome(ctx) + } + + g.Go(func() error { + return hs.Start() + }) + + g.Go(func() error { + return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) + }) + + if err := g.Wait(); err != nil { + logger.Error(fmt.Sprintf("%s service terminated: %s", svcName, err)) + } +} + +func newService(db *sqlx.DB, dbConfig pgclient.Config, authClient magistrala.AuthServiceClient, logger *slog.Logger, tracer trace.Tracer) journal.Service { + database := postgres.NewDatabase(db, dbConfig, tracer) + repo := journalpg.NewRepository(database) + idp := uuid.New() + + svc := journal.NewService(idp, repo, authClient) + svc = middleware.LoggingMiddleware(svc, logger) + counter, latency := internal.MakeMetrics("journal", "journal_writer") + svc = middleware.MetricsMiddleware(svc, counter, latency) + svc = middleware.Tracing(svc, tracer) + + return svc +} diff --git a/config.toml b/config.toml index f18feb5240..cfe2eb8ea8 100644 --- a/config.toml +++ b/config.toml @@ -10,6 +10,7 @@ user_token = "" topic = "" [remotes] + journal_url = "http://localhost:9021" bootstrap_url = "http://localhost:9013" certs_url = "http://localhost:9019" domains_url = "http://localhost:8189" diff --git a/docker/.env b/docker/.env index 2161902502..dce160fd70 100644 --- a/docker/.env +++ b/docker/.env @@ -122,7 +122,6 @@ MG_SPICEDB_HOST=magistrala-spicedb MG_SPICEDB_PORT=50051 MG_SPICEDB_DATASTORE_ENGINE=postgres - ### Invitations MG_INVITATIONS_LOG_LEVEL=info MG_INVITATIONS_HTTP_HOST=invitations @@ -613,6 +612,23 @@ MG_SMPP_SRC_ADDR_NPI=0 MG_SMPP_DST_ADDR_NPI=1 MG_SMPP_NOTIFIER_INSTANCE_ID= +### Journal +MG_JOURNAL_LOG_LEVEL=info +MG_JOURNAL_HTTP_HOST=journal +MG_JOURNAL_HTTP_PORT=9021 +MG_JOURNAL_HTTP_SERVER_CERT= +MG_JOURNAL_HTTP_SERVER_KEY= +MG_JOURNAL_HOST=magistrala-journal-db +MG_JOURNAL_PORT=5432 +MG_JOURNAL_USER=magistrala +MG_JOURNAL_PASS=magistrala +MG_JOURNAL_NAME=journal +MG_JOURNAL_SSL_MODE=disable +MG_JOURNAL_SSL_CERT= +MG_JOURNAL_SSL_KEY= +MG_JOURNAL_SSL_ROOT_CERT= +MG_JOURNAL_INSTANCE_ID= + ### GRAFANA and PROMETHEUS MG_PROMETHEUS_PORT=9090 MG_GRAFANA_PORT=3000 diff --git a/docker/addons/journal/docker-compose.yml b/docker/addons/journal/docker-compose.yml new file mode 100644 index 0000000000..fa51df0b1b --- /dev/null +++ b/docker/addons/journal/docker-compose.yml @@ -0,0 +1,67 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +# This docker-compose file contains optional Postgres and journal services +# for Magistrala platform. Since these are optional, this file is dependent of docker-compose file +# from /docker. In order to run these services, execute command: +# docker-compose -f docker/docker-compose.yml -f docker/addons/journal/docker-compose.yml up +# from project root. PostgreSQL default port (5432) is exposed, so you can use various tools for database +# inspection and data visualization. + +networks: + magistrala-base-net: + +volumes: + magistrala-journal-volume: + +services: + journal-db: + image: postgres:16.2-alpine + container_name: magistrala-journal-db + restart: on-failure + command: postgres -c "max_connections=${MG_POSTGRES_MAX_CONNECTIONS}" + environment: + POSTGRES_USER: ${MG_JOURNAL_USER} + POSTGRES_PASSWORD: ${MG_JOURNAL_PASS} + POSTGRES_DB: ${MG_JOURNAL_NAME} + MG_POSTGRES_MAX_CONNECTIONS: ${MG_POSTGRES_MAX_CONNECTIONS} + networks: + - magistrala-base-net + volumes: + - magistrala-journal-volume:/var/lib/postgresql/data + + journal: + image: magistrala/journal:${MG_RELEASE_TAG} + container_name: magistrala-journal + depends_on: + - journal-db + restart: on-failure + environment: + MG_JOURNAL_LOG_LEVEL: ${MG_JOURNAL_LOG_LEVEL} + MG_JOURNAL_HTTP_HOST: ${MG_JOURNAL_HTTP_HOST} + MG_JOURNAL_HTTP_PORT: ${MG_JOURNAL_HTTP_PORT} + MG_JOURNAL_HTTP_SERVER_CERT: ${MG_JOURNAL_HTTP_SERVER_CERT} + MG_JOURNAL_HTTP_SERVER_KEY: ${MG_JOURNAL_HTTP_SERVER_KEY} + MG_JOURNAL_HOST: ${MG_JOURNAL_HOST} + MG_JOURNAL_PORT: ${MG_JOURNAL_PORT} + MG_JOURNAL_USER: ${MG_JOURNAL_USER} + MG_JOURNAL_PASS: ${MG_JOURNAL_PASS} + MG_JOURNAL_NAME: ${MG_JOURNAL_NAME} + MG_JOURNAL_SSL_MODE: ${MG_JOURNAL_SSL_MODE} + MG_JOURNAL_SSL_CERT: ${MG_JOURNAL_SSL_CERT} + MG_JOURNAL_SSL_KEY: ${MG_JOURNAL_SSL_KEY} + MG_JOURNAL_SSL_ROOT_CERT: ${MG_JOURNAL_SSL_ROOT_CERT} + MG_AUTH_GRPC_URL: ${MG_AUTH_GRPC_URL} + MG_AUTH_GRPC_TIMEOUT: ${MG_AUTH_GRPC_TIMEOUT} + MG_AUTH_GRPC_CLIENT_CERT: ${MG_AUTH_GRPC_CLIENT_CERT:+/auth-grpc-client.crt} + MG_AUTH_GRPC_CLIENT_KEY: ${MG_AUTH_GRPC_CLIENT_KEY:+/auth-grpc-client.key} + MG_AUTH_GRPC_SERVER_CA_CERTS: ${MG_AUTH_GRPC_SERVER_CA_CERTS:+/auth-grpc-server-ca.crt} + MG_ES_URL: ${MG_ES_URL} + MG_JAEGER_URL: ${MG_JAEGER_URL} + MG_JAEGER_TRACE_RATIO: ${MG_JAEGER_TRACE_RATIO} + MG_SEND_TELEMETRY: ${MG_SEND_TELEMETRY} + MG_JOURNAL_INSTANCE_ID: ${MG_JOURNAL_INSTANCE_ID} + ports: + - ${MG_JOURNAL_HTTP_PORT}:${MG_JOURNAL_HTTP_PORT} + networks: + - magistrala-base-net diff --git a/internal/api/common.go b/internal/api/common.go index e14af7e559..df5b8e3a15 100644 --- a/internal/api/common.go +++ b/internal/api/common.go @@ -154,7 +154,12 @@ func EncodeError(_ context.Context, err error, w http.ResponseWriter) { errors.Contains(err, apiutil.ErrInvalidTopic), errors.Contains(err, bootstrap.ErrAddBootstrap), errors.Contains(err, apiutil.ErrInvalidCertData), - errors.Contains(err, apiutil.ErrEmptyMessage): + errors.Contains(err, apiutil.ErrEmptyMessage), + errors.Contains(err, apiutil.ErrInvalidLevel), + errors.Contains(err, apiutil.ErrInvalidDirection), + errors.Contains(err, apiutil.ErrInvalidEntityType), + errors.Contains(err, apiutil.ErrMissingEntityType), + errors.Contains(err, apiutil.ErrInvalidTimeFormat): err = unwrap(err) w.WriteHeader(http.StatusBadRequest) diff --git a/internal/apiutil/errors.go b/internal/apiutil/errors.go index 7230857ff8..ad19b6d67d 100644 --- a/internal/apiutil/errors.go +++ b/internal/apiutil/errors.go @@ -167,4 +167,13 @@ var ( // ErrEmptyMessage indicates empty message. ErrEmptyMessage = errors.New("empty message") + + // ErrMissingEntityType indicates missing entity type. + ErrMissingEntityType = errors.New("missing entity type") + + // ErrInvalidEntityType indicates invalid entity type. + ErrInvalidEntityType = errors.New("invalid entity type") + + // ErrInvalidTimeFormat indicates invalid time format i.e not unix time. + ErrInvalidTimeFormat = errors.New("invalid time format use unix time") ) diff --git a/internal/groups/events/events.go b/internal/groups/events/events.go index f35153c5c1..eb65fd411a 100644 --- a/internal/groups/events/events.go +++ b/internal/groups/events/events.go @@ -4,14 +4,13 @@ package events import ( - "encoding/json" "time" "github.com/absmach/magistrala/pkg/events" groups "github.com/absmach/magistrala/pkg/groups" ) -const ( +var ( groupPrefix = "group." groupCreate = groupPrefix + "create" groupUpdate = groupPrefix + "update" @@ -46,15 +45,13 @@ type assignEvent struct { } func (cge assignEvent) Encode() (map[string]interface{}, error) { - val := map[string]interface{}{ + return map[string]interface{}{ "operation": groupAssign, "member_ids": cge.memberIDs, "relation": cge.relation, "memberKind": cge.memberKind, "group_id": cge.groupID, - } - - return val, nil + }, nil } type unassignEvent struct { @@ -65,15 +62,13 @@ type unassignEvent struct { } func (cge unassignEvent) Encode() (map[string]interface{}, error) { - val := map[string]interface{}{ + return map[string]interface{}{ "operation": groupUnassign, "member_ids": cge.memberIDs, "relation": cge.relation, "memberKind": cge.memberKind, "group_id": cge.groupID, - } - - return val, nil + }, nil } type createGroupEvent struct { @@ -101,12 +96,7 @@ func (cge createGroupEvent) Encode() (map[string]interface{}, error) { val["description"] = cge.Description } if cge.Metadata != nil { - metadata, err := json.Marshal(cge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = cge.Metadata } if cge.Status.String() != "" { val["status"] = cge.Status.String() @@ -142,12 +132,7 @@ func (uge updateGroupEvent) Encode() (map[string]interface{}, error) { val["description"] = uge.Description } if uge.Metadata != nil { - metadata, err := json.Marshal(uge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = uge.Metadata } if !uge.CreatedAt.IsZero() { val["created_at"] = uge.CreatedAt @@ -199,12 +184,7 @@ func (vge viewGroupEvent) Encode() (map[string]interface{}, error) { val["description"] = vge.Description } if vge.Metadata != nil { - metadata, err := json.Marshal(vge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = vge.Metadata } if !vge.CreatedAt.IsZero() { val["created_at"] = vge.CreatedAt @@ -227,11 +207,10 @@ type viewGroupPermsEvent struct { } func (vgpe viewGroupPermsEvent) Encode() (map[string]interface{}, error) { - val := map[string]interface{}{ + return map[string]interface{}{ "operation": groupViewPerms, "permissions": vgpe.permissions, - } - return val, nil + }, nil } type listGroupEvent struct { @@ -256,12 +235,7 @@ func (lge listGroupEvent) Encode() (map[string]interface{}, error) { val["tag"] = lge.Tag } if lge.Metadata != nil { - metadata, err := json.Marshal(lge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lge.Metadata } if lge.Status.String() != "" { val["status"] = lge.Status.String() @@ -277,14 +251,12 @@ type listGroupMembershipEvent struct { } func (lgme listGroupMembershipEvent) Encode() (map[string]interface{}, error) { - val := map[string]interface{}{ + return map[string]interface{}{ "operation": groupListMemberships, - "group_id": lgme.groupID, + "id": lgme.groupID, "permission": lgme.permission, "member_kind": lgme.memberKind, - } - - return val, nil + }, nil } type deleteGroupEvent struct { diff --git a/journal/api/doc.go b/journal/api/doc.go new file mode 100644 index 0000000000..2424852cc4 --- /dev/null +++ b/journal/api/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package api contains API-related concerns: endpoint definitions, middlewares +// and all resource representations. +package api diff --git a/journal/api/endpoint.go b/journal/api/endpoint.go new file mode 100644 index 0000000000..3b366933a8 --- /dev/null +++ b/journal/api/endpoint.go @@ -0,0 +1,31 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "context" + + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/pkg/errors" + "github.com/go-kit/kit/endpoint" +) + +func retrieveJournalsEndpoint(svc journal.Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(retrieveJournalsReq) + if err := req.validate(); err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + + page, err := svc.RetrieveAll(ctx, req.token, req.page) + if err != nil { + return nil, err + } + + return pageRes{ + JournalsPage: page, + }, nil + } +} diff --git a/journal/api/endpoint_test.go b/journal/api/endpoint_test.go new file mode 100644 index 0000000000..3c9e3c342f --- /dev/null +++ b/journal/api/endpoint_test.go @@ -0,0 +1,282 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api_test + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/journal/api" + "github.com/absmach/magistrala/journal/mocks" + mglog "github.com/absmach/magistrala/logger" + svcerr "github.com/absmach/magistrala/pkg/errors/service" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var validToken = "valid" + +type testRequest struct { + client *http.Client + method string + url string + token string + body io.Reader +} + +func (tr testRequest) make() (*http.Response, error) { + req, err := http.NewRequest(tr.method, tr.url, tr.body) + if err != nil { + return nil, err + } + + if tr.token != "" { + req.Header.Set("Authorization", apiutil.BearerPrefix+tr.token) + } + + return tr.client.Do(req) +} + +func newjournalServer() (*httptest.Server, *mocks.Service) { + svc := new(mocks.Service) + + logger := mglog.NewMock() + mux := api.MakeHandler(svc, logger, "journal-log", "test") + return httptest.NewServer(mux), svc +} + +func TestListJournalsEndpoint(t *testing.T) { + es, svc := newjournalServer() + + cases := []struct { + desc string + token string + url string + contentType string + status int + svcErr error + }{ + { + desc: "successful", + token: validToken, + url: "/user/123", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "empty token", + token: "", + url: "/user/123", + status: http.StatusUnauthorized, + svcErr: nil, + }, + { + desc: "with service error", + token: validToken, + url: "/user/123", + status: http.StatusForbidden, + svcErr: svcerr.ErrAuthorization, + }, + { + desc: "with offset", + token: validToken, + url: "/user/123?offset=10", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid offset", + token: validToken, + url: "/user/123?offset=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with limit", + token: validToken, + url: "/user/123?limit=10", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid limit", + token: validToken, + url: "/user/123?limit=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with operation", + token: validToken, + url: "/user/123?operation=user.create", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with malformed operation", + token: validToken, + url: "/user/123?operation=user.create&operation=user.update", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with from", + token: validToken, + url: fmt.Sprintf("/user/123?from=%d", time.Now().Unix()), + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid from", + token: validToken, + url: "/user/123?from=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with invalid from as UnixNano", + token: validToken, + url: fmt.Sprintf("/user/123?from=%d", time.Now().UnixNano()), + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with to", + token: validToken, + url: fmt.Sprintf("/user/123?to=%d", time.Now().Unix()), + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid to", + token: validToken, + url: "/user/123?to=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with invalid to as UnixNano", + token: validToken, + url: fmt.Sprintf("/user/123?to=%d", time.Now().UnixNano()), + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with attributes", + token: validToken, + url: fmt.Sprintf("/user/123?with_attributes=%s", strconv.FormatBool(true)), + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid attributes", + token: validToken, + url: "/user/123?with_attributes=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with metadata", + token: validToken, + url: fmt.Sprintf("/user/123?with_metadata=%s", strconv.FormatBool(true)), + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid metadata", + token: validToken, + url: "/user/123?with_metadata=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with asc direction", + token: validToken, + url: "/user/123?dir=asc", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with desc direction", + token: validToken, + url: "/user/123?dir=desc", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with invalid direction", + token: validToken, + url: "/user/123?dir=ten", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with malformed direction", + token: validToken, + url: "/user/123?dir=invalid&dir=invalid2", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with invalid entity type", + token: validToken, + url: "/invalid/123", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with all query params", + token: validToken, + url: "/user/123?offset=10&limit=10&operation=user.create&from=0&to=10&with_attributes=true&with_metadata=true&dir=asc", + status: http.StatusOK, + svcErr: nil, + }, + { + desc: "with empty url", + token: validToken, + url: "", + status: http.StatusNotFound, + svcErr: nil, + }, + { + desc: "with empty entity type", + token: validToken, + url: "//123", + status: http.StatusBadRequest, + svcErr: nil, + }, + { + desc: "with empty entity ID", + token: validToken, + url: "/user/", + status: http.StatusNotFound, + svcErr: nil, + }, + } + + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + svcCall := svc.On("RetrieveAll", mock.Anything, c.token, mock.Anything).Return(journal.JournalsPage{}, c.svcErr) + req := testRequest{ + client: es.Client(), + method: http.MethodGet, + url: es.URL + "/journal" + c.url, + token: c.token, + } + + resp, err := req.make() + assert.Nil(t, err, c.desc) + defer resp.Body.Close() + assert.Equal(t, c.status, resp.StatusCode, c.desc) + svcCall.Unset() + }) + } +} diff --git a/journal/api/requests.go b/journal/api/requests.go new file mode 100644 index 0000000000..e42d274a3d --- /dev/null +++ b/journal/api/requests.go @@ -0,0 +1,32 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "github.com/absmach/magistrala/internal/api" + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" +) + +type retrieveJournalsReq struct { + token string + page journal.Page +} + +func (req retrieveJournalsReq) validate() error { + if req.token == "" { + return apiutil.ErrBearerToken + } + if req.page.Limit > api.DefLimit { + return apiutil.ErrLimitSize + } + if req.page.Direction != "" && req.page.Direction != api.AscDir && req.page.Direction != api.DescDir { + return apiutil.ErrInvalidDirection + } + if req.page.EntityID == "" { + return apiutil.ErrMissingID + } + + return nil +} diff --git a/journal/api/requests_test.go b/journal/api/requests_test.go new file mode 100644 index 0000000000..e378808343 --- /dev/null +++ b/journal/api/requests_test.go @@ -0,0 +1,126 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "testing" + + "github.com/absmach/magistrala/internal/api" + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" + "github.com/stretchr/testify/assert" +) + +var ( + token = "token" + limit uint64 = 10 +) + +func TestRetrieveJournalsReqValidate(t *testing.T) { + cases := []struct { + desc string + req retrieveJournalsReq + err error + }{ + { + desc: "valid", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + EntityID: "id", + EntityType: journal.UserEntity, + }, + }, + err: nil, + }, + { + desc: "missing token", + req: retrieveJournalsReq{ + page: journal.Page{ + Limit: limit, + EntityID: "id", + EntityType: journal.UserEntity, + }, + }, + err: apiutil.ErrBearerToken, + }, + { + desc: "invalid limit size", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: api.DefLimit + 1, + EntityID: "id", + EntityType: journal.UserEntity, + }, + }, + err: apiutil.ErrLimitSize, + }, + { + desc: "invalid sorting direction", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + Direction: "invalid", + EntityID: "id", + EntityType: journal.UserEntity, + }, + }, + err: apiutil.ErrInvalidDirection, + }, + { + desc: "valid id and entity type", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + EntityID: "id", + EntityType: journal.UserEntity, + }, + }, + err: nil, + }, + { + desc: "valid id and empty entity type", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + EntityID: "id", + }, + }, + err: nil, + }, + { + desc: "empty id and empty entity type", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + }, + }, + err: apiutil.ErrMissingID, + }, + { + desc: "empty id and valid entity type", + req: retrieveJournalsReq{ + token: token, + page: journal.Page{ + Limit: limit, + EntityType: journal.UserEntity, + }, + }, + err: apiutil.ErrMissingID, + }, + } + + for _, c := range cases { + t.Run(c.desc, func(t *testing.T) { + err := c.req.validate() + assert.Equal(t, c.err, err) + }) + } +} diff --git a/journal/api/responses.go b/journal/api/responses.go new file mode 100644 index 0000000000..81b3702cf4 --- /dev/null +++ b/journal/api/responses.go @@ -0,0 +1,29 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "net/http" + + "github.com/absmach/magistrala" + "github.com/absmach/magistrala/journal" +) + +var _ magistrala.Response = (*pageRes)(nil) + +type pageRes struct { + journal.JournalsPage `json:",inline"` +} + +func (res pageRes) Headers() map[string]string { + return map[string]string{} +} + +func (res pageRes) Code() int { + return http.StatusOK +} + +func (res pageRes) Empty() bool { + return false +} diff --git a/journal/api/transport.go b/journal/api/transport.go new file mode 100644 index 0000000000..cbbbb393f0 --- /dev/null +++ b/journal/api/transport.go @@ -0,0 +1,129 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package api + +import ( + "context" + "log/slog" + "math" + "net/http" + "strings" + "time" + + "github.com/absmach/magistrala" + "github.com/absmach/magistrala/internal/api" + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/pkg/errors" + "github.com/go-chi/chi/v5" + kithttp "github.com/go-kit/kit/transport/http" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +const ( + operationKey = "operation" + fromKey = "from" + toKey = "to" + attributesKey = "with_attributes" + metadataKey = "with_metadata" + entityIDKey = "id" + entityTypeKey = "entity_type" +) + +// MakeHandler returns a HTTP API handler with health check and metrics. +func MakeHandler(svc journal.Service, logger *slog.Logger, svcName, instanceID string) http.Handler { + opts := []kithttp.ServerOption{ + kithttp.ServerErrorEncoder(apiutil.LoggingErrorEncoder(logger, api.EncodeError)), + } + + mux := chi.NewRouter() + + mux.Get("/journal/{entityType}/{entityID}", otelhttp.NewHandler(kithttp.NewServer( + retrieveJournalsEndpoint(svc), + decodeRetrieveJournalReq, + api.EncodeResponse, + opts..., + ), "list_journals").ServeHTTP) + + mux.Get("/health", magistrala.Health(svcName, instanceID)) + mux.Handle("/metrics", promhttp.Handler()) + + return mux +} + +func decodeRetrieveJournalReq(_ context.Context, r *http.Request) (interface{}, error) { + offset, err := apiutil.ReadNumQuery[uint64](r, api.OffsetKey, api.DefOffset) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + limit, err := apiutil.ReadNumQuery[uint64](r, api.LimitKey, api.DefLimit) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + operation, err := apiutil.ReadStringQuery(r, operationKey, "") + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + from, err := apiutil.ReadNumQuery[int64](r, fromKey, 0) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + if from > math.MaxInt32 { + return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidTimeFormat) + } + var fromTime time.Time + if from != 0 { + fromTime = time.Unix(from, 0) + } + to, err := apiutil.ReadNumQuery[int64](r, toKey, 0) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + if to > math.MaxInt32 { + return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidTimeFormat) + } + var toTime time.Time + if to != 0 { + toTime = time.Unix(to, 0) + } + attributes, err := apiutil.ReadBoolQuery(r, attributesKey, false) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + metadata, err := apiutil.ReadBoolQuery(r, metadataKey, false) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + dir, err := apiutil.ReadStringQuery(r, api.DirKey, api.DescDir) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + + entityType, err := journal.ToEntityType(chi.URLParam(r, "entityType")) + if err != nil { + return nil, errors.Wrap(apiutil.ErrValidation, err) + } + if entityType == journal.ChannelEntity { + operation = strings.ReplaceAll(operation, "channel", "group") + } + + req := retrieveJournalsReq{ + token: apiutil.ExtractBearerToken(r), + page: journal.Page{ + Offset: offset, + Limit: limit, + Operation: operation, + From: fromTime, + To: toTime, + WithAttributes: attributes, + WithMetadata: metadata, + EntityID: chi.URLParam(r, "entityID"), + EntityType: entityType, + Direction: dir, + }, + } + + return req, nil +} diff --git a/journal/doc.go b/journal/doc.go new file mode 100644 index 0000000000..3b6860678e --- /dev/null +++ b/journal/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package journal contains the journal service. +// This service is responsible for storing events from the event store to a +// journal log repository. It is also responsible for providing a REST API to query events. +package journal diff --git a/journal/events/consumer.go b/journal/events/consumer.go new file mode 100644 index 0000000000..e2636ed7f2 --- /dev/null +++ b/journal/events/consumer.go @@ -0,0 +1,85 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package events + +import ( + "context" + "errors" + "time" + + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/pkg/events" + "github.com/absmach/magistrala/pkg/events/store" +) + +var ErrMissingOccurredAt = errors.New("missing occurred_at") + +// Start method starts consuming messages received from Event store. +func Start(ctx context.Context, consumer string, sub events.Subscriber, service journal.Service) error { + subCfg := events.SubscriberConfig{ + Consumer: consumer, + Stream: store.StreamAllEvents, + Handler: Handle(service), + } + + return sub.Subscribe(ctx, subCfg) +} + +func Handle(service journal.Service) handleFunc { + return func(ctx context.Context, event events.Event) error { + data, err := event.Encode() + if err != nil { + return err + } + + operation, ok := data["operation"].(string) + if !ok { + return errors.New("missing operation") + } + delete(data, "operation") + + if operation == "" { + return errors.New("missing operation") + } + + occurredAt, ok := data["occurred_at"].(float64) + if !ok { + return ErrMissingOccurredAt + } + delete(data, "occurred_at") + + if occurredAt == 0 { + return ErrMissingOccurredAt + } + + metadata, ok := data["metadata"].(map[string]interface{}) + if !ok { + metadata = make(map[string]interface{}) + } + delete(data, "metadata") + + if len(data) == 0 { + return errors.New("missing attributes") + } + + j := journal.Journal{ + Operation: operation, + OccurredAt: time.Unix(0, int64(occurredAt)), + Attributes: data, + Metadata: metadata, + } + + return service.Save(ctx, j) + } +} + +type handleFunc func(ctx context.Context, event events.Event) error + +func (h handleFunc) Handle(ctx context.Context, event events.Event) error { + return h(ctx, event) +} + +func (h handleFunc) Cancel() error { + return nil +} diff --git a/journal/events/consumer_test.go b/journal/events/consumer_test.go new file mode 100644 index 0000000000..8915e61c6d --- /dev/null +++ b/journal/events/consumer_test.go @@ -0,0 +1,276 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package events_test + +import ( + "context" + "encoding/json" + "errors" + "math/rand" + "strings" + "testing" + "time" + + "github.com/absmach/magistrala/internal/testsutil" + "github.com/absmach/magistrala/journal" + aevents "github.com/absmach/magistrala/journal/events" + "github.com/absmach/magistrala/journal/mocks" + repoerr "github.com/absmach/magistrala/pkg/errors/repository" + "github.com/absmach/magistrala/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + operation = "users.create" + payload = map[string]interface{}{ + "temperature": rand.Float64(), + "humidity": float64(rand.Intn(1000)), + "locations": []interface{}{ + strings.Repeat("a", 100), + strings.Repeat("a", 100), + }, + "status": "active", + } + idProvider = uuid.New() +) + +type testEvent struct { + data map[string]interface{} + err error +} + +func (e testEvent) Encode() (map[string]interface{}, error) { + return e.data, e.err +} + +func NewTestEvent(data map[string]interface{}, err error) testEvent { + return testEvent{data: data, err: err} +} + +func TestHandle(t *testing.T) { + repo := new(mocks.Repository) + svc := journal.NewService(idProvider, repo, nil) + + cases := []struct { + desc string + event map[string]interface{} + encodeErr error + repoErr error + err error + }{ + { + desc: "success", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: nil, + }, + { + desc: "with encode error", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + encodeErr: errors.New("encode error"), + err: errors.New("encode error"), + }, + { + desc: "with missing operation", + event: map[string]interface{}{ + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: errors.New("missing operation"), + }, + { + desc: "with empty operation", + event: map[string]interface{}{ + "operation": "", + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: errors.New("missing operation"), + }, + { + desc: "with invalid operation", + event: map[string]interface{}{ + "operation": 1, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: errors.New("missing operation"), + }, + { + desc: "with missing occurred_at", + event: map[string]interface{}{ + "operation": operation, + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: aevents.ErrMissingOccurredAt, + }, + { + desc: "with empty occurred_at", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(0), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: aevents.ErrMissingOccurredAt, + }, + { + desc: "with invalid occurred_at", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": "invalid", + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + err: aevents.ErrMissingOccurredAt, + }, + { + desc: "with missing metadata", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + }, + err: nil, + }, + { + desc: "with empty metadata", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": map[string]interface{}{}, + }, + err: nil, + }, + { + desc: "with invalid metadata", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": 1, + }, + err: nil, + }, + { + desc: "with missing attributes", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "metadata": payload, + }, + err: errors.New("missing attributes"), + }, + { + desc: "with empty attributes", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": "", + "tags": []interface{}{}, + "number": float64(0), + "metadata": payload, + }, + err: nil, + }, + { + desc: "with invalid attributes", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + "nested": map[string]interface{}{ + "key": float64(rand.Intn(1000)), + }, + }, + }, + }, + }, + }, + "metadata": payload, + }, + err: nil, + }, + { + desc: "success", + event: map[string]interface{}{ + "operation": operation, + "occurred_at": float64(time.Now().UnixNano()), + "id": testsutil.GenerateUUID(t), + "tags": []interface{}{testsutil.GenerateUUID(t), testsutil.GenerateUUID(t)}, + "number": float64(rand.Intn(1000)), + "metadata": payload, + }, + repoErr: repoerr.ErrCreateEntity, + err: repoerr.ErrCreateEntity, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + data, err := json.Marshal(tc.event) + assert.NoError(t, err) + + event := map[string]interface{}{} + err = json.Unmarshal(data, &event) + assert.NoError(t, err) + + repoCall := repo.On("Save", context.Background(), mock.Anything).Return(tc.repoErr) + err = aevents.Handle(svc)(context.Background(), NewTestEvent(event, tc.encodeErr)) + switch { + case tc.err == nil: + assert.NoError(t, err) + default: + assert.ErrorContains(t, err, tc.err.Error()) + } + repoCall.Unset() + }) + } +} diff --git a/journal/events/doc.go b/journal/events/doc.go new file mode 100644 index 0000000000..5023696f86 --- /dev/null +++ b/journal/events/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package events provides the event consumer for the journal service. +// This package is responsible for consuming events from the event store and +// processing them. +package events diff --git a/journal/journal.go b/journal/journal.go new file mode 100644 index 0000000000..7eab106daf --- /dev/null +++ b/journal/journal.go @@ -0,0 +1,158 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package journal + +import ( + "context" + "encoding/json" + "time" + + "github.com/absmach/magistrala/auth" + "github.com/absmach/magistrala/internal/apiutil" +) + +type EntityType uint8 + +const ( + UserEntity EntityType = iota + GroupEntity + ThingEntity + ChannelEntity +) + +// String representation of the possible entity type values. +const ( + userEntityType = "user" + groupEntityType = "group" + thingEntityType = "thing" + channelEntityType = "channel" +) + +// String converts entity type to string literal. +func (e EntityType) String() string { + switch e { + case UserEntity: + return userEntityType + case GroupEntity: + return groupEntityType + case ThingEntity: + return thingEntityType + case ChannelEntity: + return channelEntityType + default: + return "" + } +} + +// AuthString returns the entity type as a string for authorization. +func (e EntityType) AuthString() string { + switch e { + case UserEntity: + return auth.UserType + case GroupEntity, ChannelEntity: + return auth.GroupType + case ThingEntity: + return auth.ThingType + default: + return "" + } +} + +// ToEntityType converts string value to a valid entity type. +func ToEntityType(entityType string) (EntityType, error) { + switch entityType { + case userEntityType: + return UserEntity, nil + case groupEntityType: + return GroupEntity, nil + case thingEntityType: + return ThingEntity, nil + case channelEntityType: + return ChannelEntity, nil + default: + return EntityType(0), apiutil.ErrInvalidEntityType + } +} + +// Query returns the SQL condition for the entity type. +func (e EntityType) Query() string { + switch e { + case UserEntity: + return "((operation LIKE 'user.%' AND attributes->>'id' = :entity_id) OR (attributes->>'user_id' = :entity_id))" + case GroupEntity, ChannelEntity: + return "((operation LIKE 'group.%' AND attributes->>'id' = :entity_id) OR (attributes->>'group_id' = :entity_id))" + case ThingEntity: + return "((operation LIKE 'thing.%' AND attributes->>'id' = :entity_id) OR (attributes->>'thing_id' = :entity_id))" + default: + return "" + } +} + +// Journal represents an event journal that occurred in the system. +type Journal struct { + ID string `json:"id,omitempty" db:"id"` + Operation string `json:"operation,omitempty" db:"operation,omitempty"` + OccurredAt time.Time `json:"occurred_at,omitempty" db:"occurred_at,omitempty"` + Attributes map[string]interface{} `json:"attributes,omitempty" db:"attributes,omitempty"` // This is extra information about the journal for example thing_id, user_id, group_id etc. + Metadata map[string]interface{} `json:"metadata,omitempty" db:"metadata,omitempty"` // This is decoded metadata from the journal. +} + +// JournalsPage represents a page of journals. +type JournalsPage struct { + Total uint64 `json:"total"` + Offset uint64 `json:"offset"` + Limit uint64 `json:"limit"` + Journals []Journal `json:"journals"` +} + +// Page is used to filter journals. +type Page struct { + Offset uint64 `json:"offset" db:"offset"` + Limit uint64 `json:"limit" db:"limit"` + Operation string `json:"operation,omitempty" db:"operation,omitempty"` + From time.Time `json:"from,omitempty" db:"from,omitempty"` + To time.Time `json:"to,omitempty" db:"to,omitempty"` + WithAttributes bool `json:"with_attributes,omitempty"` + WithMetadata bool `json:"with_metadata,omitempty"` + EntityID string `json:"entity_id,omitempty" db:"entity_id,omitempty"` + EntityType EntityType `json:"entity_type,omitempty" db:"entity_type,omitempty"` + Direction string `json:"direction,omitempty"` +} + +func (page JournalsPage) MarshalJSON() ([]byte, error) { + type Alias JournalsPage + a := struct { + Alias + }{ + Alias: Alias(page), + } + + if a.Journals == nil { + a.Journals = make([]Journal, 0) + } + + return json.Marshal(a) +} + +// Service provides access to the journal log service. +// +//go:generate mockery --name Service --output=./mocks --filename service.go --quiet --note "Copyright (c) Abstract Machines" +type Service interface { + // Save saves the journal to the database. + Save(ctx context.Context, journal Journal) error + + // RetrieveAll retrieves all journals from the database with the given page. + RetrieveAll(ctx context.Context, token string, page Page) (JournalsPage, error) +} + +// Repository provides access to the journal log database. +// +//go:generate mockery --name Repository --output=./mocks --filename repository.go --quiet --note "Copyright (c) Abstract Machines" +type Repository interface { + // Save persists the journal to a database. + Save(ctx context.Context, journal Journal) error + + // RetrieveAll retrieves all journals from the database with the given page. + RetrieveAll(ctx context.Context, page Page) (JournalsPage, error) +} diff --git a/journal/journal_test.go b/journal/journal_test.go new file mode 100644 index 0000000000..f0d8980edf --- /dev/null +++ b/journal/journal_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package journal_test + +import ( + "fmt" + "testing" + "time" + + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/journal" + "github.com/stretchr/testify/assert" +) + +func TestJournalsPage_MarshalJSON(t *testing.T) { + occurredAt := time.Now() + + cases := []struct { + desc string + page journal.JournalsPage + res string + }{ + { + desc: "empty page", + page: journal.JournalsPage{ + Journals: []journal.Journal(nil), + }, + res: `{"total":0,"offset":0,"limit":0,"journals":[]}`, + }, + { + desc: "page with journals", + page: journal.JournalsPage{ + Total: 1, + Offset: 0, + Limit: 0, + Journals: []journal.Journal{ + { + Operation: "123", + OccurredAt: occurredAt, + Attributes: map[string]interface{}{"123": "123"}, + Metadata: map[string]interface{}{"123": "123"}, + }, + }, + }, + res: fmt.Sprintf(`{"total":1,"offset":0,"limit":0,"journals":[{"operation":"123","occurred_at":"%s","attributes":{"123":"123"},"metadata":{"123":"123"}}]}`, occurredAt.Format(time.RFC3339Nano)), + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + data, err := tc.page.MarshalJSON() + assert.NoError(t, err, "Unexpected error: %v", err) + assert.Equal(t, tc.res, string(data)) + }) + } +} + +func TestEntityType(t *testing.T) { + cases := []struct { + desc string + e journal.EntityType + str string + authString string + queryString string + }{ + { + desc: "UserEntity", + e: journal.UserEntity, + str: "user", + authString: "user", + }, + { + desc: "ThingEntity", + e: journal.ThingEntity, + str: "thing", + authString: "thing", + }, + { + desc: "GroupEntity", + e: journal.GroupEntity, + str: "group", + authString: "group", + }, + { + desc: "ChannelEntity", + e: journal.ChannelEntity, + str: "channel", + authString: "group", + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + assert.Equal(t, tc.str, tc.e.String()) + assert.Equal(t, tc.authString, tc.e.AuthString()) + assert.NotEmpty(t, tc.e.Query()) + }) + } +} + +func TestToEntityType(t *testing.T) { + cases := []struct { + desc string + entityType string + expected journal.EntityType + expectedErr error + }{ + { + desc: "UserEntity", + entityType: "user", + expected: journal.UserEntity, + }, + { + desc: "ThingEntity", + entityType: "thing", + expected: journal.ThingEntity, + }, + { + desc: "GroupEntity", + entityType: "group", + expected: journal.GroupEntity, + }, + { + desc: "ChannelEntity", + entityType: "channel", + expected: journal.ChannelEntity, + }, + { + desc: "Invalid entity type", + entityType: "invalid", + expectedErr: apiutil.ErrInvalidEntityType, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + entityType, err := journal.ToEntityType(tc.entityType) + assert.Equal(t, tc.expected, entityType) + assert.Equal(t, tc.expectedErr, err) + }) + } +} diff --git a/journal/middleware/doc.go b/journal/middleware/doc.go new file mode 100644 index 0000000000..71d2571337 --- /dev/null +++ b/journal/middleware/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package middleware provides middleware for the journal service. +// This is logging, metrics, and tracing middleware. +package middleware diff --git a/journal/middleware/logging.go b/journal/middleware/logging.go new file mode 100644 index 0000000000..5ab991a672 --- /dev/null +++ b/journal/middleware/logging.go @@ -0,0 +1,70 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "context" + "log/slog" + "time" + + "github.com/absmach/magistrala/journal" +) + +var _ journal.Service = (*loggingMiddleware)(nil) + +type loggingMiddleware struct { + logger *slog.Logger + service journal.Service +} + +// LoggingMiddleware adds logging facilities to the adapter. +func LoggingMiddleware(service journal.Service, logger *slog.Logger) journal.Service { + return &loggingMiddleware{ + logger: logger, + service: service, + } +} + +func (lm *loggingMiddleware) Save(ctx context.Context, j journal.Journal) (err error) { + defer func(begin time.Time) { + args := []any{ + slog.String("duration", time.Since(begin).String()), + slog.Group("journal", + slog.String("occurred_at", j.OccurredAt.Format(time.RFC3339Nano)), + slog.String("operation", j.Operation), + ), + } + if err != nil { + args = append(args, slog.Any("error", err)) + lm.logger.Warn("Save journal failed", args...) + return + } + lm.logger.Info("Save journal completed successfully", args...) + }(time.Now()) + + return lm.service.Save(ctx, j) +} + +func (lm *loggingMiddleware) RetrieveAll(ctx context.Context, token string, page journal.Page) (journalsPage journal.JournalsPage, err error) { + defer func(begin time.Time) { + args := []any{ + slog.String("duration", time.Since(begin).String()), + slog.Group("page", + slog.String("operation", page.Operation), + slog.String("entity_type", page.EntityType.String()), + slog.Uint64("offset", page.Offset), + slog.Uint64("limit", page.Limit), + slog.Uint64("total", journalsPage.Total), + ), + } + if err != nil { + args = append(args, slog.Any("error", err)) + lm.logger.Warn("Retrieve all journals failed", args...) + return + } + lm.logger.Info("Retrieve all journals completed successfully", args...) + }(time.Now()) + + return lm.service.RetrieveAll(ctx, token, page) +} diff --git a/journal/middleware/metrics.go b/journal/middleware/metrics.go new file mode 100644 index 0000000000..fdd098d9d3 --- /dev/null +++ b/journal/middleware/metrics.go @@ -0,0 +1,48 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "context" + "time" + + "github.com/absmach/magistrala/journal" + "github.com/go-kit/kit/metrics" +) + +var _ journal.Service = (*metricsMiddleware)(nil) + +type metricsMiddleware struct { + counter metrics.Counter + latency metrics.Histogram + service journal.Service +} + +// MetricsMiddleware returns new message repository +// with Save method wrapped to expose metrics. +func MetricsMiddleware(service journal.Service, counter metrics.Counter, latency metrics.Histogram) journal.Service { + return &metricsMiddleware{ + counter: counter, + latency: latency, + service: service, + } +} + +func (mm *metricsMiddleware) Save(ctx context.Context, j journal.Journal) error { + defer func(begin time.Time) { + mm.counter.With("method", "save").Add(1) + mm.latency.With("method", "save").Observe(time.Since(begin).Seconds()) + }(time.Now()) + + return mm.service.Save(ctx, j) +} + +func (mm *metricsMiddleware) RetrieveAll(ctx context.Context, token string, page journal.Page) (journal.JournalsPage, error) { + defer func(begin time.Time) { + mm.counter.With("method", "retrieve_all").Add(1) + mm.latency.With("method", "retrieve_all").Observe(time.Since(begin).Seconds()) + }(time.Now()) + + return mm.service.RetrieveAll(ctx, token, page) +} diff --git a/journal/middleware/tracing.go b/journal/middleware/tracing.go new file mode 100644 index 0000000000..9ea96ff91f --- /dev/null +++ b/journal/middleware/tracing.go @@ -0,0 +1,46 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "context" + + "github.com/absmach/magistrala/journal" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var _ journal.Service = (*tracing)(nil) + +type tracing struct { + tracer trace.Tracer + svc journal.Service +} + +func Tracing(svc journal.Service, tracer trace.Tracer) journal.Service { + return &tracing{tracer, svc} +} + +func (tm *tracing) Save(ctx context.Context, j journal.Journal) error { + ctx, span := tm.tracer.Start(ctx, "save", trace.WithAttributes( + attribute.String("occurred_at", j.OccurredAt.String()), + attribute.String("operation", j.Operation), + )) + defer span.End() + + return tm.svc.Save(ctx, j) +} + +func (tm *tracing) RetrieveAll(ctx context.Context, token string, page journal.Page) (resp journal.JournalsPage, err error) { + ctx, span := tm.tracer.Start(ctx, "retrieve_all", trace.WithAttributes( + attribute.Int64("offset", int64(page.Offset)), + attribute.Int64("limit", int64(page.Limit)), + attribute.Int64("total", int64(resp.Total)), + attribute.String("entity_type", page.EntityType.String()), + attribute.String("operation", page.Operation), + )) + defer span.End() + + return tm.svc.RetrieveAll(ctx, token, page) +} diff --git a/journal/mocks/doc.go b/journal/mocks/doc.go new file mode 100644 index 0000000000..16ed198afd --- /dev/null +++ b/journal/mocks/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package mocks contains mocks for testing purposes. +package mocks diff --git a/journal/mocks/repository.go b/journal/mocks/repository.go new file mode 100644 index 0000000000..8b3fb51294 --- /dev/null +++ b/journal/mocks/repository.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +// Copyright (c) Abstract Machines + +package mocks + +import ( + context "context" + + journal "github.com/absmach/magistrala/journal" + mock "github.com/stretchr/testify/mock" +) + +// Repository is an autogenerated mock type for the Repository type +type Repository struct { + mock.Mock +} + +// RetrieveAll provides a mock function with given fields: ctx, page +func (_m *Repository) RetrieveAll(ctx context.Context, page journal.Page) (journal.JournalsPage, error) { + ret := _m.Called(ctx, page) + + if len(ret) == 0 { + panic("no return value specified for RetrieveAll") + } + + var r0 journal.JournalsPage + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, journal.Page) (journal.JournalsPage, error)); ok { + return rf(ctx, page) + } + if rf, ok := ret.Get(0).(func(context.Context, journal.Page) journal.JournalsPage); ok { + r0 = rf(ctx, page) + } else { + r0 = ret.Get(0).(journal.JournalsPage) + } + + if rf, ok := ret.Get(1).(func(context.Context, journal.Page) error); ok { + r1 = rf(ctx, page) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: ctx, _a1 +func (_m *Repository) Save(ctx context.Context, _a1 journal.Journal) error { + ret := _m.Called(ctx, _a1) + + if len(ret) == 0 { + panic("no return value specified for Save") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, journal.Journal) error); ok { + r0 = rf(ctx, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewRepository creates a new instance of Repository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewRepository(t interface { + mock.TestingT + Cleanup(func()) +}) *Repository { + mock := &Repository{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/journal/mocks/service.go b/journal/mocks/service.go new file mode 100644 index 0000000000..ac7c34c114 --- /dev/null +++ b/journal/mocks/service.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +// Copyright (c) Abstract Machines + +package mocks + +import ( + context "context" + + journal "github.com/absmach/magistrala/journal" + mock "github.com/stretchr/testify/mock" +) + +// Service is an autogenerated mock type for the Service type +type Service struct { + mock.Mock +} + +// RetrieveAll provides a mock function with given fields: ctx, token, page +func (_m *Service) RetrieveAll(ctx context.Context, token string, page journal.Page) (journal.JournalsPage, error) { + ret := _m.Called(ctx, token, page) + + if len(ret) == 0 { + panic("no return value specified for RetrieveAll") + } + + var r0 journal.JournalsPage + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, journal.Page) (journal.JournalsPage, error)); ok { + return rf(ctx, token, page) + } + if rf, ok := ret.Get(0).(func(context.Context, string, journal.Page) journal.JournalsPage); ok { + r0 = rf(ctx, token, page) + } else { + r0 = ret.Get(0).(journal.JournalsPage) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, journal.Page) error); ok { + r1 = rf(ctx, token, page) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Save provides a mock function with given fields: ctx, _a1 +func (_m *Service) Save(ctx context.Context, _a1 journal.Journal) error { + ret := _m.Called(ctx, _a1) + + if len(ret) == 0 { + panic("no return value specified for Save") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, journal.Journal) error); ok { + r0 = rf(ctx, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewService(t interface { + mock.TestingT + Cleanup(func()) +}) *Service { + mock := &Service{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/journal/postgres/doc.go b/journal/postgres/doc.go new file mode 100644 index 0000000000..1007b3120d --- /dev/null +++ b/journal/postgres/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package postgres provides a postgres implementation of the journal log repository. +package postgres diff --git a/journal/postgres/init.go b/journal/postgres/init.go new file mode 100644 index 0000000000..adad7979c4 --- /dev/null +++ b/journal/postgres/init.go @@ -0,0 +1,36 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package postgres + +import ( + _ "github.com/jackc/pgx/v5/stdlib" // required for SQL access + migrate "github.com/rubenv/sql-migrate" +) + +func Migration() *migrate.MemoryMigrationSource { + return &migrate.MemoryMigrationSource{ + Migrations: []*migrate.Migration{ + { + Id: "journal_01", + Up: []string{ + `CREATE TABLE IF NOT EXISTS journal ( + id VARCHAR(36) PRIMARY KEY, + operation VARCHAR NOT NULL, + occurred_at TIMESTAMP NOT NULL, + attributes JSONB NOT NULL, + metadata JSONB, + UNIQUE(operation, occurred_at, attributes) + )`, + `CREATE INDEX idx_journal_default_user_filter ON journal(operation, (attributes->>'id'), (attributes->>'user_id'), occurred_at DESC);`, + `CREATE INDEX idx_journal_default_group_filter ON journal(operation, (attributes->>'id'), (attributes->>'group_id'), occurred_at DESC);`, + `CREATE INDEX idx_journal_default_thing_filter ON journal(operation, (attributes->>'id'), (attributes->>'thing_id'), occurred_at DESC);`, + `CREATE INDEX idx_journal_default_channel_filter ON journal(operation, (attributes->>'id'), (attributes->>'channel_id'), occurred_at DESC);`, + }, + Down: []string{ + `DROP TABLE IF EXISTS journal`, + }, + }, + }, + } +} diff --git a/journal/postgres/journal.go b/journal/postgres/journal.go new file mode 100644 index 0000000000..0469a84c93 --- /dev/null +++ b/journal/postgres/journal.go @@ -0,0 +1,178 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package postgres + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/absmach/magistrala/internal/postgres" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/pkg/errors" + repoerr "github.com/absmach/magistrala/pkg/errors/repository" +) + +type repository struct { + db postgres.Database +} + +func NewRepository(db postgres.Database) journal.Repository { + return &repository{db: db} +} + +func (repo *repository) Save(ctx context.Context, j journal.Journal) (err error) { + q := `INSERT INTO journal (id, operation, occurred_at, attributes, metadata) + VALUES (:id, :operation, :occurred_at, :attributes, :metadata);` + + dbJournal, err := toDBJournal(j) + if err != nil { + return errors.Wrap(repoerr.ErrCreateEntity, err) + } + + if _, err = repo.db.NamedExecContext(ctx, q, dbJournal); err != nil { + return postgres.HandleError(repoerr.ErrCreateEntity, err) + } + + return nil +} + +func (repo *repository) RetrieveAll(ctx context.Context, page journal.Page) (journal.JournalsPage, error) { + query := pageQuery(page) + + sq := "operation, occurred_at" + if page.WithAttributes { + sq += ", attributes" + } + if page.WithMetadata { + sq += ", metadata" + } + if page.Direction == "" { + page.Direction = "ASC" + } + q := fmt.Sprintf("SELECT %s FROM journal %s ORDER BY occurred_at %s LIMIT :limit OFFSET :offset;", sq, query, page.Direction) + + rows, err := repo.db.NamedQueryContext(ctx, q, page) + if err != nil { + return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err) + } + defer rows.Close() + + var items []journal.Journal + for rows.Next() { + var item dbJournal + if err = rows.StructScan(&item); err != nil { + return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err) + } + j, err := toJournal(item) + if err != nil { + return journal.JournalsPage{}, err + } + items = append(items, j) + } + + tq := fmt.Sprintf(`SELECT COUNT(*) FROM journal %s;`, query) + + total, err := postgres.Total(ctx, repo.db, tq, page) + if err != nil { + return journal.JournalsPage{}, postgres.HandleError(repoerr.ErrViewEntity, err) + } + + journalsPage := journal.JournalsPage{ + Total: total, + Offset: page.Offset, + Limit: page.Limit, + Journals: items, + } + + return journalsPage, nil +} + +func pageQuery(pm journal.Page) string { + var query []string + var emq string + if pm.Operation != "" { + query = append(query, "operation = :operation") + } + if !pm.From.IsZero() { + query = append(query, "occurred_at >= :from") + } + if !pm.To.IsZero() { + query = append(query, "occurred_at <= :to") + } + if pm.EntityID != "" { + query = append(query, pm.EntityType.Query()) + } + + if len(query) > 0 { + emq = fmt.Sprintf("WHERE %s", strings.Join(query, " AND ")) + } + + return emq +} + +type dbJournal struct { + ID string `db:"id"` + Operation string `db:"operation"` + OccurredAt time.Time `db:"occurred_at"` + Attributes []byte `db:"attributes"` + Metadata []byte `db:"metadata"` +} + +func toDBJournal(j journal.Journal) (dbJournal, error) { + if j.OccurredAt.IsZero() { + j.OccurredAt = time.Now() + } + + attributes := []byte("{}") + if len(j.Attributes) > 0 { + b, err := json.Marshal(j.Attributes) + if err != nil { + return dbJournal{}, errors.Wrap(repoerr.ErrMalformedEntity, err) + } + attributes = b + } + + metadata := []byte("{}") + if len(j.Metadata) > 0 { + b, err := json.Marshal(j.Metadata) + if err != nil { + return dbJournal{}, errors.Wrap(repoerr.ErrMalformedEntity, err) + } + metadata = b + } + + return dbJournal{ + ID: j.ID, + Operation: j.Operation, + OccurredAt: j.OccurredAt, + Attributes: attributes, + Metadata: metadata, + }, nil +} + +func toJournal(dbj dbJournal) (journal.Journal, error) { + var attributes map[string]interface{} + if dbj.Attributes != nil { + if err := json.Unmarshal(dbj.Attributes, &attributes); err != nil { + return journal.Journal{}, errors.Wrap(repoerr.ErrMalformedEntity, err) + } + } + + var metadata map[string]interface{} + if dbj.Metadata != nil { + if err := json.Unmarshal(dbj.Metadata, &metadata); err != nil { + return journal.Journal{}, errors.Wrap(repoerr.ErrMalformedEntity, err) + } + } + + return journal.Journal{ + Operation: dbj.Operation, + OccurredAt: dbj.OccurredAt, + Attributes: attributes, + Metadata: metadata, + }, nil +} diff --git a/journal/postgres/journal_test.go b/journal/postgres/journal_test.go new file mode 100644 index 0000000000..677d38bc25 --- /dev/null +++ b/journal/postgres/journal_test.go @@ -0,0 +1,724 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package postgres_test + +import ( + "context" + "fmt" + "math/rand" + "sort" + "strings" + "testing" + "time" + + "github.com/absmach/magistrala/internal/testsutil" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/journal/postgres" + "github.com/absmach/magistrala/pkg/errors" + repoerr "github.com/absmach/magistrala/pkg/errors/repository" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + operation = "user.create" + payload = map[string]interface{}{ + "temperature": rand.Float64(), + "humidity": float64(rand.Intn(1000)), + "locations": []interface{}{ + strings.Repeat("a", 100), + strings.Repeat("a", 100), + }, + "status": "active", + "nested": map[string]interface{}{ + "nested": map[string]interface{}{ + "nested": map[string]interface{}{ + "nested": map[string]interface{}{ + "key": "value", + }, + }, + }, + }, + } + + entityID = testsutil.GenerateUUID(&testing.T{}) + thingOperation = "thing.create" + thingAttributesV1 = map[string]interface{}{ + "id": entityID, + "status": "enabled", + "created_at": time.Now().Add(-time.Hour), + "name": "thing", + "tags": []interface{}{"tag1", "tag2"}, + "domain": testsutil.GenerateUUID(&testing.T{}), + "metadata": payload, + "identity": testsutil.GenerateUUID(&testing.T{}), + } + thingAttributesV2 = map[string]interface{}{ + "thing_id": entityID, + "metadata": payload, + } + userAttributesV1 = map[string]interface{}{ + "id": entityID, + "status": "enabled", + "created_at": time.Now().Add(-time.Hour), + "name": "user", + "tags": []interface{}{"tag1", "tag2"}, + "domain": testsutil.GenerateUUID(&testing.T{}), + "metadata": payload, + "identity": testsutil.GenerateUUID(&testing.T{}), + } + userAttributesV2 = map[string]interface{}{ + "user_id": entityID, + "metadata": payload, + } +) + +func TestJournalSave(t *testing.T) { + t.Cleanup(func() { + _, err := db.Exec("DELETE FROM journal") + require.Nil(t, err, fmt.Sprintf("clean journal unexpected error: %s", err)) + }) + repo := postgres.NewRepository(database) + + occurredAt := time.Now() + id := testsutil.GenerateUUID(t) + + cases := []struct { + desc string + journal journal.Journal + err error + }{ + { + desc: "new journal successfully", + journal: journal.Journal{ + ID: id, + Operation: operation, + OccurredAt: occurredAt, + Attributes: payload, + Metadata: payload, + }, + err: nil, + }, + { + desc: "with duplicate journal", + journal: journal.Journal{ + ID: id, + Operation: operation, + OccurredAt: occurredAt, + Attributes: payload, + Metadata: payload, + }, + err: repoerr.ErrConflict, + }, + { + desc: "with massive journal metadata and attributes", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation, + OccurredAt: time.Now(), + Attributes: map[string]interface{}{ + "attributes": map[string]interface{}{ + "attributes": map[string]interface{}{ + "attributes": map[string]interface{}{ + "attributes": map[string]interface{}{ + "attributes": map[string]interface{}{ + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + Metadata: map[string]interface{}{ + "metadata": map[string]interface{}{ + "metadata": map[string]interface{}{ + "metadata": map[string]interface{}{ + "metadata": map[string]interface{}{ + "metadata": map[string]interface{}{ + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + "data": payload, + }, + }, + err: nil, + }, + { + desc: "with nil journal operation", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + OccurredAt: time.Now(), + Attributes: payload, + Metadata: payload, + }, + err: repoerr.ErrCreateEntity, + }, + { + desc: "with empty journal operation", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: "", + OccurredAt: time.Now().Add(-time.Hour), + Attributes: payload, + Metadata: payload, + }, + err: nil, + }, + { + desc: "with nil journal occurred_at", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation, + Attributes: payload, + Metadata: payload, + }, + err: repoerr.ErrCreateEntity, + }, + { + desc: "with empty journal occurred_at", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation, + OccurredAt: time.Time{}, + Attributes: payload, + Metadata: payload, + }, + err: nil, + }, + { + desc: "with nil journal attributes", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation + ".with.nil.attributes", + OccurredAt: time.Now(), + Metadata: payload, + }, + err: nil, + }, + { + desc: "with invalid journal attributes", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation, + OccurredAt: time.Now(), + Attributes: map[string]interface{}{"invalid": make(chan struct{})}, + Metadata: payload, + }, + err: repoerr.ErrCreateEntity, + }, + { + desc: "with empty journal attributes", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation + ".with.empty.attributes", + OccurredAt: time.Now(), + Attributes: map[string]interface{}{}, + Metadata: payload, + }, + err: nil, + }, + { + desc: "with nil journal metadata", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation + ".with.nil.metadata", + OccurredAt: time.Now(), + Attributes: payload, + }, + err: nil, + }, + { + desc: "with invalid journal metadata", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation, + OccurredAt: time.Now(), + Metadata: map[string]interface{}{"invalid": make(chan struct{})}, + Attributes: payload, + }, + err: repoerr.ErrCreateEntity, + }, + { + desc: "with empty journal metadata", + journal: journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: operation + ".with.empty.metadata", + OccurredAt: time.Now(), + Metadata: map[string]interface{}{}, + Attributes: payload, + }, + err: nil, + }, + { + desc: "with empty journal", + journal: journal.Journal{}, + err: repoerr.ErrCreateEntity, + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + switch err := repo.Save(context.Background(), tc.journal); { + case err == nil: + assert.Nil(t, err) + default: + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) + } + }) + } +} + +func TestJournalRetrieveAll(t *testing.T) { + t.Cleanup(func() { + _, err := db.Exec("DELETE FROM journal") + require.Nil(t, err, fmt.Sprintf("clean journal unexpected error: %s", err)) + }) + repo := postgres.NewRepository(database) + + num := 200 + + var items []journal.Journal + for i := 0; i < num; i++ { + j := journal.Journal{ + ID: testsutil.GenerateUUID(t), + Operation: fmt.Sprintf("%s-%d", operation, i), + OccurredAt: time.Now().UTC().Truncate(time.Millisecond), + Attributes: userAttributesV1, + Metadata: payload, + } + if i%2 == 0 { + j.Operation = fmt.Sprintf("%s-%d", thingOperation, i) + j.Attributes = thingAttributesV1 + } + if i%3 == 0 { + j.Attributes = userAttributesV2 + } + if i%5 == 0 { + j.Attributes = thingAttributesV2 + } + err := repo.Save(context.Background(), j) + require.Nil(t, err, fmt.Sprintf("create journal unexpected error: %s", err)) + j.ID = "" + items = append(items, j) + } + + reversedItems := make([]journal.Journal, len(items)) + copy(reversedItems, items) + sort.Slice(reversedItems, func(i, j int) bool { + return reversedItems[i].OccurredAt.After(reversedItems[j].OccurredAt) + }) + + cases := []struct { + desc string + page journal.Page + response journal.JournalsPage + err error + }{ + { + desc: "successfully", + page: journal.Page{ + Offset: 0, + Limit: 1, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 1, + Journals: items[:1], + }, + err: nil, + }, + { + desc: "with offset and empty limit", + page: journal.Page{ + Offset: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 10, + Limit: 0, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with limit and empty offset", + page: journal.Page{ + Limit: 50, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 50, + Journals: items[:50], + }, + }, + { + desc: "with offset and limit", + page: journal.Page{ + Offset: 10, + Limit: 50, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 10, + Limit: 50, + Journals: items[10:60], + }, + }, + { + desc: "with offset out of range", + page: journal.Page{ + Offset: 1000, + Limit: 50, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 1000, + Limit: 50, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with offset and limit out of range", + page: journal.Page{ + Offset: 170, + Limit: 50, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 170, + Limit: 50, + Journals: items[170:200], + }, + }, + { + desc: "with limit out of range", + page: journal.Page{ + Offset: 0, + Limit: 1000, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 1000, + Journals: items, + }, + }, + { + desc: "with empty page", + page: journal.Page{}, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 0, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with operation", + page: journal.Page{ + Operation: items[0].Operation, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: 1, + Offset: 0, + Limit: 10, + Journals: []journal.Journal{items[0]}, + }, + }, + { + desc: "with invalid operation", + page: journal.Page{ + Operation: strings.Repeat("a", 37), + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: 0, + Offset: 0, + Limit: 10, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with attributes", + page: journal.Page{ + WithAttributes: true, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with metadata", + page: journal.Page{ + WithMetadata: true, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with attributes and Metadata", + page: journal.Page{ + WithAttributes: true, + WithMetadata: true, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with from", + page: journal.Page{ + From: items[0].OccurredAt, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with invalid from", + page: journal.Page{ + From: time.Now().UTC().Truncate(time.Millisecond).Add(time.Hour), + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: 0, + Offset: 0, + Limit: 10, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with to", + page: journal.Page{ + To: items[num-1].OccurredAt, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with invalid to", + page: journal.Page{ + To: time.Now().UTC().Truncate(time.Millisecond).Add(-time.Hour), + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: 0, + Offset: 0, + Limit: 10, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with from and to", + page: journal.Page{ + From: items[0].OccurredAt, + To: items[num-1].OccurredAt, + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with asc direction", + page: journal.Page{ + Direction: "ASC", + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: items[:10], + }, + }, + { + desc: "with desc direction", + page: journal.Page{ + Direction: "DESC", + Offset: 0, + Limit: 10, + }, + response: journal.JournalsPage{ + Total: uint64(num), + Offset: 0, + Limit: 10, + Journals: reversedItems[:10], + }, + }, + { + desc: "with user entity type", + page: journal.Page{ + Offset: 0, + Limit: 10, + EntityID: entityID, + EntityType: journal.UserEntity, + }, + response: journal.JournalsPage{ + Total: uint64(len(extractEntities(items, journal.UserEntity, entityID))), + Offset: 0, + Limit: 10, + Journals: extractEntities(items, journal.UserEntity, entityID)[:10], + }, + }, + { + desc: "with user entity type, attributes and metadata", + page: journal.Page{ + Offset: 0, + Limit: 10, + EntityID: entityID, + EntityType: journal.UserEntity, + WithAttributes: true, + WithMetadata: true, + }, + response: journal.JournalsPage{ + Total: uint64(len(extractEntities(items, journal.UserEntity, entityID))), + Offset: 0, + Limit: 10, + Journals: extractEntities(items, journal.UserEntity, entityID)[:10], + }, + }, + { + desc: "with thing entity type", + page: journal.Page{ + Offset: 0, + Limit: 10, + EntityID: entityID, + EntityType: journal.ThingEntity, + }, + response: journal.JournalsPage{ + Total: uint64(len(extractEntities(items, journal.ThingEntity, entityID))), + Offset: 0, + Limit: 10, + Journals: extractEntities(items, journal.ThingEntity, entityID)[:10], + }, + }, + { + desc: "with invalid entity id", + page: journal.Page{ + Offset: 0, + Limit: 10, + EntityID: testsutil.GenerateUUID(&testing.T{}), + EntityType: journal.ChannelEntity, + }, + response: journal.JournalsPage{ + Total: 0, + Offset: 0, + Limit: 10, + Journals: []journal.Journal(nil), + }, + }, + { + desc: "with all filters", + page: journal.Page{ + Offset: 0, + Limit: 10, + Operation: items[0].Operation, + From: items[0].OccurredAt, + To: items[num-1].OccurredAt, + WithAttributes: true, + WithMetadata: true, + Direction: "asc", + }, + response: journal.JournalsPage{ + Total: 1, + Offset: 0, + Limit: 10, + Journals: []journal.Journal{items[0]}, + }, + }, + } + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + page, err := repo.RetrieveAll(context.Background(), tc.page) + assert.Equal(t, tc.response.Total, page.Total) + assert.Equal(t, tc.response.Offset, page.Offset) + assert.Equal(t, tc.response.Limit, page.Limit) + for i := range tc.response.Journals { + tc.response.Journals[i].Attributes = map[string]interface{}{} + page.Journals[i].Attributes = map[string]interface{}{} + tc.response.Journals[i].Metadata = map[string]interface{}{} + page.Journals[i].Metadata = map[string]interface{}{} + } + assert.ElementsMatch(t, tc.response.Journals, page.Journals) + + assert.Equal(t, tc.err, err) + }) + } +} + +func extractEntities(journals []journal.Journal, entityType journal.EntityType, entityID string) []journal.Journal { + var entities []journal.Journal + for _, j := range journals { + switch entityType { + case journal.UserEntity: + if strings.HasPrefix(j.Operation, "user.") && j.Attributes["id"] == entityID || j.Attributes["user_id"] == entityID { + entities = append(entities, j) + } + case journal.GroupEntity: + if strings.HasPrefix(j.Operation, "group.") && j.Attributes["id"] == entityID || j.Attributes["group_id"] == entityID { + entities = append(entities, j) + } + case journal.ThingEntity: + if strings.HasPrefix(j.Operation, "thing.") && j.Attributes["id"] == entityID || j.Attributes["thing_id"] == entityID { + entities = append(entities, j) + } + case journal.ChannelEntity: + if strings.HasPrefix(j.Operation, "channel.") && j.Attributes["id"] == entityID || j.Attributes["group_id"] == entityID { + entities = append(entities, j) + } + } + } + + return entities +} diff --git a/journal/postgres/setup_test.go b/journal/postgres/setup_test.go new file mode 100644 index 0000000000..da5ade6c3a --- /dev/null +++ b/journal/postgres/setup_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package postgres_test + +import ( + "database/sql" + "fmt" + "log" + "os" + "testing" + "time" + + pgclient "github.com/absmach/magistrala/internal/clients/postgres" + "github.com/absmach/magistrala/internal/postgres" + apostgres "github.com/absmach/magistrala/journal/postgres" + "github.com/jmoiron/sqlx" + dockertest "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "go.opentelemetry.io/otel" +) + +var ( + db *sqlx.DB + database postgres.Database + tracer = otel.Tracer("repo_tests") +) + +func TestMain(m *testing.M) { + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + container, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "16.2-alpine", + Env: []string{ + "POSTGRES_USER=test", + "POSTGRES_PASSWORD=test", + "POSTGRES_DB=test", + "listen_addresses = '*'", + }, + }, func(config *docker.HostConfig) { + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + log.Fatalf("Could not start container: %s", err) + } + + port := container.GetPort("5432/tcp") + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + pool.MaxWait = 120 * time.Second + if err := pool.Retry(func() error { + url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) + db, err := sql.Open("pgx", url) + if err != nil { + return err + } + return db.Ping() + }); err != nil { + log.Fatalf("Could not connect to docker: %s", err) + } + + dbConfig := pgclient.Config{ + Host: "localhost", + Port: port, + User: "test", + Pass: "test", + Name: "test", + SSLMode: "disable", + SSLCert: "", + SSLKey: "", + SSLRootCert: "", + } + + if db, err = pgclient.Setup(dbConfig, *apostgres.Migration()); err != nil { + log.Fatalf("Could not setup test DB connection: %s", err) + } + + database = postgres.NewDatabase(db, dbConfig, tracer) + + code := m.Run() + + // Defers will not be run when using os.Exit + db.Close() + if err := pool.Purge(container); err != nil { + log.Fatalf("Could not purge container: %s", err) + } + + os.Exit(code) +} diff --git a/journal/service.go b/journal/service.go new file mode 100644 index 0000000000..4f72bca205 --- /dev/null +++ b/journal/service.go @@ -0,0 +1,85 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package journal + +import ( + "context" + + "github.com/absmach/magistrala" + "github.com/absmach/magistrala/auth" + "github.com/absmach/magistrala/pkg/errors" + svcerr "github.com/absmach/magistrala/pkg/errors/service" +) + +type service struct { + idProvider magistrala.IDProvider + auth magistrala.AuthServiceClient + repository Repository +} + +func NewService(idp magistrala.IDProvider, repository Repository, authClient magistrala.AuthServiceClient) Service { + return &service{ + idProvider: idp, + auth: authClient, + repository: repository, + } +} + +func (svc *service) Save(ctx context.Context, journal Journal) error { + id, err := svc.idProvider.ID() + if err != nil { + return err + } + journal.ID = id + + return svc.repository.Save(ctx, journal) +} + +func (svc *service) RetrieveAll(ctx context.Context, token string, page Page) (JournalsPage, error) { + if err := svc.authorize(ctx, token, page.EntityID, page.EntityType.AuthString()); err != nil { + return JournalsPage{}, err + } + + return svc.repository.RetrieveAll(ctx, page) +} + +func (svc *service) authorize(ctx context.Context, token, entityID, entityType string) error { + user, err := svc.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) + if err != nil { + return errors.Wrap(svcerr.ErrAuthentication, err) + } + + permission := auth.ViewPermission + objectType := entityType + object := entityID + subject := user.GetId() + + // If the entity is a user, we need to check if the user is an admin + if entityType == auth.UserType { + permission = auth.AdminPermission + objectType = auth.PlatformType + object = auth.MagistralaObject + subject = user.GetUserId() + } + + req := &magistrala.AuthorizeReq{ + Domain: user.GetDomainId(), + SubjectType: auth.UserType, + SubjectKind: auth.UsersKind, + Subject: subject, + Permission: permission, + ObjectType: objectType, + Object: object, + } + + res, err := svc.auth.Authorize(ctx, req) + if err != nil { + return errors.Wrap(svcerr.ErrAuthorization, err) + } + if !res.GetAuthorized() { + return svcerr.ErrAuthorization + } + + return nil +} diff --git a/journal/service_test.go b/journal/service_test.go new file mode 100644 index 0000000000..b1520e995e --- /dev/null +++ b/journal/service_test.go @@ -0,0 +1,210 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package journal_test + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/absmach/magistrala" + "github.com/absmach/magistrala/auth" + authmocks "github.com/absmach/magistrala/auth/mocks" + "github.com/absmach/magistrala/internal/testsutil" + "github.com/absmach/magistrala/journal" + "github.com/absmach/magistrala/journal/mocks" + "github.com/absmach/magistrala/pkg/errors" + repoerr "github.com/absmach/magistrala/pkg/errors/repository" + svcerr "github.com/absmach/magistrala/pkg/errors/service" + "github.com/absmach/magistrala/pkg/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var ( + validJournal = journal.Journal{ + Operation: "user.create", + OccurredAt: time.Now().Add(-time.Hour), + Attributes: map[string]interface{}{ + "temperature": rand.Float64(), + "humidity": rand.Float64(), + }, + Metadata: map[string]interface{}{ + "sensor_id": rand.Intn(1000), + }, + } + idProvider = uuid.New() +) + +func TestSave(t *testing.T) { + repo := new(mocks.Repository) + authsvc := new(authmocks.AuthClient) + svc := journal.NewService(idProvider, repo, authsvc) + + cases := []struct { + desc string + journal journal.Journal + repoErr error + err error + }{ + { + desc: "successful with ID and EntityType", + journal: validJournal, + repoErr: nil, + err: nil, + }, + { + desc: "with repo error", + repoErr: repoerr.ErrCreateEntity, + err: repoerr.ErrCreateEntity, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + repoCall := repo.On("Save", context.Background(), mock.Anything).Return(tc.repoErr) + err := svc.Save(context.Background(), tc.journal) + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) + repoCall.Unset() + }) + } +} + +func TestReadAll(t *testing.T) { + repo := new(mocks.Repository) + authsvc := new(authmocks.AuthClient) + svc := journal.NewService(idProvider, repo, authsvc) + + validToken := "token" + validPage := journal.Page{ + Offset: 0, + Limit: 10, + EntityID: testsutil.GenerateUUID(t), + EntityType: journal.ThingEntity, + } + + cases := []struct { + desc string + token string + page journal.Page + resp journal.JournalsPage + identifyRes *magistrala.IdentityRes + identifyErr error + authRes *magistrala.AuthorizeRes + authErr error + repoErr error + err error + }{ + { + desc: "successful", + token: validToken, + page: validPage, + resp: journal.JournalsPage{ + Total: 1, + Offset: 0, + Limit: 10, + Journals: []journal.Journal{validJournal}, + }, + identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)}, + authRes: &magistrala.AuthorizeRes{Authorized: true}, + authErr: nil, + repoErr: nil, + err: nil, + }, + { + desc: "successful for user", + token: validToken, + page: journal.Page{ + Offset: 0, + Limit: 10, + EntityID: testsutil.GenerateUUID(t), + EntityType: journal.UserEntity, + }, + resp: journal.JournalsPage{ + Total: 1, + Offset: 0, + Limit: 10, + Journals: []journal.Journal{validJournal}, + }, + identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)}, + authRes: &magistrala.AuthorizeRes{Authorized: true}, + authErr: nil, + repoErr: nil, + err: nil, + }, + { + desc: "with identify error", + token: validToken, + page: validPage, + resp: journal.JournalsPage{}, + identifyRes: &magistrala.IdentityRes{}, + identifyErr: svcerr.ErrAuthentication, + err: svcerr.ErrAuthentication, + }, + { + desc: "with repo error", + token: validToken, + page: validPage, + resp: journal.JournalsPage{}, + identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)}, + authRes: &magistrala.AuthorizeRes{Authorized: true}, + repoErr: repoerr.ErrViewEntity, + err: repoerr.ErrViewEntity, + }, + { + desc: "with failed to authorize", + token: validToken, + page: validPage, + resp: journal.JournalsPage{}, + identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)}, + authRes: &magistrala.AuthorizeRes{Authorized: false}, + authErr: nil, + repoErr: nil, + err: svcerr.ErrAuthorization, + }, + { + desc: "with error on authorize", + token: validToken, + page: validPage, + resp: journal.JournalsPage{}, + identifyRes: &magistrala.IdentityRes{Id: testsutil.GenerateUUID(t), UserId: testsutil.GenerateUUID(t)}, + authRes: &magistrala.AuthorizeRes{Authorized: true}, + authErr: svcerr.ErrAuthorization, + repoErr: nil, + err: svcerr.ErrAuthorization, + }, + } + + for _, tc := range cases { + t.Run(tc.desc, func(t *testing.T) { + authReq := &magistrala.AuthorizeReq{ + SubjectType: auth.UserType, + SubjectKind: auth.UsersKind, + Subject: tc.identifyRes.GetId(), + ObjectType: tc.page.EntityType.AuthString(), + Object: tc.page.EntityID, + Permission: auth.ViewPermission, + } + if tc.page.EntityType == journal.UserEntity { + authReq.Permission = auth.AdminPermission + authReq.ObjectType = auth.PlatformType + authReq.Object = auth.MagistralaObject + authReq.Subject = tc.identifyRes.GetUserId() + } + authCall := authsvc.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) + authCall1 := authsvc.On("Authorize", context.Background(), authReq).Return(tc.authRes, tc.authErr) + repoCall := repo.On("RetrieveAll", context.Background(), tc.page).Return(tc.resp, tc.repoErr) + resp, err := svc.RetrieveAll(context.Background(), tc.token, tc.page) + if tc.err == nil { + assert.Equal(t, tc.resp, resp, tc.desc) + } + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) + repoCall.Unset() + authCall.Unset() + authCall1.Unset() + }) + } +} diff --git a/lora/events/events.go b/lora/events/events.go index 98cc9cde2c..a0d0be25de 100644 --- a/lora/events/events.go +++ b/lora/events/events.go @@ -22,6 +22,6 @@ type removeChannelEvent struct { } type connectionThingEvent struct { - chanID string - thingID string + chanID string + thingIDs []string } diff --git a/lora/events/streams.go b/lora/events/streams.go index 9f46e3151a..a4aa4b181a 100644 --- a/lora/events/streams.go +++ b/lora/events/streams.go @@ -5,7 +5,6 @@ package events import ( "context" - "encoding/json" "errors" "github.com/absmach/magistrala/lora" @@ -58,35 +57,20 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { } switch msg["operation"] { - case thingCreate: + case thingCreate, thingUpdate: cte, derr := decodeCreateThing(msg) if derr != nil { err = derr break } err = es.svc.CreateThing(ctx, cte.id, cte.loraDevEUI) - case thingUpdate: - ute, derr := decodeCreateThing(msg) - if derr != nil { - err = derr - break - } - err = es.svc.CreateThing(ctx, ute.id, ute.loraDevEUI) - - case channelCreate: + case channelCreate, channelUpdate: cce, derr := decodeCreateChannel(msg) if derr != nil { err = derr break } err = es.svc.CreateChannel(ctx, cce.id, cce.loraAppID) - case channelUpdate: - uce, derr := decodeCreateChannel(msg) - if derr != nil { - err = derr - break - } - err = es.svc.CreateChannel(ctx, uce.id, uce.loraAppID) case thingRemove: rte := decodeRemoveThing(msg) err = es.svc.RemoveThing(ctx, rte.id) @@ -95,10 +79,22 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { err = es.svc.RemoveChannel(ctx, rce.id) case thingConnect: tce := decodeConnectionThing(msg) - err = es.svc.ConnectThing(ctx, tce.chanID, tce.thingID) + + for _, thingID := range tce.thingIDs { + err = es.svc.ConnectThing(ctx, tce.chanID, thingID) + if err != nil { + return err + } + } case thingDisconnect: tde := decodeConnectionThing(msg) - err = es.svc.DisconnectThing(ctx, tde.chanID, tde.thingID) + + for _, thingID := range tde.thingIDs { + err = es.svc.DisconnectThing(ctx, tde.chanID, thingID) + if err != nil { + return err + } + } } if err != nil && err != errMetadataType { return err @@ -108,14 +104,10 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { } func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) { - strmeta := read(event, "metadata", "{}") - var metadata map[string]interface{} - if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil { - return createThingEvent{}, err - } + metadata := events.Read(event, "metadata", map[string]interface{}{}) cte := createThingEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } m, ok := metadata[keyType] @@ -139,19 +131,15 @@ func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) { func decodeRemoveThing(event map[string]interface{}) removeThingEvent { return removeThingEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, error) { - strmeta := read(event, "metadata", "{}") - var metadata map[string]interface{} - if err := json.Unmarshal([]byte(strmeta), &metadata); err != nil { - return createChannelEvent{}, err - } + metadata := events.Read(event, "metadata", map[string]interface{}{}) cce := createChannelEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } m, ok := metadata[keyType] @@ -175,22 +163,13 @@ func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, erro func decodeConnectionThing(event map[string]interface{}) connectionThingEvent { return connectionThingEvent{ - chanID: read(event, "chan_id", ""), - thingID: read(event, "thing_id", ""), + chanID: events.Read(event, "group_id", ""), + thingIDs: events.ReadStringSlice(event, "member_ids"), } } func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent { return removeChannelEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } - -func read(event map[string]interface{}, key, def string) string { - val, ok := event[key].(string) - if !ok { - return def - } - - return val -} diff --git a/mqtt/events/events.go b/mqtt/events/events.go index bbe8682c47..9ae960bed1 100644 --- a/mqtt/events/events.go +++ b/mqtt/events/events.go @@ -9,14 +9,14 @@ var _ events.Event = (*mqttEvent)(nil) type mqttEvent struct { clientID string - eventType string + operation string instance string } func (me mqttEvent) Encode() (map[string]interface{}, error) { return map[string]interface{}{ - "thing_id": me.clientID, - "event_type": me.eventType, - "instance": me.instance, + "thing_id": me.clientID, + "operation": me.operation, + "instance": me.instance, }, nil } diff --git a/mqtt/events/streams.go b/mqtt/events/streams.go index 8889accab2..780d1a6ea4 100644 --- a/mqtt/events/streams.go +++ b/mqtt/events/streams.go @@ -42,7 +42,7 @@ func NewEventStore(ctx context.Context, url, instance string) (EventStore, error func (es *eventStore) Connect(ctx context.Context, clientID string) error { ev := mqttEvent{ clientID: clientID, - eventType: "connect", + operation: "connect", instance: es.instance, } @@ -53,7 +53,7 @@ func (es *eventStore) Connect(ctx context.Context, clientID string) error { func (es *eventStore) Disconnect(ctx context.Context, clientID string) error { ev := mqttEvent{ clientID: clientID, - eventType: "disconnect", + operation: "disconnect", instance: es.instance, } diff --git a/opcua/events/streams.go b/opcua/events/streams.go index ccee65e733..7294a447fa 100644 --- a/opcua/events/streams.go +++ b/opcua/events/streams.go @@ -5,8 +5,6 @@ package events import ( "context" - "encoding/base64" - "encoding/json" "errors" "github.com/absmach/magistrala/opcua" @@ -14,16 +12,16 @@ import ( ) const ( - keyType = "opcua" - keyNodeID = "node_id" - keyServerURI = "server_uri" - channelPrefix = "group." - thingPrefix = "thing." + keyType = "opcua" + keyNodeID = "node_id" + keyServerURI = "server_uri" + thingPrefix = "thing." thingCreate = thingPrefix + "create" thingUpdate = thingPrefix + "update" thingRemove = thingPrefix + "remove" + channelPrefix = "channel." channelCreate = channelPrefix + "create" channelUpdate = channelPrefix + "update" channelRemove = channelPrefix + "remove" @@ -108,20 +106,10 @@ func (es *eventHandler) Handle(ctx context.Context, event events.Event) error { } func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) { - strmeta := read(event, "metadata", "{}") - - // Metadata is base64 encoded since it is marshalled as []byte. - meta, err := base64.StdEncoding.DecodeString(strmeta) - if err != nil { - return createThingEvent{}, err - } - var metadata map[string]interface{} - if err := json.Unmarshal(meta, &metadata); err != nil { - return createThingEvent{}, err - } + metadata := events.Read(event, "metadata", map[string]interface{}{}) cte := createThingEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } metadataOpcua, ok := metadata[keyType] @@ -145,23 +133,15 @@ func decodeCreateThing(event map[string]interface{}) (createThingEvent, error) { func decodeRemoveThing(event map[string]interface{}) removeThingEvent { return removeThingEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, error) { - strmeta := read(event, "metadata", "{}") - meta, err := base64.StdEncoding.DecodeString(strmeta) - if err != nil { - return createChannelEvent{}, err - } - var metadata map[string]interface{} - if err := json.Unmarshal(meta, &metadata); err != nil { - return createChannelEvent{}, err - } + metadata := events.Read(event, "metadata", map[string]interface{}{}) cce := createChannelEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } metadataOpcua, ok := metadata[keyType] @@ -185,45 +165,20 @@ func decodeCreateChannel(event map[string]interface{}) (createChannelEvent, erro func decodeRemoveChannel(event map[string]interface{}) removeChannelEvent { return removeChannelEvent{ - id: read(event, "id", ""), + id: events.Read(event, "id", ""), } } func decodeConnectThing(event map[string]interface{}) connectThingEvent { return connectThingEvent{ - chanID: read(event, "group_id", ""), - thingIDs: readMemberIDs(event, "member_ids"), + chanID: events.Read(event, "group_id", ""), + thingIDs: events.ReadStringSlice(event, "member_ids"), } } func decodeDisconnectThing(event map[string]interface{}) connectThingEvent { return connectThingEvent{ - chanID: read(event, "chan_id", ""), - thingIDs: readMemberIDs(event, "member_ids"), - } -} - -func read(event map[string]interface{}, key, def string) string { - val, ok := event[key].(string) - if !ok { - return def - } - - return val -} - -func readMemberIDs(event map[string]interface{}, key string) []string { - var memberIDs []string - val, ok := event[key].([]interface{}) - if !ok { - return memberIDs + chanID: events.Read(event, "group_id", ""), + thingIDs: events.ReadStringSlice(event, "member_ids"), } - - for _, v := range val { - if str, ok := v.(string); ok { - memberIDs = append(memberIDs, str) - } - } - - return memberIDs } diff --git a/pkg/events/events.go b/pkg/events/events.go index 626bef11d3..65845a785c 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -55,3 +55,33 @@ type Subscriber interface { // Close gracefully closes event subscriber's connection. Close() error } + +// Read reads value from event map. +// If value is not of type T, returns default value. +func Read[T any](event map[string]interface{}, key string, def T) T { + val, ok := event[key].(T) + if !ok { + return def + } + + return val +} + +// ReadStringSlice reads string slice from event map. +// If value is not a string slice, returns empty slice. +func ReadStringSlice(event map[string]interface{}, key string) []string { + var res []string + + vals, ok := event[key].([]interface{}) + if !ok { + return res + } + + for _, v := range vals { + if s, ok := v.(string); ok { + res = append(res, s) + } + } + + return res +} diff --git a/pkg/events/redis/publisher.go b/pkg/events/redis/publisher.go index e6a29626c0..c3a6201c96 100644 --- a/pkg/events/redis/publisher.go +++ b/pkg/events/redis/publisher.go @@ -5,6 +5,7 @@ package redis import ( "context" + "encoding/json" "sync" "time" @@ -45,11 +46,16 @@ func (es *pubEventStore) Publish(ctx context.Context, event events.Event) error } values["occurred_at"] = time.Now().UnixNano() + data, err := json.Marshal(values) + if err != nil { + return err + } + record := &redis.XAddArgs{ Stream: es.stream, MaxLen: events.MaxEventStreamLen, Approx: true, - Values: values, + Values: map[string]interface{}{"data": string(data)}, } switch err := es.checkConnection(ctx); err { diff --git a/pkg/events/redis/publisher_test.go b/pkg/events/redis/publisher_test.go index ca1320a4b0..5760d79dc6 100644 --- a/pkg/events/redis/publisher_test.go +++ b/pkg/events/redis/publisher_test.go @@ -5,11 +5,9 @@ package redis_test import ( "context" - "encoding/json" "errors" "fmt" "math/rand" - "strconv" "testing" "time" @@ -33,23 +31,11 @@ type testEvent struct { } func (te testEvent) Encode() (map[string]interface{}, error) { - data := make(map[string]interface{}) - for k, v := range te.Data { - switch v.(type) { - case string: - data[k] = v - case float64: - data[k] = v - default: - b, err := json.Marshal(v) - if err != nil { - return nil, err - } - data[k] = string(b) - } + if te.Data == nil { + return map[string]interface{}{}, nil } - return data, nil + return te.Data, nil } func TestPublish(t *testing.T) { @@ -87,12 +73,12 @@ func TestPublish(t *testing.T) { desc: "publish event successfully", err: nil, event: map[string]interface{}{ - "temperature": fmt.Sprintf("%f", rand.Float64()), - "humidity": fmt.Sprintf("%f", rand.Float64()), + "temperature": float64(rand.Float64()), + "humidity": float64(rand.Float64()), "sensor_id": "abc123", "location": "Earth", "status": "normal", - "timestamp": fmt.Sprintf("%d", time.Now().UnixNano()), + "timestamp": float64(time.Now().UnixNano()), "operation": "create", "occurred_at": time.Now().UnixNano(), }, @@ -106,22 +92,22 @@ func TestPublish(t *testing.T) { desc: "publish event with invalid event location", err: fmt.Errorf("json: unsupported type: chan int"), event: map[string]interface{}{ - "temperature": fmt.Sprintf("%f", rand.Float64()), - "humidity": fmt.Sprintf("%f", rand.Float64()), + "temperature": float64(rand.Float64()), + "humidity": float64(rand.Float64()), "sensor_id": "abc123", "location": make(chan int), "status": "normal", "timestamp": "invalid", "operation": "create", - "occurred_at": time.Now().UnixNano(), + "occurred_at": float64(time.Now().UnixNano()), }, }, { desc: "publish event with nested sting value", err: nil, event: map[string]interface{}{ - "temperature": fmt.Sprintf("%f", rand.Float64()), - "humidity": fmt.Sprintf("%f", rand.Float64()), + "temperature": float64(rand.Float64()), + "humidity": float64(rand.Float64()), "sensor_id": "abc123", "location": map[string]string{ "lat": fmt.Sprintf("%f", rand.Float64()), @@ -130,7 +116,7 @@ func TestPublish(t *testing.T) { "status": "normal", "timestamp": "invalid", "operation": "create", - "occurred_at": time.Now().UnixNano(), + "occurred_at": float64(time.Now().UnixNano()), }, }, } @@ -144,9 +130,9 @@ func TestPublish(t *testing.T) { case nil: receivedEvent := <-eventsChan - roa, err := strconv.ParseInt(receivedEvent["occurred_at"].(string), 10, 64) + roa := receivedEvent["occurred_at"].(float64) assert.Nil(t, err) - if assert.WithinRange(t, time.Unix(0, roa), time.Now().Add(-time.Second), time.Now().Add(time.Second)) { + if assert.WithinRange(t, time.Unix(0, int64(roa)), time.Now().Add(-time.Second), time.Now().Add(time.Second)) { delete(receivedEvent, "occurred_at") delete(tc.event, "occurred_at") } diff --git a/pkg/events/redis/subscriber.go b/pkg/events/redis/subscriber.go index 910ecca348..f05b52de7e 100644 --- a/pkg/events/redis/subscriber.go +++ b/pkg/events/redis/subscriber.go @@ -5,6 +5,7 @@ package redis import ( "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -98,8 +99,15 @@ func (re redisEvent) Encode() (map[string]interface{}, error) { func (es *subEventStore) handle(ctx context.Context, stream string, msgs []redis.XMessage, h events.EventHandler) { for _, msg := range msgs { + var data map[string]interface{} + if err := json.Unmarshal([]byte(msg.Values["data"].(string)), &data); err != nil { + es.logger.Warn(fmt.Sprintf("failed to unmarshal redis event: %s", err)) + + return + } + event := redisEvent{ - Data: msg.Values, + Data: data, } if err := h.Handle(ctx, event); err != nil { diff --git a/pkg/events/store/store_nats.go b/pkg/events/store/store_nats.go index e344253bb2..dd9c2d130f 100644 --- a/pkg/events/store/store_nats.go +++ b/pkg/events/store/store_nats.go @@ -15,6 +15,9 @@ import ( "github.com/absmach/magistrala/pkg/events/nats" ) +// StreamAllEvents represents subject to subscribe for all the events. +const StreamAllEvents = "events.>" + func init() { log.Println("The binary was build using nats as the events store") } diff --git a/pkg/events/store/store_rabbitmq.go b/pkg/events/store/store_rabbitmq.go index 0af15e0d70..233ff78cb2 100644 --- a/pkg/events/store/store_rabbitmq.go +++ b/pkg/events/store/store_rabbitmq.go @@ -15,6 +15,9 @@ import ( "github.com/absmach/magistrala/pkg/events/rabbitmq" ) +// StreamAllEvents represents subject to subscribe for all the events. +const StreamAllEvents = "events.#" + func init() { log.Println("The binary was build using rabbitmq as the events store") } diff --git a/pkg/events/store/store_redis.go b/pkg/events/store/store_redis.go index 136d01b794..12241c487b 100644 --- a/pkg/events/store/store_redis.go +++ b/pkg/events/store/store_redis.go @@ -15,6 +15,9 @@ import ( "github.com/absmach/magistrala/pkg/events/redis" ) +// StreamAllEvents represents subject to subscribe for all the events. +const StreamAllEvents = ">" + func init() { log.Println("The binary was build using redis as the events store") } diff --git a/pkg/sdk/go/groups_test.go b/pkg/sdk/go/groups_test.go index 248fb4872b..166bbe4ca6 100644 --- a/pkg/sdk/go/groups_test.go +++ b/pkg/sdk/go/groups_test.go @@ -441,7 +441,7 @@ func TestListGroups(t *testing.T) { svcRes: groups.Page{}, svcErr: nil, response: sdk.GroupsPage{}, - err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidLevel), http.StatusInternalServerError), + err: errors.NewSDKErrorWithStatus(errors.Wrap(apiutil.ErrValidation, apiutil.ErrInvalidLevel), http.StatusBadRequest), }, { desc: "list groups with invalid page metadata", diff --git a/pkg/sdk/go/journal.go b/pkg/sdk/go/journal.go new file mode 100644 index 0000000000..46eb74b1b9 --- /dev/null +++ b/pkg/sdk/go/journal.go @@ -0,0 +1,56 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package sdk + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/absmach/magistrala/internal/apiutil" + "github.com/absmach/magistrala/pkg/errors" +) + +const journalEndpoint = "journal" + +type Journal struct { + ID string `json:"id,omitempty"` + Operation string `json:"operation,omitempty"` + OccurredAt time.Time `json:"occurred_at,omitempty"` + Payload Metadata `json:"payload,omitempty"` +} + +type JournalsPage struct { + Total uint64 `json:"total"` + Offset uint64 `json:"offset"` + Limit uint64 `json:"limit"` + Journals []Journal `json:"journals"` +} + +func (sdk mgSDK) Journal(entityType, entityID string, pm PageMetadata, token string) (journals JournalsPage, err error) { + if entityID == "" { + return JournalsPage{}, errors.NewSDKError(apiutil.ErrMissingID) + } + if entityType == "" { + return JournalsPage{}, errors.NewSDKError(apiutil.ErrMissingEntityType) + } + + url, err := sdk.withQueryParams(sdk.journalURL, fmt.Sprintf("%s/%s/%s", journalEndpoint, entityType, entityID), pm) + if err != nil { + return JournalsPage{}, errors.NewSDKError(err) + } + + _, body, sdkerr := sdk.processRequest(http.MethodGet, url, token, nil, nil, http.StatusOK) + if sdkerr != nil { + return JournalsPage{}, sdkerr + } + + var journalsPage JournalsPage + if err := json.Unmarshal(body, &journalsPage); err != nil { + return JournalsPage{}, errors.NewSDKError(err) + } + + return journalsPage, nil +} diff --git a/pkg/sdk/go/sdk.go b/pkg/sdk/go/sdk.go index ba273aa821..7b2e6b1714 100644 --- a/pkg/sdk/go/sdk.go +++ b/pkg/sdk/go/sdk.go @@ -116,6 +116,12 @@ type PageMetadata struct { UserID string `json:"user_id,omitempty"` DomainID string `json:"domain_id,omitempty"` Relation string `json:"relation,omitempty"` + Operation string `json:"operation,omitempty"` + From int64 `json:"from,omitempty"` + To int64 `json:"to,omitempty"` + WithMetadata bool `json:"with_metadata,omitempty"` + WithAttributes bool `json:"with_attributes,omitempty"` + ID string `json:"id,omitempty"` } // Credentials represent client credentials: it contains @@ -1148,6 +1154,13 @@ type SDK interface { // err := sdk.DeleteInvitation("userID", "domainID", "token") // fmt.Println(err) DeleteInvitation(userID, domainID, token string) (err error) + + // Journal returns a list of journal logs. + // + // For example: + // journals, _ := sdk.Journal("thing", "thingID", PageMetadata{Offset: 0, Limit: 10, Operation: "users.create"}, "token") + // fmt.Println(journals) + Journal(entityType, entityID string, pm PageMetadata, token string) (journal JournalsPage, err error) } type mgSDK struct { @@ -1159,6 +1172,7 @@ type mgSDK struct { usersURL string domainsURL string invitationsURL string + journalURL string HostURL string msgContentType ContentType @@ -1176,6 +1190,7 @@ type Config struct { UsersURL string DomainsURL string InvitationsURL string + JournalURL string HostURL string MsgContentType ContentType @@ -1194,6 +1209,7 @@ func NewSDK(conf Config) SDK { usersURL: conf.UsersURL, domainsURL: conf.DomainsURL, invitationsURL: conf.InvitationsURL, + journalURL: conf.JournalURL, HostURL: conf.HostURL, msgContentType: conf.MsgContentType, @@ -1354,6 +1370,17 @@ func (pm PageMetadata) query() (string, error) { if pm.Relation != "" { q.Add("relation", pm.Relation) } + if pm.Operation != "" { + q.Add("operation", pm.Operation) + } + if pm.From != 0 { + q.Add("from", strconv.FormatInt(pm.From, 10)) + } + if pm.To != 0 { + q.Add("to", strconv.FormatInt(pm.To, 10)) + } + q.Add("with_attributes", strconv.FormatBool(pm.WithAttributes)) + q.Add("with_metadata", strconv.FormatBool(pm.WithMetadata)) return q.Encode(), nil } diff --git a/pkg/sdk/mocks/sdk.go b/pkg/sdk/mocks/sdk.go index fb922cf486..02ed58467d 100644 --- a/pkg/sdk/mocks/sdk.go +++ b/pkg/sdk/mocks/sdk.go @@ -1412,6 +1412,34 @@ func (_m *SDK) IssueCert(thingID string, validity string, token string) (sdk.Cer return r0, r1 } +// Journal provides a mock function with given fields: entityType, entityID, pm, token +func (_m *SDK) Journal(entityType string, entityID string, pm sdk.PageMetadata, token string) (sdk.JournalsPage, error) { + ret := _m.Called(entityType, entityID, pm, token) + + if len(ret) == 0 { + panic("no return value specified for Journal") + } + + var r0 sdk.JournalsPage + var r1 error + if rf, ok := ret.Get(0).(func(string, string, sdk.PageMetadata, string) (sdk.JournalsPage, error)); ok { + return rf(entityType, entityID, pm, token) + } + if rf, ok := ret.Get(0).(func(string, string, sdk.PageMetadata, string) sdk.JournalsPage); ok { + r0 = rf(entityType, entityID, pm, token) + } else { + r0 = ret.Get(0).(sdk.JournalsPage) + } + + if rf, ok := ret.Get(1).(func(string, string, sdk.PageMetadata, string) error); ok { + r1 = rf(entityType, entityID, pm, token) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ListChannelUserGroups provides a mock function with given fields: channelID, pm, token func (_m *SDK) ListChannelUserGroups(channelID string, pm sdk.PageMetadata, token string) (sdk.GroupsPage, errors.SDKError) { ret := _m.Called(channelID, pm, token) diff --git a/things/events/events.go b/things/events/events.go index 3d2cd953d5..56b68b6ddc 100644 --- a/things/events/events.go +++ b/things/events/events.go @@ -4,9 +4,6 @@ package events import ( - "encoding/json" - "fmt" - "strings" "time" mgclients "github.com/absmach/magistrala/pkg/clients" @@ -57,19 +54,13 @@ func (cce createClientEvent) Encode() (map[string]interface{}, error) { val["name"] = cce.Name } if len(cce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(cce.Tags, ",")) - val["tags"] = tags + val["tags"] = cce.Tags } if cce.Domain != "" { val["domain"] = cce.Domain } if cce.Metadata != nil { - metadata, err := json.Marshal(cce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = cce.Metadata } if cce.Credentials.Identity != "" { val["identity"] = cce.Credentials.Identity @@ -100,8 +91,7 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) { val["name"] = uce.Name } if len(uce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(uce.Tags, ",")) - val["tags"] = tags + val["tags"] = uce.Tags } if uce.Domain != "" { val["domain"] = uce.Domain @@ -110,12 +100,7 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) { val["identity"] = uce.Credentials.Identity } if uce.Metadata != nil { - metadata, err := json.Marshal(uce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = uce.Metadata } if !uce.CreatedAt.IsZero() { val["created_at"] = uce.CreatedAt @@ -158,8 +143,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) { val["name"] = vce.Name } if len(vce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(vce.Tags, ",")) - val["tags"] = tags + val["tags"] = vce.Tags } if vce.Domain != "" { val["domain"] = vce.Domain @@ -168,12 +152,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) { val["identity"] = vce.Credentials.Identity } if vce.Metadata != nil { - metadata, err := json.Marshal(vce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = vce.Metadata } if !vce.CreatedAt.IsZero() { val["created_at"] = vce.CreatedAt @@ -227,12 +206,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) { val["dir"] = lce.Dir } if lce.Metadata != nil { - metadata, err := json.Marshal(lce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lce.Metadata } if lce.Domain != "" { val["domain"] = lce.Domain @@ -247,8 +221,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) { val["status"] = lce.Status.String() } if len(lce.IDs) > 0 { - ids := fmt.Sprintf("[%s]", strings.Join(lce.IDs, ",")) - val["ids"] = ids + val["ids"] = lce.IDs } if lce.Identity != "" { val["identity"] = lce.Identity @@ -281,12 +254,7 @@ func (lcge listClientByGroupEvent) Encode() (map[string]interface{}, error) { val["dir"] = lcge.Dir } if lcge.Metadata != nil { - metadata, err := json.Marshal(lcge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lcge.Metadata } if lcge.Domain != "" { val["domain"] = lcge.Domain @@ -314,7 +282,7 @@ type identifyClientEvent struct { func (ice identifyClientEvent) Encode() (map[string]interface{}, error) { return map[string]interface{}{ "operation": clientIdentify, - "thing_id": ice.thingID, + "id": ice.thingID, }, nil } @@ -334,7 +302,7 @@ type authorizeClientEvent struct { func (ice authorizeClientEvent) Encode() (map[string]interface{}, error) { val := map[string]interface{}{ "operation": clientAuthorize, - "thing_id": ice.thingID, + "id": ice.thingID, } if ice.namespace != "" { val["namespace"] = ice.namespace @@ -379,7 +347,7 @@ func (sce shareClientEvent) Encode() (map[string]interface{}, error) { "operation": clientPrefix + sce.action, "id": sce.id, "relation": sce.relation, - "user_ids": strings.Join(sce.userIDs, ","), + "user_ids": sce.userIDs, }, nil } diff --git a/users/api/endpoint_test.go b/users/api/endpoint_test.go index fb7d98164a..f69a4d90d4 100644 --- a/users/api/endpoint_test.go +++ b/users/api/endpoint_test.go @@ -616,6 +616,13 @@ func TestListClients(t *testing.T) { status: http.StatusBadRequest, err: apiutil.ErrInvalidQueryParams, }, + { + desc: "list users with invalid order direction", + token: validToken, + query: "dir=invalid", + status: http.StatusBadRequest, + err: apiutil.ErrValidation, + }, { desc: "list users with duplicate order direction", token: validToken, diff --git a/users/events/events.go b/users/events/events.go index 7325fc4dc3..d46a4dfdb5 100644 --- a/users/events/events.go +++ b/users/events/events.go @@ -4,9 +4,6 @@ package events import ( - "encoding/json" - "fmt" - "strings" "time" mgclients "github.com/absmach/magistrala/pkg/clients" @@ -64,19 +61,13 @@ func (cce createClientEvent) Encode() (map[string]interface{}, error) { val["name"] = cce.Name } if len(cce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(cce.Tags, ",")) - val["tags"] = tags + val["tags"] = cce.Tags } if cce.Domain != "" { val["domain"] = cce.Domain } if cce.Metadata != nil { - metadata, err := json.Marshal(cce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = cce.Metadata } if cce.Credentials.Identity != "" { val["identity"] = cce.Credentials.Identity @@ -107,19 +98,13 @@ func (uce updateClientEvent) Encode() (map[string]interface{}, error) { val["name"] = uce.Name } if len(uce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(uce.Tags, ",")) - val["tags"] = tags + val["tags"] = uce.Tags } if uce.Credentials.Identity != "" { val["identity"] = uce.Credentials.Identity } if uce.Metadata != nil { - metadata, err := json.Marshal(uce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = uce.Metadata } if !uce.CreatedAt.IsZero() { val["created_at"] = uce.CreatedAt @@ -162,8 +147,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) { val["name"] = vce.Name } if len(vce.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(vce.Tags, ",")) - val["tags"] = tags + val["tags"] = vce.Tags } if vce.Domain != "" { val["domain"] = vce.Domain @@ -172,12 +156,7 @@ func (vce viewClientEvent) Encode() (map[string]interface{}, error) { val["identity"] = vce.Credentials.Identity } if vce.Metadata != nil { - metadata, err := json.Marshal(vce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = vce.Metadata } if !vce.CreatedAt.IsZero() { val["created_at"] = vce.CreatedAt @@ -209,8 +188,7 @@ func (vpe viewProfileEvent) Encode() (map[string]interface{}, error) { val["name"] = vpe.Name } if len(vpe.Tags) > 0 { - tags := fmt.Sprintf("[%s]", strings.Join(vpe.Tags, ",")) - val["tags"] = tags + val["tags"] = vpe.Tags } if vpe.Domain != "" { val["domain"] = vpe.Domain @@ -219,12 +197,7 @@ func (vpe viewProfileEvent) Encode() (map[string]interface{}, error) { val["identity"] = vpe.Credentials.Identity } if vpe.Metadata != nil { - metadata, err := json.Marshal(vpe.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = vpe.Metadata } if !vpe.CreatedAt.IsZero() { val["created_at"] = vpe.CreatedAt @@ -264,12 +237,7 @@ func (lce listClientEvent) Encode() (map[string]interface{}, error) { val["dir"] = lce.Dir } if lce.Metadata != nil { - metadata, err := json.Marshal(lce.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lce.Metadata } if lce.Domain != "" { val["domain"] = lce.Domain @@ -316,12 +284,7 @@ func (lcge listClientByGroupEvent) Encode() (map[string]interface{}, error) { val["dir"] = lcge.Dir } if lcge.Metadata != nil { - metadata, err := json.Marshal(lcge.Metadata) - if err != nil { - return map[string]interface{}{}, err - } - - val["metadata"] = metadata + val["metadata"] = lcge.Metadata } if lcge.Domain != "" { val["domain"] = lcge.Domain @@ -349,7 +312,7 @@ type identifyClientEvent struct { func (ice identifyClientEvent) Encode() (map[string]interface{}, error) { return map[string]interface{}{ "operation": clientIdentify, - "user_id": ice.userID, + "id": ice.userID, }, nil }