diff --git a/docs/site/content/en/docs/Tasks/import-export/index.md b/docs/site/content/en/docs/Tasks/import-export/index.md new file mode 100644 index 000000000..00a690234 --- /dev/null +++ b/docs/site/content/en/docs/Tasks/import-export/index.md @@ -0,0 +1,64 @@ +--- +title: Import and Export Tests and Schemas +date: 2023-11-30 +description: How to import and export Tests and Schemas in Horreum +categories: [Tutorial] +weight: 3 +--- + +> **Prerequisites**: + +> 1. Horreum is running +> 2. To export you have previously defined a `Schema` for the JSON data you wish to analyze, please see [Define a Schema](/docs/tasks/define-schema-and-views/) +> 3. To export you have previously defined a Test, please see [Create new Test](/docs/tasks/create-new-test/) + +## Background + +To simplify copying [Tests](https://horreum.hyperfoil.io/docs/concepts/core-concepts/#test) and [Schemas](https://horreum.hyperfoil.io/docs/concepts/core-concepts/#schema) between Horreum instances Horreum provides a simple API to export and import new Tests and Schemas. Horreum also support updating exising Schemas and Tests by importing Tests or Schemas with existing Id's. + +## TestExport + +The export object for Tests is called [TestExport](https://horreum.hyperfoil.io/openapi/#tag/Test/operation/importTest) and contains a lot of other fields in addition to what's defined in [Test](https://horreum.hyperfoil.io/docs/concepts/core-concepts/#test). This includes, variables, experiments, actions, subscriptions, datastore and missingDataRules. This is to simplify the import/export experience and make sure that all the data related to a Test has a single entrypoint with regards to import and export. Note that secrets defined on [Action](https://horreum.hyperfoil.io/docs/tasks/configure-actions/) are not portable between Horreum instances and there might be security concerns so they are omitted. The apiKey and password attributs defined on the config field in [Datastore](https://horreum.hyperfoil.io/docs/integrations/) are also omitted and will have to be manually added in a separate step. + +## TestSchema + +The export object for Schemas is called [SchemaExport](https://horreum.hyperfoil.io/openapi/#tag/Schema/operation/importSchema) and contains other fields in addition to what's defined in [Schema](https://horreum.hyperfoil.io/docs/concepts/core-concepts/#schema). This includes, labels, extractors and transformers. This is to simplify the import/export experience and make sure that all the data related to a Schema has a single entrypoint with regards to import and export. + +## Import Schemas + +```bash +curl 'http://localhost:8080/api/schema/import/' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -d @/path/to/schema.json +``` + +If you are unfamiliar with creating the auth token please see [Upload Run](/docs/tasks/upload-new-run/). + +## Import Tests + +```bash +curl 'http://localhost:8080/api/test/import/' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -d @/path/to/test.json +``` + +## Export Schemas + +```bash +SCHEMAID='123' +curl 'http://localhost:8080/api/schema/export/?id='$SCHEMAID \ + -H 'Authorization: Bearer '$TOKEN \ + -O --output-dir /path/to/folder +``` + +## Export Tests + +```bash +TESTID='123' +curl 'http://localhost:8080/api/test/export/?id=$TESTID' \ + -s -X POST -H 'content-type: application/json' \ + -H 'Authorization: Bearer '$TOKEN \ + -O --output-dir /path/to/folder +``` diff --git a/docs/site/content/en/openapi/openapi.yaml b/docs/site/content/en/openapi/openapi.yaml index 7cff19652..271e29932 100644 --- a/docs/site/content/en/openapi/openapi.yaml +++ b/docs/site/content/en/openapi/openapi.yaml @@ -1355,16 +1355,18 @@ paths: post: tags: - Schema - description: Import an previously exported Schema + description: Import an previously exported Schema either as a new Schema or + to update an existing Schema operationId: importSchema requestBody: content: application/json: schema: - type: string + $ref: '#/components/schemas/SchemaExport' + required: true responses: - "201": - description: Created + "204": + description: Import a new Schema or update an existing Schema /api/schema/{id}: get: tags: @@ -1449,11 +1451,11 @@ paths: example: 101 responses: "200": - description: A JSON representation of the Schema object + description: A JSON representation of the SchemaExport object content: application/json: schema: - type: string + $ref: '#/components/schemas/SchemaExport' /api/schema/{id}/resetToken: post: tags: @@ -1780,16 +1782,18 @@ paths: post: tags: - Test - description: Import a previously exported Test + description: Import a previously exported Test either as a new Test or to update + an existing Test operationId: importTest requestBody: content: application/json: schema: - type: string + $ref: '#/components/schemas/TestExport' + required: true responses: "204": - description: Import a new test + description: Import a new Test or update an existing Test /api/test/summary: get: tags: @@ -1898,11 +1902,11 @@ paths: type: integer responses: "200": - description: A Test defintion formatted as json + description: A Test definition formatted as json content: application/json: schema: - type: string + $ref: '#/components/schemas/TestExport' /api/test/{id}/fingerprint: get: tags: @@ -2174,6 +2178,41 @@ components: - PROTECTED - PRIVATE type: string + Action: + required: + - id + - event + - type + - config + - testId + - active + - runAlways + type: object + properties: + id: + format: int32 + type: integer + event: + type: string + nullable: false + type: + type: string + nullable: false + config: + type: array + nullable: false + testId: + format: int32 + type: integer + nullable: false + active: + type: boolean + nullable: false + runAlways: + type: boolean + nullable: false + secrets: + type: array ActionLog: description: Action Log required: @@ -2199,6 +2238,21 @@ components: - SAME - WORSE type: string + ChangeDetection: + required: + - id + - model + - config + type: object + properties: + id: + format: int32 + type: integer + model: + type: string + config: + type: array + nullable: false ComparisonResult: description: Result of performing a Comparison type: object @@ -2551,6 +2605,8 @@ components: - POSTGRES - ELASTICSEARCH type: string + example: ELASTICSEARCH + nullable: false ElasticsearchDatastoreConfig: description: Type of backend datastore required: @@ -2981,6 +3037,34 @@ components: \ array or JSON object" type: string example: "1724" + MissingDataRule: + required: + - id + - maxStaleness + - testId + type: object + properties: + id: + format: int32 + type: integer + name: + type: string + labels: + type: array + items: + type: string + condition: + type: string + maxStaleness: + format: int64 + type: integer + nullable: false + lastNotification: + format: date-time + type: string + testId: + format: int32 + type: integer PersistentLog: description: Persistent Log required: @@ -3487,6 +3571,23 @@ components: type: string example: uri:my-schmea:0.1 nullable: false + SchemaExport: + description: Represents a Schema with all associated data used for export/import + operations. + type: object + allOf: + - $ref: '#/components/schemas/Schema' + properties: + labels: + description: Array of Labels associated with schema + type: array + items: + $ref: '#/components/schemas/Label' + transformers: + description: Array of Transformers associated with schema + type: array + items: + $ref: '#/components/schemas/Transformer' SchemaQueryResult: required: - schemas @@ -3964,6 +4065,41 @@ components: type: boolean example: true nullable: false + TestExport: + description: Represents a Test with all associated data used for export/import + operations. + type: object + allOf: + - $ref: '#/components/schemas/Test' + properties: + variables: + description: Array of Variables associated with test + type: array + items: + $ref: '#/components/schemas/Variable' + missingDataRules: + description: Array of MissingDataRules associated with test + type: array + items: + $ref: '#/components/schemas/MissingDataRule' + experiments: + description: Array of ExperimentProfiles associated with test + type: array + items: + $ref: '#/components/schemas/ExperimentProfile' + actions: + description: Array of Actions associated with test + type: array + items: + $ref: '#/components/schemas/Action' + subscriptions: + allOf: + - $ref: '#/components/schemas/Watch' + - description: Watcher object associated with test + datastore: + allOf: + - $ref: '#/components/schemas/Datastore' + - description: Datastore associated with test TestListing: type: object properties: @@ -4207,6 +4343,43 @@ components: allOf: - $ref: '#/components/schemas/ErrorDetails' - description: Validation Error Details + Variable: + required: + - id + - testId + - name + - order + - labels + - changeDetection + type: object + properties: + id: + format: int32 + type: integer + testId: + format: int32 + type: integer + nullable: false + name: + type: string + nullable: false + group: + type: string + order: + format: int32 + type: integer + nullable: false + labels: + type: array + items: + type: string + nullable: false + calculation: + type: string + changeDetection: + type: array + items: + $ref: '#/components/schemas/ChangeDetection' VersionInfo: required: - version @@ -4223,3 +4396,32 @@ components: description: Timestamp of server startup type: integer example: 2023-10-18 18:00:57 + Watch: + required: + - users + - optout + - teams + - testId + type: object + properties: + id: + format: int32 + type: integer + users: + type: array + items: + type: string + nullable: false + optout: + type: array + items: + type: string + nullable: false + teams: + type: array + items: + type: string + nullable: false + testId: + format: int32 + type: integer diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java index 6daa212b7..6f42e2402 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/ChangeDetection.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; -import org.eclipse.microprofile.openapi.annotations.media.Schema; public class ChangeDetection { @JsonProperty( required = true ) diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java index a2ee91934..732a8b0d5 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/alerting/Variable.java @@ -2,10 +2,10 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.media.Schema; +import java.util.List; import java.util.Set; public class Variable { @@ -23,19 +23,16 @@ public class Variable { public int order; @NotNull @JsonProperty(required = true) - public JsonNode labels; + public List labels; @JsonInclude(JsonInclude.Include.NON_NULL) public String calculation; - @Schema( - required = true, - implementation = ChangeDetection[].class - ) + @Schema(required = true, implementation = ChangeDetection[].class) public Set changeDetection; public Variable() { } - public Variable(Integer id, int testId, String name, String group, int order, JsonNode labels, String calculation, + public Variable(Integer id, int testId, String name, String group, int order, List labels, String calculation, Set changeDetection) { this.id = id; this.testId = testId; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java index 52ec52bc1..4e217bbb2 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/ExperimentProfile.java @@ -1,6 +1,8 @@ package io.hyperfoil.tools.horreum.api.data; +import com.fasterxml.jackson.annotation.JsonIdentityInfo; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.ObjectIdGenerators; import com.fasterxml.jackson.databind.JsonNode; import jakarta.validation.constraints.NotNull; import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; @@ -8,6 +10,7 @@ import java.util.Collection; +@JsonIdentityInfo( property = "id", generator = ObjectIdGenerators.PropertyGenerator.class) @Schema(description = "An Experiment Profile defines the labels and filters for the dataset and baseline") public class ExperimentProfile { @JsonProperty(required = true ) diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java index 32f940890..15263006b 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Label.java @@ -44,6 +44,11 @@ public class Label extends ProtectedType { public Label() { } + public Label(String name, int schemaId) { + this.name = name; + this.schemaId = schemaId; + } + public static class Value implements Serializable { public int datasetId; public int labelId; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java index 2c0915274..d8d333187 100644 --- a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/Schema.java @@ -46,6 +46,17 @@ public Schema() { access = Access.PUBLIC; } + public Schema(Schema s) { + this.id = s.id; + this.uri = s.uri; + this.name = s.name; + this.description = s.description; + this.schema = s.schema; + this.token = s.token; + this.access = s.access; + this.owner = s.owner; + } + public static class ValidationEvent { public int id; public Collection errors; diff --git a/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java new file mode 100644 index 000000000..80707f3e8 --- /dev/null +++ b/horreum-api/src/main/java/io/hyperfoil/tools/horreum/api/data/SchemaExport.java @@ -0,0 +1,23 @@ +package io.hyperfoil.tools.horreum.api.data; + +import org.eclipse.microprofile.openapi.annotations.enums.SchemaType; + +import java.util.List; + +@org.eclipse.microprofile.openapi.annotations.media.Schema(type = SchemaType.OBJECT, allOf = Schema.class, + description = "Represents a Schema with all associated data used for export/import operations.") +public class SchemaExport extends Schema { + + @org.eclipse.microprofile.openapi.annotations.media.Schema(description = "Array of Labels associated with schema") + public List