Add e2e support for setting backend via Policy

.editorconfig

@@ -4,8 +4,8 @@ insert_final_newline = true
 [{*.yml, *.yaml}]
 indent_size = 2
 
-[*.graphql]
-indent_size = 4
+[{*.graphql, *.js, *.ts}]
+indent_size = 2
 indent_style = space
 
 [Makefile]

cmd/k8s-engine/main.go

@@ -42,6 +42,7 @@ var (
 const (
 	graphQLServerName = "engine-graphql"
 	policyServiceName = "policy-svc"
+	argoRendererName  = "argo-renderer"
 )
 
 // Config holds application related configuration
@@ -111,7 +112,7 @@ func main() {
 	interfaceIOValidator := actionvalidation.NewValidator(hubClient)
 	policyIOValidator := policyvalidation.NewValidator(hubClient)
 	wfValidator := renderer.NewWorkflowInputValidator(interfaceIOValidator, policyIOValidator)
-	argoRenderer := argo.NewRenderer(cfg.Renderer, hubClient, typeInstanceHandler, wfValidator)
+	argoRenderer := argo.NewRenderer(logger.Named(argoRendererName), cfg.Renderer, hubClient, typeInstanceHandler, wfValidator)
 
 	wfCli, err := wfclientset.NewForConfig(k8sCfg)
 	exitOnError(err, "while creating Argo client")

deploy/kubernetes/crds/core.capact.io_actions.yaml

@@ -272,6 +272,18 @@ spec:
                       description: OutputTypeInstanceDetails describes the output
                         TypeInstance.
                       properties:
+                        backend:
+                          description: Backend contains information in which backend
+                            this TypeInstance is stored.
+                          properties:
+                            abstract:
+                              type: boolean
+                            id:
+                              type: string
+                          required:
+                          - abstract
+                          - id
+                          type: object
                         id:
                           description: ID is a unique identifier of the TypeInstance.
                           type: string
@@ -291,6 +303,7 @@ spec:
                           - path
                           type: object
                       required:
+                      - backend
                       - id
                       - typeReference
                       type: object

hack/lib/utilities.sh

@@ -220,7 +220,8 @@ capact::install() {
     export ENABLE_ADDING_TRUSTED_CERT=${ENABLE_ADDING_TRUSTED_CERT:-"true"}
     export HUB_MANIFESTS_SOURCE_REPO_REF=${HUB_MANIFESTS_SOURCE_REPO_REF:-${CAPACT_HUB_MANIFESTS_SOURCE_REPO_REF}}
     export HUB_MANIFESTS_SOURCE_REPO_URL=${HUB_MANIFESTS_SOURCE_REPO_URL:-${CAPACT_HUB_MANIFESTS_SOURCE_REPO_URL}}
-    export COMPONENTS="neo4j,ingress-nginx,argo,cert-manager,capact"
+    export COMPONENTS=${COMPONENTS:-"neo4j,ingress-nginx,argo,cert-manager,capact"}
+    export BUILD_IMAGES_LIST=${BUILD_IMAGES_LIST:-"argo-actions,argo-runner,e2e-test,gateway,hub-js,k8s-engine,populator"}
     export INGRESS_CONTROLLER_OVERRIDES=${INGRESS_CONTROLLER_OVERRIDES:=""}
     export CAPACT_OVERRIDES=${CAPACT_OVERRIDES:=""}
     export CAPACT_INSTALL_ADDITIONAL_OPTS=""
@@ -243,19 +244,19 @@ capact::install() {
     fi
 
     if [ -n "${HUB_MANIFESTS_SOURCE_REPO_REF:-}" ]; then
-      CAPACT_OVERRIDES+=",hub-public.populator.manifestsLocation.branch=${HUB_MANIFESTS_SOURCE_REPO_REF}"
+      CAPACT_OVERRIDES+=",hub-public.populator.manifestsLocations[0].branch=${HUB_MANIFESTS_SOURCE_REPO_REF}"
     fi
 
     if [ -n "${HUB_MANIFESTS_SOURCE_REPO_URL:-}" ]; then
-      CAPACT_OVERRIDES+=",hub-public.populator.manifestsLocation.repository=${HUB_MANIFESTS_SOURCE_REPO_URL}"
+      CAPACT_OVERRIDES+=",hub-public.populator.manifestsLocations[0].repository=${HUB_MANIFESTS_SOURCE_REPO_URL}"
     fi
 
     if [ -n "${DOCKER_REPOSITORY:-}" ]; then
       CAPACT_OVERRIDES+=",global.containerRegistry.path=${DOCKER_REPOSITORY}"
     fi
 
     if [[ "${BUILD_IMAGES:-"true"}" == "false" ]]; then
-      BUILD_IMAGES_FLAG=--build-image=""
+      BUILD_IMAGES_LIST=""
     fi
 
     if [ -n "${CAPACT_HELM_REPO:-}" ]; then
@@ -277,7 +278,7 @@ capact::install() {
         --update-hosts-file="${ENABLE_HOSTS_UPDATE}" \
         --update-trusted-certs="${ENABLE_ADDING_TRUSTED_CERT}" \
         --install-component="${COMPONENTS}" \
-        ${BUILD_IMAGES_FLAG:-} \
+        --build-image="${BUILD_IMAGES_LIST}" \
         --version="${CAPACT_VERSION}" \
         ${CAPACT_INSTALL_ADDITIONAL_OPTS}
 }

hub-js/graphql/local/examples.graphql

@@ -167,6 +167,10 @@ fragment TypeInstance on TypeInstance {
     revision
   }
   lockedBy
+  backend {
+    id
+    abstract
+  }
 
   latestResourceVersion {
     ...TypeInstanceResourceVersion

hub-js/graphql/local/schema.graphql

@@ -44,6 +44,7 @@ type TypeInstance {
     @relation(name: "OF_TYPE", direction: "OUT")
   uses: [TypeInstance!]! @relation(name: "USES", direction: "OUT")
   usedBy: [TypeInstance!]! @relation(name: "USES", direction: "IN")
+  backend: TypeInstanceBackendReference! @relation(name: "STORED_IN", direction: "OUT")
 
   latestResourceVersion: TypeInstanceResourceVersion
     @cypher(
@@ -95,6 +96,11 @@ type TypeInstanceResourceVersionSpec {
     @relation(name: "INSTRUMENTED_WITH", direction: "OUT")
 }
 
+type TypeInstanceBackendReference {
+  id: String!
+  abstract: Boolean!
+}
+
 type TypeInstanceTypeReference {
   path: NodePath!
   revision: Version!
@@ -200,6 +206,10 @@ input TypeInstanceTypeReferenceInput {
   revision: Version!
 }
 
+input TypeInstanceBackendInput {
+  id: String!
+}
+
 input CreateTypeInstanceInput {
   """
   Used to define the relationships, between the created TypeInstances
@@ -210,6 +220,10 @@ input CreateTypeInstanceInput {
   typeRef: TypeInstanceTypeReferenceInput!
   attributes: [AttributeReferenceInput!]
   value: Any
+  """
+  If not provided, TypeInstance value is stored as static value in Local Hub core storage.
+  """
+  backend: TypeInstanceBackendInput
 }
 
 input TypeInstanceUsesRelationInput {
@@ -342,17 +356,41 @@ type Mutation {
   createTypeInstance(in: CreateTypeInstanceInput!): TypeInstance!
     @cypher(
       statement: """
-      WITH apoc.convert.toJson($in.value) as value
-      MERGE (typeRef:TypeInstanceTypeReference {path: $in.typeRef.path, revision: $in.typeRef.revision})
-
       CREATE (ti:TypeInstance {id: apoc.create.uuid()})
+
+      // Backend
+      WITH *
+      CALL apoc.do.when(
+          $in.backend.id IS NOT NULL,
+          '
+           WITH false as abstract
+           RETURN $in.backend.id as id, abstract
+          ',
+          '
+           // TODO(storage): this should be resolved by Local Hub server during the insertion, not in cypher.
+           WITH true as abstract
+           MATCH (backend:TypeInstance)-[:OF_TYPE]->(typeRef {path: "cap.core.type.hub.storage.neo4j"})
+           RETURN backend.id as id, abstract
+          ',
+          {in: $in}
+      ) YIELD value as backend
+      MATCH (backendTI:TypeInstance {id: backend.id})
+      CREATE (ti)-[:USES]->(backendTI)
+      // TODO(storage): It should be taken from the uses relation but we don't have access to the TypeRef.additionalRefs to check
+      // if a given type is a backend or not. Maybe we will introduce a dedicated property to distinguish them from others.
+      MERGE (storageRef:TypeInstanceBackendReference {abstract: backend.abstract, id: backendTI.id})
+      CREATE (ti)-[:STORED_IN]->(storageRef)
+
+      // TypeRef
+      MERGE (typeRef:TypeInstanceTypeReference {path: $in.typeRef.path, revision: $in.typeRef.revision})
       CREATE (ti)-[:OF_TYPE]->(typeRef)
 
+      // Revision
       CREATE (tir: TypeInstanceResourceVersion {resourceVersion: 1, createdBy: $in.createdBy})
       CREATE (ti)-[:CONTAINS]->(tir)
 
       CREATE (tir)-[:DESCRIBED_BY]->(metadata: TypeInstanceResourceVersionMetadata)
-      CREATE (tir)-[:SPECIFIED_BY]->(spec: TypeInstanceResourceVersionSpec {value: value})
+      CREATE (tir)-[:SPECIFIED_BY]->(spec: TypeInstanceResourceVersionSpec {value: apoc.convert.toJson($in.value)})
 
       FOREACH (attr in $in.attributes |
         MERGE (attrRef: AttributeReference {path: attr.path, revision: attr.revision})

hub-js/src/index.ts

@@ -1,20 +1,21 @@
-import { ApolloServer } from "apollo-server-express";
+import {ApolloServer} from "apollo-server-express";
 import * as express from "express";
-import neo4j, { Driver } from "neo4j-driver";
+import neo4j, {Driver} from "neo4j-driver";
 import {
   createTerminus,
   HealthCheck,
   HealthCheckError,
 } from "@godaddy/terminus";
 import * as http from "http";
-import { GraphQLSchema } from "graphql";
+import {GraphQLSchema} from "graphql";
 
-import { assertSchemaOnDatabase, getSchemaForMode } from "./schema";
-import { config } from "./config";
-import { logger } from "./logger";
+import {assertSchemaOnDatabase, getSchemaForMode, HubMode} from "./schema";
+import {config} from "./config";
+import {logger} from "./logger";
+import {ensureCoreStorageTypeInstance} from "./schema/local";
 
 async function main() {
-  logger.info("Using Neo4j database", { endpoint: config.neo4j.endpoint });
+  logger.info("Using Neo4j database", {endpoint: config.neo4j.endpoint});
 
   const driver = neo4j.driver(
     config.neo4j.endpoint,
@@ -37,9 +38,14 @@ async function main() {
   };
 
   const server = await setupHttpServer(schema, driver, healthCheck);
-  const { bindPort, bindAddress } = config.graphql;
+  const {bindPort, bindAddress} = config.graphql;
 
-  logger.info("Starting Hub", { mode: config.hubMode });
+  logger.info("Starting Hub", {mode: config.hubMode});
+
+  if (config.hubMode === HubMode.Local) {
+    await ensureCoreStorageTypeInstance({driver})
+    logger.info("Successfully registered TypeInstance for core backend storage");
+  }
 
   server.listen(bindPort, bindAddress, () => {
     logger.info("GraphQL API is listening", {
@@ -54,16 +60,16 @@ async function setupHttpServer(
   healthCheck: HealthCheck
 ): Promise<http.Server> {
   const app = express();
-  app.use(express.json({ limit: config.express.bodySizeLimit }));
+  app.use(express.json({limit: config.express.bodySizeLimit}));
 
-  const apolloServer = new ApolloServer({ schema, context: { driver } });
+  const apolloServer = new ApolloServer({schema, context: {driver}});
   await apolloServer.start();
-  apolloServer.applyMiddleware({ app });
+  apolloServer.applyMiddleware({app});
 
   const server = http.createServer(app);
 
   createTerminus(server, {
-    healthChecks: { "/healthz": healthCheck },
+    healthChecks: {"/healthz": healthCheck},
   });
 
   return server;