Merge branch 'main' into twunderlich/switch-to-nomad-native-ip

 Conflicts:
	nomad/grapl-core.nomad
	pulumi/grapl/__main__.py

.buildkite/pipeline.verify.yml

@@ -4,7 +4,7 @@ env:
   BUILDKITE_PLUGIN_VAULT_ENV_SECRET_PREFIX: "secret/data/buildkite/env"
   # TODO: Figure out a way for this to be sourced from our
   # rust-toolchain file
-  RUST_VERSION: 1.62.1
+  RUST_VERSION: 1.63.0
 
 steps:
   - group: ":white_check_mark: Validation"

.github/CODEOWNERS

@@ -43,7 +43,6 @@ BUILD @christophermaier
 /src/js/engagement_view/ @andrea-grapl
 /src/js/graphql_endpoint/ @andrea-grapl
 /src/proto/ @grapl-security/wg-data-infra
-/src/python/engagement-creator/ @wimax-grapl @grapl-security/wg-data-infra
 /src/python/graphql_endpoint_tests/ @wimax-grapl
 /src/python/grapl_analyzerlib/ @colin-grapl
 /src/python/grapl-common/ @wimax-grapl

3rdparty/python/requirements.txt

@@ -31,7 +31,6 @@ pyjwt
 pytest
 python-hcl2
 python-nomad==1.4.1
-redis
 requests
 setuptools
 toml==0.10.2

Dockerfile.pulumi

@@ -18,7 +18,7 @@ USER grapl
 ENV USER=grapl
 WORKDIR /home/grapl
 
-RUN curl -O https://get.pulumi.com/releases/sdk/pulumi-v3.33.1-linux-x64.tar.gz && \
+RUN curl -O https://get.pulumi.com/releases/sdk/pulumi-v3.38.0-linux-x64.tar.gz && \
     tar -xzvf pulumi-*.tar.gz && \
     mv pulumi pulumi-bin
 ENV PATH=/home/grapl/pulumi-bin:${PATH}
@@ -41,14 +41,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install --no-cache-dir --requirement constraints.txt
 
 COPY --chown=grapl pulumi pulumi
-
-# Create a directory *as the grapl user*, at which we will mount an
-# external volume to share Pulumi run outputs with other
-# containers. Because of Docker magic, this has to be present first,
-# and with the correct permissions, in order for us to be able to
-# write to it.
-RUN mkdir /home/grapl/pulumi-outputs
-
 # Make nomad files available to Pulumi
 COPY --chown=grapl nomad nomad
+
 WORKDIR /home/grapl/pulumi

docker-bake.hcl

@@ -181,7 +181,6 @@ group "rust-services" {
 group "python-services" {
   # NOTE: Please keep this list sorted in alphabetical order
   targets = [
-    "engagement-creator",
     "provisioner"
   ]
 }
@@ -421,14 +420,6 @@ target "_python-base" {
   }
 }
 
-target "engagement-creator" {
-  inherits = ["_python-base"]
-  target   = "engagement-creator-deploy"
-  tags = [
-    upstream_aware_tag("engagement-creator")
-  ]
-}
-
 target "provisioner" {
   inherits = ["_python-base"]
   target   = "provisioner-deploy"

etc/ci_scripts/dump_artifacts/cli.py

@@ -3,6 +3,7 @@
 import argparse
 import logging
 import os
+import shutil
 import sys
 from datetime import datetime
 from pathlib import Path
@@ -100,6 +101,12 @@ def main() -> None:
     os.makedirs(analysis_dir, exist_ok=False)
     pipeline_message_flow.analyze_grapl_core(artifacts_dir, analysis_dir)
 
+    # Zip up everything. We can't zip up directly into artifacts_dir or you
+    # get a recursive zip - that is to say, eating up all the space on disk.
+    zip_filename = "/tmp/ALL_ARTIFACTS"
+    shutil.make_archive(base_name=zip_filename, format="zip", root_dir=artifacts_dir)
+    shutil.move(src="/tmp/ALL_ARTIFACTS.zip", dst=artifacts_dir)
+
     LOGGER.info(f"--- Artifacts dumped to {artifacts_dir}")
 
 

nomad/consul-intentions/dgraph-alpha-0-grpc-public.json

@@ -2,12 +2,6 @@
   "Kind": "service-intentions",
   "Name": "dgraph-alpha-0-grpc-public",
   "Sources": [
-    {
-      "Name": "engagement-creator",
-      "Action": "allow",
-      "Precedence": 9,
-      "Type": "consul"
-    },
     {
       "Name": "graph-merger",
       "Action": "allow",

nomad/grapl-core.nomad

@@ -60,11 +60,6 @@ variable "kafka_bootstrap_servers" {
   description = "The URL(s) (possibly comma-separated) of the Kafka bootstrap servers."
 }
 
-variable "redis_endpoint" {
-  type        = string
-  description = "Where can services find Redis?"
-}
-
 variable "schema_table_name" {
   type        = string
   description = "What is the name of the schema table?"
@@ -229,11 +224,6 @@ locals {
     source = "grapl-data-dgraph"
   }
 
-  _redis_trimmed = trimprefix(var.redis_endpoint, "redis://")
-  _redis         = split(":", local._redis_trimmed)
-  redis_host     = local._redis[0]
-  redis_port     = local._redis[1]
-
   dns_servers = [attr.unique.network.ip-address]
 
   # Grapl services
@@ -689,7 +679,6 @@ job "grapl-core" {
         AWS_REGION         = var.aws_region
         RUST_LOG           = var.rust_log
         RUST_BACKTRACE     = local.rust_backtrace
-        REDIS_ENDPOINT     = var.redis_endpoint
         MG_ALPHAS          = local.alpha_grpc_connect_str
         GRAPL_SCHEMA_TABLE = var.schema_table_name
 
@@ -775,66 +764,6 @@ job "grapl-core" {
     }
   }
 
-  group "engagement-creator" {
-    count = 1
-
-    network {
-      mode = "bridge"
-      dns {
-        servers = local.dns_servers
-      }
-    }
-
-    task "engagement-creator" {
-      driver = "docker"
-
-      config {
-        image = var.container_images["engagement-creator"]
-      }
-
-      template {
-        data        = var.aws_env_vars_for_local
-        destination = "aws-env-vars-for-local.env"
-        env         = true
-      }
-
-      template {
-        data        = var.observability_env_vars
-        destination = "observability.env"
-        env         = true
-      }
-
-      env {
-        AWS_DEFAULT_REGION = var.aws_region
-
-        GRAPL_LOG_LEVEL = var.py_log_level
-
-        MG_ALPHAS = local.alpha_grpc_connect_str
-
-        SOURCE_QUEUE_URL = "fake"
-      }
-    }
-
-    service {
-      name = "engagement-creator"
-      connect {
-        sidecar_service {
-          proxy {
-            dynamic "upstreams" {
-              iterator = alpha
-              for_each = local.dgraph_alphas
-
-              content {
-                destination_name = "dgraph-alpha-${alpha.value.id}-grpc-public"
-                local_bind_port  = alpha.value.grpc_public_port
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
   group "graphql-endpoint" {
     count = 2
 
@@ -1187,10 +1116,11 @@ job "grapl-core" {
         PLUGIN_REGISTRY_KERNEL_ARTIFACT_URL             = var.plugin_registry_kernel_artifact_url
         PLUGIN_REGISTRY_ROOTFS_ARTIFACT_URL             = var.plugin_registry_rootfs_artifact_url
         PLUGIN_REGISTRY_HAX_DOCKER_PLUGIN_RUNTIME_IMAGE = var.container_images["hax-docker-plugin-runtime"]
-        PLUGIN_EXECUTION_IMAGE                          = var.container_images["generator-execution-sidecar"] # TODO: add support for analyzer too
         PLUGIN_REGISTRY_BUCKET_AWS_ACCOUNT_ID           = var.plugin_registry_bucket_aws_account_id
         PLUGIN_REGISTRY_BUCKET_NAME                     = var.plugin_registry_bucket_name
         PLUGIN_EXECUTION_OBSERVABILITY_ENV_VARS         = var.observability_env_vars
+        PLUGIN_EXECUTION_GENERATOR_SIDECAR_IMAGE        = var.container_images["generator-execution-sidecar"]
+        PLUGIN_EXECUTION_ANALYZER_SIDECAR_IMAGE         = var.container_images["analyzer-execution-sidecar"]
 
         # common Rust env vars
         RUST_BACKTRACE = local.rust_backtrace

nomad/local/grapl-local-infra.nomad

@@ -85,51 +85,12 @@ locals {
 #   local-infra services via Consul Connect; use bridge+static.
 #   This is because these services won't exist in prod.
 
-# This job is to spin up infrastructure needed to run Grapl locally (e.g. Redis) that we don't necessarily want to deploy in production (because AWS will manage it)
+# This job is to spin up infrastructure needed to run Grapl locally that we don't necessarily want to deploy in production (because AWS will manage it)
 job "grapl-local-infra" {
   datacenters = ["dc1"]
 
   type = "service"
 
-  group "redis" {
-    # Redis will be available to Nomad Jobs (sans Consul Connect)
-    # and the Host OS at localhost:6379
-    network {
-      mode = "bridge"
-      port "redis" {
-        static = 6379
-      }
-    }
-
-    task "redis" {
-      driver = "docker"
-
-      config {
-        image = "redis:latest"
-        ports = ["redis"]
-      }
-
-      service {
-        name = "redis"
-
-        check {
-          type     = "script"
-          name     = "check_redis"
-          command  = "redis-cli"
-          args     = ["ping"]
-          interval = "20s"
-          timeout  = "10s"
-
-          check_restart {
-            limit           = 2
-            grace           = "30s"
-            ignore_warnings = false
-          }
-        }
-      }
-    }
-  }
-
   group "localstack" {
     # Localstack will be available to Nomad Jobs (sans Consul Connect)
     # and the Host OS at localhost:4566