Use host's private IP as dns server for containers

With the move of dnsmasq to the host, we now need a way to tell the containers to use the host for their dns lookups if they want to use consul dns. Typically on a host you would just add a nameserver for 127.0.0.1, but that doesn't work because in the container 127.0.0.1 points to the container not the host. Ideally we'd just be passing in attr.unique.network.ip-address, but unfortunately, variable interpolation is only available for network.dns as of Nomad 1.3 per hashicorp/nomad#11851 So instead we're stuck doing grabbing an IP from `hostname -I` and passing that in as an environment variable. This may actually be for the best if we end up setting a distinct dns service in production
grapl-security · Apr 9, 2022 · ff69207 · ff69207
1 parent cc12efc
commit ff69207
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -22,6 +22,8 @@ DIST_DIR = $(GRAPL_ROOT)/dist
 COMPOSE_USER=${UID}:${GID}
 COMPOSE_IGNORE_ORPHANS=1
 COMPOSE_PROJECT_NAME ?= grapl
+# Get a non-loopback private ip for the host. Order is not guaranteed
+LOCAL_HOST_IP = $(shell hostname --all-ip-addresses | awk '{ print $$1 }')
 
 export
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -45,6 +45,7 @@ services:
       PULUMI_CONFIG_PASSPHRASE: local-grapl-passphrase
       DOCKER_USER: "${UID}:${GID}"
       GRAPL_ROOT: "${GRAPL_ROOT}"
+      LOCAL_HOST_IP: "${LOCAL_HOST_IP}"
       # Other environment variables like MG_ALPHAS are passed in via
       # Pulumi.local-grapl.yaml
     extra_hosts:

diff --git a/nomad/grapl-core.nomad b/nomad/grapl-core.nomad
@@ -274,6 +274,11 @@ variable "tracing_endpoint" {
   default = ""
 }
 
+variable "dns_server" {
+  type        = string
+  description = "The network.dns.server value. This should be equivalent to the host's ip in order to communicate with dnsmasq and allow consul dns to be available from within containers. This can be replaced as of Nomad 1.3.0 with variable interpolation per https://github.com/hashicorp/nomad/issues/11851."
+}
+
 locals {
   dgraph_zero_grpc_private_port_base  = 5080
   dgraph_alpha_grpc_private_port_base = 7080
@@ -318,6 +323,7 @@ locals {
 
   # enabled
   rust_backtrace = 1
+  dns_servers    = [var.dns_server]
 }
 
 job "grapl-core" {
@@ -343,6 +349,9 @@ job "grapl-core" {
   group "dgraph-zero-0" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "dgraph-zero" {
@@ -404,6 +413,9 @@ job "grapl-core" {
     content {
       network {
         mode = "bridge"
+        dns {
+          servers = local.dns_servers
+        }
         port "healthcheck" {
           to = -1
         }
@@ -503,6 +515,9 @@ job "grapl-core" {
     content {
       network {
         mode = "bridge"
+        dns {
+          servers = local.dns_servers
+        }
         port "healthcheck" {
           to = -1
         }
@@ -628,6 +643,9 @@ job "grapl-core" {
 
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "graph-merger" {
@@ -686,6 +704,9 @@ job "grapl-core" {
 
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "node-identifier" {
@@ -728,6 +749,9 @@ job "grapl-core" {
 
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "node-identifier-retry" {
@@ -766,6 +790,12 @@ job "grapl-core" {
   }
 
   group "analyzer-dispatcher" {
+    network {
+      mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
+    }
 
     task "analyzer-dispatcher" {
       driver = "docker"
@@ -807,6 +837,9 @@ job "grapl-core" {
   group "analyzer-executor" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "analyzer-executor" {
@@ -867,6 +900,9 @@ job "grapl-core" {
   group "engagement-creator" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "engagement-creator" {
@@ -920,6 +956,9 @@ job "grapl-core" {
   group "graphql-endpoint" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
       port "graphql-endpoint-port" {}
     }
 
@@ -976,6 +1015,9 @@ job "grapl-core" {
   group "model-plugin-deployer" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
       port "model-plugin-deployer" {
       }
     }
@@ -1008,6 +1050,9 @@ job "grapl-core" {
   group "web-ui" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
 
       port "web-ui-port" {
       }
@@ -1066,6 +1111,9 @@ job "grapl-core" {
   group "sysmon-generator" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "sysmon-generator" {
@@ -1098,6 +1146,9 @@ job "grapl-core" {
   group "osquery-generator" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     task "osquery-generator" {
@@ -1130,6 +1181,9 @@ job "grapl-core" {
   group "organization-management" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
       port "organization-management-port" {
       }
     }
@@ -1176,6 +1230,9 @@ job "grapl-core" {
   group "plugin-registry" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
 
       port "plugin-registry-port" {
       }
@@ -1230,6 +1287,9 @@ job "grapl-core" {
   group "plugin-work-queue" {
     network {
       mode = "bridge"
+      dns {
+        servers = local.dns_servers
+      }
 
       port "plugin-work-queue-port" {
       }

diff --git a/nomad/grapl-ingress.nomad b/nomad/grapl-ingress.nomad
@@ -1,5 +1,11 @@
+variable "dns_server" {
+  type        = string
+  description = "The network.dns.server value. This should be equivalent to the host's ip in order to communicate with dnsmasq and allow consul dns to be available from within containers. This can be replaced as of Nomad 1.3.0 with variable interpolation per https://github.com/hashicorp/nomad/issues/11851."
+}
+
 locals {
   web_ui_port = 1234
+  dns_servers = [var.dns_server]
 }
 
 job "grapl-ingress" {
@@ -13,6 +19,9 @@ job "grapl-ingress" {
 
     network {
       mode = "host"
+      dns {
+        servers = local.dns_servers
+      }
     }
 
     service {

diff --git a/pulumi/grapl/__main__.py b/pulumi/grapl/__main__.py
@@ -230,6 +230,7 @@ def main() -> None:
         aws_env_vars_for_local=aws_env_vars_for_local,
         aws_region=aws.get_region().name,
         container_images=_container_images(artifacts),
+        dns_server=config.LOCAL_HOST_IP,
         engagement_creator_queue=engagement_creator_queue.main_queue_url,
         graph_merger_queue=graph_merger_queue.main_queue_url,
         graph_merger_dead_letter_queue=graph_merger_queue.dead_letter_queue_url,
@@ -320,7 +321,7 @@ def main() -> None:
     nomad_grapl_ingress = NomadJob(
         "grapl-ingress",
         jobspec=path_from_root("nomad/grapl-ingress.nomad").resolve(),
-        vars={},
+        vars={"dns_server": config.LOCAL_HOST_IP},
         opts=pulumi.ResourceOptions(
             provider=nomad_provider,
             # This dependson ensures we've switched the web-ui protocol to http instead of tcp prior. Otherwise there's

diff --git a/pulumi/infra/config.py b/pulumi/infra/config.py
@@ -25,6 +25,10 @@
 # note: this ${} is interpolated inside Nomad
 HOST_IP_IN_NOMAD: Final[str] = "${attr.unique.network.ip-address}"
 
+# This is equivalent to what "${attr.unique.network.ip-address}" resolves to but is used for cases where variable
+# interpolation is not available such as network.dns prior to Nomad 1.3.0
+LOCAL_HOST_IP: Final[str] = os.environ["LOCAL_HOST_IP"]
+
 
 def to_bool(input: Optional[Union[str, bool]]) -> Optional[bool]:
     if isinstance(input, bool):