From 9c66acf136348c43509b8de2ebe0e973b78c7043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Tue, 7 Jan 2025 17:27:09 +0100 Subject: [PATCH 1/8] Allow usage of tcp socket between agent and worker --- Common/src/ArmoniK.Core.Common.csproj | 2 +- Common/src/Pollster/AgentHandler.cs | 26 +++++++++++++++---- .../src/ArmoniK.Samples.Bench.Server.csproj | 2 +- .../src/ArmoniK.Core.Common.Client.csproj | 2 +- .../ArmoniK.Core.Tests.Connectivity.csproj | 2 +- ...moniK.Samples.CrashingWorker.Client.csproj | 2 +- ...moniK.Samples.CrashingWorker.Server.csproj | 2 +- .../src/ArmoniK.Samples.HtcMock.Server.csproj | 2 +- ...s.Common.StreamWrapper.Tests.Client.csproj | 2 +- ...s.Common.StreamWrapper.Tests.Server.csproj | 2 +- terraform/modules/compute_plane/inputs.tf | 1 + terraform/modules/compute_plane/locals.tf | 12 ++++++--- terraform/variables.tf | 1 + 13 files changed, 40 insertions(+), 18 deletions(-) diff --git a/Common/src/ArmoniK.Core.Common.csproj b/Common/src/ArmoniK.Core.Common.csproj index 3d77842c9..b543ade4f 100644 --- a/Common/src/ArmoniK.Core.Common.csproj +++ b/Common/src/ArmoniK.Core.Common.csproj @@ -26,7 +26,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Common/src/Pollster/AgentHandler.cs b/Common/src/Pollster/AgentHandler.cs index bcef12d56..453ff8758 100644 --- a/Common/src/Pollster/AgentHandler.cs +++ b/Common/src/Pollster/AgentHandler.cs @@ -103,13 +103,29 @@ public AgentHandler(LoggerInit loggerInit, builder.WebHost.ConfigureKestrel(options => { - if (File.Exists(computePlaneOptions.AgentChannel.Address)) + var address = computePlaneOptions.AgentChannel.Address; + switch (computePlaneOptions.AgentChannel.SocketType) { - File.Delete(computePlaneOptions.AgentChannel.Address); + case GrpcSocketType.UnixDomainSocket: + { + if (File.Exists(address)) + { + File.Delete(address); + } + + options.ListenUnixSocket(address, + listenOptions => listenOptions.Protocols = HttpProtocols.Http2); + break; + } + case GrpcSocketType.Tcp: + var uri = new Uri(address); + options.ListenAnyIP(uri.Port, + listenOptions => listenOptions.Protocols = HttpProtocols.Http2); + + break; + default: + throw new InvalidOperationException("Socket type unknown"); } - - options.ListenUnixSocket(computePlaneOptions.AgentChannel.Address, - listenOptions => listenOptions.Protocols = HttpProtocols.Http2); }); app_ = builder.Build(); diff --git a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj index 8aaa0892e..8bdab9277 100644 --- a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj +++ b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj index 38c0a9afd..a8eca505b 100644 --- a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj +++ b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj @@ -25,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj index 3a0e4c932..f5795b56d 100644 --- a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj +++ b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj @@ -25,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj index e563a8a68..c439c37ef 100644 --- a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj +++ b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj @@ -34,7 +34,7 @@ - + diff --git a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj index 8aaa0892e..8bdab9277 100644 --- a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj +++ b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj index c7e12ca67..c9ef115fb 100644 --- a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj +++ b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj @@ -23,7 +23,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj index 9d98e5be4..a8c3551b7 100644 --- a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj +++ b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj @@ -24,7 +24,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj index dc50eb0af..d9abbf278 100644 --- a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj +++ b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/terraform/modules/compute_plane/inputs.tf b/terraform/modules/compute_plane/inputs.tf index d599e9b0e..539592776 100644 --- a/terraform/modules/compute_plane/inputs.tf +++ b/terraform/modules/compute_plane/inputs.tf @@ -12,6 +12,7 @@ variable "polling_agent" { worker_check_delay = string, shared_socket = string shared_data = string + socket_type = string }) } diff --git a/terraform/modules/compute_plane/locals.tf b/terraform/modules/compute_plane/locals.tf index 4256b20e2..7bab5b159 100644 --- a/terraform/modules/compute_plane/locals.tf +++ b/terraform/modules/compute_plane/locals.tf @@ -9,11 +9,15 @@ locals { "Amqp__PartitionId=TestPartition${local.partition_chooser}", "PubSub__PartitionId=TestPartition${local.partition_chooser}", ] + worker_tcp = format("%s://%s:%s", "http", "${var.worker.name}${var.replica_counter}","10667") + worker_socket = format("%s/%s", var.polling_agent.shared_socket,"armonik_worker.sock") + agent_tcp = format("%s://%s:%s", "http", "${var.polling_agent.name}${var.replica_counter}","10666") + agent_socket = format("%s/%s", var.polling_agent.shared_socket,"armonik_agent.sock") common_env = [ - "ComputePlane__WorkerChannel__SocketType=unixdomainsocket", - "ComputePlane__WorkerChannel__Address=${var.polling_agent.shared_socket}/armonik_worker.sock", - "ComputePlane__AgentChannel__SocketType=unixdomainsocket", - "ComputePlane__AgentChannel__Address=${var.polling_agent.shared_socket}/armonik_agent.sock", + "ComputePlane__WorkerChannel__SocketType=${var.polling_agent.socket_type}", + "ComputePlane__WorkerChannel__Address=${var.polling_agent.socket_type == "tcp" ? local.worker_tcp : local.worker_socket}", + "ComputePlane__AgentChannel__SocketType=${var.polling_agent.socket_type}", + "ComputePlane__AgentChannel__Address=${var.polling_agent.socket_type == "tcp" ? local.agent_tcp : local.agent_socket}" ] gen_env = [for k, v in var.generated_env_vars : "${k}=${v}"] polling_agent_name = "${var.polling_agent.name}${var.replica_counter}" diff --git a/terraform/variables.tf b/terraform/variables.tf index 23efed83c..1d0688049 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -126,6 +126,7 @@ variable "compute_plane" { // They will be used for both shared_socket = optional(string, "/cache") shared_data = optional(string, "/cache") + socket_type = optional(string, "unixdomainsocket") }) }) default = { From 950f995fcb30fc3ce924ad5afbb2c37927e17837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Tue, 14 Jan 2025 10:28:02 +0100 Subject: [PATCH 2/8] apply terraform fmt --- terraform/modules/compute_plane/locals.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/modules/compute_plane/locals.tf b/terraform/modules/compute_plane/locals.tf index 7bab5b159..53f432305 100644 --- a/terraform/modules/compute_plane/locals.tf +++ b/terraform/modules/compute_plane/locals.tf @@ -9,10 +9,10 @@ locals { "Amqp__PartitionId=TestPartition${local.partition_chooser}", "PubSub__PartitionId=TestPartition${local.partition_chooser}", ] - worker_tcp = format("%s://%s:%s", "http", "${var.worker.name}${var.replica_counter}","10667") - worker_socket = format("%s/%s", var.polling_agent.shared_socket,"armonik_worker.sock") - agent_tcp = format("%s://%s:%s", "http", "${var.polling_agent.name}${var.replica_counter}","10666") - agent_socket = format("%s/%s", var.polling_agent.shared_socket,"armonik_agent.sock") + worker_tcp = format("%s://%s:%s", "http", "${var.worker.name}${var.replica_counter}", "10667") + worker_socket = format("%s/%s", var.polling_agent.shared_socket, "armonik_worker.sock") + agent_tcp = format("%s://%s:%s", "http", "${var.polling_agent.name}${var.replica_counter}", "10666") + agent_socket = format("%s/%s", var.polling_agent.shared_socket, "armonik_agent.sock") common_env = [ "ComputePlane__WorkerChannel__SocketType=${var.polling_agent.socket_type}", "ComputePlane__WorkerChannel__Address=${var.polling_agent.socket_type == "tcp" ? local.worker_tcp : local.worker_socket}", From a4538bd74ab664dab625ef36c428afcb3b6dccd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Wed, 15 Jan 2025 16:05:40 +0100 Subject: [PATCH 3/8] modify deployment to ease testing --- .github/workflows/build.yml | 62 +++++++++++++++-------- justfile | 3 +- terraform/main.tf | 1 + terraform/modules/compute_plane/inputs.tf | 5 +- terraform/modules/compute_plane/locals.tf | 8 +-- terraform/variables.tf | 11 +++- 6 files changed, 62 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 457587805..9669c29ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -441,20 +441,40 @@ jobs: strategy: fail-fast: false matrix: - queue: - - activemq - - rabbitmq - - rabbitmq091 - - pubsub - - sqs - object: - - redis - - minio - - embed - log-level: - - Information - - Verbose - name: HtcMock ${{ matrix.queue }} ${{ matrix.object }} ${{ matrix.log-level }} + target: + - { queue: activemq, object: redis, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq, object: redis, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq091, object: redis, log-level: Information, socket_type: unixdomainsocket } + - { queue: pubsub, object: redis, log-level: Information, socket_type: unixdomainsocket } + - { queue: sqs, object: redis, log-level: Information, socket_type: unixdomainsocket } + + - { queue: activemq, object: redis, log-level: Information, socket_type: tcp } + - { queue: rabbitmq, object: redis, log-level: Information, socket_type: tcp } + - { queue: rabbitmq091, object: redis, log-level: Information, socket_type: tcp } + - { queue: pubsub, object: redis, log-level: Information, socket_type: tcp } + - { queue: sqs, object: redis, log-level: Information, socket_type: tcp } + + - { queue: activemq, object: redis, log-level: Verbose, socket_type: unixdomainsocket } + - { queue: rabbitmq, object: redis, log-level: Verbose, socket_type: unixdomainsocket } + - { queue: rabbitmq091, object: redis, log-level: Verbose, socket_type: unixdomainsocket } + - { queue: pubsub, object: redis, log-level: Verbose, socket_type: unixdomainsocket } + - { queue: sqs, object: redis, log-level: Verbose, socket_type: unixdomainsocket } + + - { queue: activemq, object: minio, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq, object: minio, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq091, object: minio, log-level: Information, socket_type: unixdomainsocket } + - { queue: pubsub, object: minio, log-level: Information, socket_type: unixdomainsocket } + - { queue: sqs, object: minio, log-level: Information, socket_type: unixdomainsocket } + + - { queue: activemq, object: embed, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq, object: embed, log-level: Information, socket_type: unixdomainsocket } + - { queue: rabbitmq091, object: embed, log-level: Information, socket_type: unixdomainsocket } + - { queue: pubsub, object: embed, log-level: Information, socket_type: unixdomainsocket } + - { queue: sqs, object: embed, log-level: Information, socket_type: unixdomainsocket } + + - { queue: activemq, object: redis, log-level: Information, socket_type: unixdomainsocket } + + name: HtcMock ${{ matrix.target.queue }} ${{ matrix.target.object }} ${{ matrix.target.log-level }} ${{ matrix.target.socket_type }} steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 @@ -478,7 +498,7 @@ jobs: - name: Deploy Core run: | MONITOR_PREFIX="monitor/deploy/" tools/retry.sh -w 30 -- tools/monitor.sh \ - just log_level=${{ matrix.log-level }} tag=${VERSION} queue=${{ matrix.queue }} object=${{ matrix.object }} worker=htcmock deploy + just log_level=${{ matrix.target.log-level }} tag=${VERSION} queue=${{ matrix.target.queue }} object=${{ matrix.target.object }} socket_type=${{ matrix.target.socket_type }} worker=htcmock deploy sleep 10 - name: Print And Time Metrics @@ -547,7 +567,7 @@ jobs: - name: Run HtcMock test 1000 tasks 1 level timeout-minutes: 3 - if: ${{ matrix.log-level != 'Verbose' }} + if: ${{ matrix.target.log-level != 'Verbose' }} run: | MONITOR_PREFIX="monitor/htcmock-1000-1/" tools/monitor.sh \ docker run --net armonik_network --rm \ @@ -562,7 +582,7 @@ jobs: - name: Run HtcMock test 1000 tasks 4 levels timeout-minutes: 3 - if: ${{ matrix.log-level != 'Verbose' }} + if: ${{ matrix.target.log-level != 'Verbose' }} run: | MONITOR_PREFIX="monitor/htcmock-1000-4/" tools/monitor.sh \ docker run --net armonik_network --rm \ @@ -586,8 +606,8 @@ jobs: run: | export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} - docker cp fluentd:/armonik-logs.json - | gzip -c | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}.json.gz - tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-monitor.tar.gz + docker cp fluentd:/armonik-logs.json - | gzip -c | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.target.queue }}-${{ matrix.target.object }}-${{ matrix.target.log-level }}-${{ matrix.target.socket_type }}.json.gz + tar -czf - monitor/ | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.target.queue }}-${{ matrix.target.object }}-${{ matrix.target.log-level }}-${{ matrix.target.socket_type }}-monitor.tar.gz - name: Collect docker container logs uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e # v2 @@ -599,14 +619,14 @@ jobs: run: | export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} - tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-container-logs.tar.gz + tar -cvf - ./container-logs | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.target.queue }}-${{ matrix.target.object }}-${{ matrix.target.log-level }}-${{ matrix.target.socket_type }}-container-logs.tar.gz - name: Export and upload database if: always() run: | export AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} bash tools/export_mongodb.sh - tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.queue }}-${{ matrix.object }}-${{ matrix.log-level }}-database.tar.gz + tar -cvf - *.json | aws s3 cp - s3://${{ secrets.AWS_LOG_BUCKET_NAME }}/core-pipeline/${{ github.run_number }}/${{ github.run_attempt }}/htcmock-${{ matrix.target.queue }}-${{ matrix.target.object }}-${{ matrix.target.log-level }}-${{ matrix.target.socket_type }}-database.tar.gz testWindowsDocker: needs: diff --git a/justfile b/justfile index dad19ad66..424083a90 100644 --- a/justfile +++ b/justfile @@ -20,6 +20,7 @@ ingress := "true" prometheus := "true" grafana := "true" seq := "true" +socket_type := "unixdomainsocket" # Export them as terraform environment variables export TF_VAR_core_tag := tag @@ -30,7 +31,7 @@ export TF_VAR_num_partitions := partitions export TF_VAR_enable_grafana := grafana export TF_VAR_enable_seq := seq export TF_VAR_enable_prometheus := prometheus - +export TF_VAR_socket_type := socket_type # Sets the queue export TF_VAR_queue_storage := if queue == "rabbitmq" { diff --git a/terraform/main.tf b/terraform/main.tf index f57495154..07d717d98 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -131,6 +131,7 @@ module "compute_plane" { core_tag = local.compute_plane.tag polling_agent = local.compute_plane.polling_agent worker = local.compute_plane.worker + socket_type = var.socket_type generated_env_vars = local.environment volumes = local.volumes network = docker_network.armonik.id diff --git a/terraform/modules/compute_plane/inputs.tf b/terraform/modules/compute_plane/inputs.tf index 9c570a418..e80463c12 100644 --- a/terraform/modules/compute_plane/inputs.tf +++ b/terraform/modules/compute_plane/inputs.tf @@ -2,6 +2,10 @@ variable "core_tag" { type = string } +variable "socket_type" { + type = string +} + variable "polling_agent" { type = object({ name = string, @@ -12,7 +16,6 @@ variable "polling_agent" { worker_check_delay = string, shared_socket = string shared_data = string - socket_type = string }) } diff --git a/terraform/modules/compute_plane/locals.tf b/terraform/modules/compute_plane/locals.tf index 53f432305..fa9bd840f 100644 --- a/terraform/modules/compute_plane/locals.tf +++ b/terraform/modules/compute_plane/locals.tf @@ -14,10 +14,10 @@ locals { agent_tcp = format("%s://%s:%s", "http", "${var.polling_agent.name}${var.replica_counter}", "10666") agent_socket = format("%s/%s", var.polling_agent.shared_socket, "armonik_agent.sock") common_env = [ - "ComputePlane__WorkerChannel__SocketType=${var.polling_agent.socket_type}", - "ComputePlane__WorkerChannel__Address=${var.polling_agent.socket_type == "tcp" ? local.worker_tcp : local.worker_socket}", - "ComputePlane__AgentChannel__SocketType=${var.polling_agent.socket_type}", - "ComputePlane__AgentChannel__Address=${var.polling_agent.socket_type == "tcp" ? local.agent_tcp : local.agent_socket}" + "ComputePlane__WorkerChannel__SocketType=${var.socket_type}", + "ComputePlane__WorkerChannel__Address=${var.socket_type == "tcp" ? local.worker_tcp : local.worker_socket}", + "ComputePlane__AgentChannel__SocketType=${var.socket_type}", + "ComputePlane__AgentChannel__Address=${var.socket_type == "tcp" ? local.agent_tcp : local.agent_socket}" ] gen_env = [for k, v in var.generated_env_vars : "${k}=${v}"] polling_agent_name = "${var.polling_agent.name}${var.replica_counter}" diff --git a/terraform/variables.tf b/terraform/variables.tf index 1d0688049..414764fc7 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -106,6 +106,16 @@ variable "worker_image" { default = "dockerhubaneo/armonik_core_htcmock_test_worker" } +variable "socket_type" { + type = string + description = "Socket type used by agent and worker to communicate" + validation { + condition = can(regex("^(unixdomainsocket|tcp)$", var.socket_type)) + error_message = "Socket must be either unixdomainsocket or tcp" + } + default = "unixdomaisocket" +} + variable "compute_plane" { type = object({ worker = object({ @@ -126,7 +136,6 @@ variable "compute_plane" { // They will be used for both shared_socket = optional(string, "/cache") shared_data = optional(string, "/cache") - socket_type = optional(string, "unixdomainsocket") }) }) default = { From b67b82cd7b68805dbfef1ff63e89f2227bf4281e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Wed, 15 Jan 2025 16:23:09 +0100 Subject: [PATCH 4/8] fix typo --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 414764fc7..23bd69540 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -113,7 +113,7 @@ variable "socket_type" { condition = can(regex("^(unixdomainsocket|tcp)$", var.socket_type)) error_message = "Socket must be either unixdomainsocket or tcp" } - default = "unixdomaisocket" + default = "unixdomainsocket" } variable "compute_plane" { From 1330940bc726571bc99b3673e375c119a94da635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Thu, 16 Jan 2025 11:35:42 +0100 Subject: [PATCH 5/8] inline doc --- justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/justfile b/justfile index 424083a90..9f70566c6 100644 --- a/justfile +++ b/justfile @@ -190,6 +190,8 @@ _usage: local_images: Let terraform build the docker images locally (default = false) + socket_type: Socket type used by agent and worker to communicate (default = unixdomainsocket) + IMPORTANT: In order to properly destroy the resources created you should call the recipe destroy with the same parameters used for deploy EOF From f640150c09e79fce045cecc4d4105ee98c696ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Thu, 16 Jan 2025 15:09:28 +0100 Subject: [PATCH 6/8] This commit introduces support to: - Mount a separate volume for the shared data - Do not mount the volume destinated for the unix socket if tcp is chosen as socket type for the communication agent/worker --- Dockerfile | 2 +- terraform/modules/compute_plane/locals.tf | 1 + terraform/modules/compute_plane/main.tf | 34 +++++++++++++++++++---- terraform/variables.tf | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 780b0e63f..c104d91ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM mcr.microsoft.com/dotnet/aspnet:8.0 as base-linux RUN groupadd --gid 5000 armonikuser && useradd --home-dir /home/armonikuser --create-home --uid 5000 --gid 5000 --shell /bin/sh --skel /dev/null armonikuser -RUN mkdir /cache /local_storage && chown armonikuser: /cache /local_storage +RUN mkdir /cache /local_storage /comm && chown armonikuser: /cache /local_storage /comm USER armonikuser ENTRYPOINT [ "dotnet" ] diff --git a/terraform/modules/compute_plane/locals.tf b/terraform/modules/compute_plane/locals.tf index fa9bd840f..1e6466539 100644 --- a/terraform/modules/compute_plane/locals.tf +++ b/terraform/modules/compute_plane/locals.tf @@ -21,4 +21,5 @@ locals { ] gen_env = [for k, v in var.generated_env_vars : "${k}=${v}"] polling_agent_name = "${var.polling_agent.name}${var.replica_counter}" + socket_vol = var.socket_type == "tcp" ? {} : { (var.polling_agent.shared_socket) = one(docker_volume.socket_vol[*].name) } } diff --git a/terraform/modules/compute_plane/main.tf b/terraform/modules/compute_plane/main.tf index b6202b1b6..fefb7757d 100644 --- a/terraform/modules/compute_plane/main.tf +++ b/terraform/modules/compute_plane/main.tf @@ -1,5 +1,11 @@ resource "docker_volume" "socket_vol" { - name = "socket_vol${var.replica_counter}" + count = var.socket_type == "tcp" ? 0 : 1 + name = "socket_vol${var.replica_counter}" + +} + +resource "docker_volume" "comm_vol" { + name = "comm_vol${var.replica_counter}" } resource "docker_image" "worker" { @@ -27,10 +33,19 @@ resource "docker_container" "worker" { external = var.worker.port + var.replica_counter } + dynamic "mounts" { + for_each = local.socket_vol + content { + type = "volume" + target = mounts.key + source = mounts.value + } + } + mounts { type = "volume" - target = var.polling_agent.shared_socket - source = docker_volume.socket_vol.name + target = var.polling_agent.shared_data + source = docker_volume.comm_vol.name } } @@ -59,8 +74,17 @@ resource "docker_container" "polling_agent" { mounts { type = "volume" - target = var.polling_agent.shared_socket - source = docker_volume.socket_vol.name + target = var.polling_agent.shared_data + source = docker_volume.comm_vol.name + } + + dynamic "mounts" { + for_each = local.socket_vol + content { + type = "volume" + target = mounts.key + source = mounts.value + } } restart = "unless-stopped" diff --git a/terraform/variables.tf b/terraform/variables.tf index 23bd69540..0d588d008 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -135,7 +135,7 @@ variable "compute_plane" { // env for the agent and env for the worker // They will be used for both shared_socket = optional(string, "/cache") - shared_data = optional(string, "/cache") + shared_data = optional(string, "/comm") }) }) default = { From dfeaf179e3618f12859e39145d0807a56eeb5095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Thu, 16 Jan 2025 16:12:36 +0100 Subject: [PATCH 7/8] correct shared_data mount point for windows build --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index 9f70566c6..82a12110a 100644 --- a/justfile +++ b/justfile @@ -138,7 +138,7 @@ export TF_VAR_mongodb_params:= if os_family() == "windows" { } export TF_VAR_compute_plane:= if os_family() == "windows" { - '{ "polling_agent" : { "image" : "' + image_polling_agent + '", "shared_socket" : "c:/cache", "shared_data" : "c:/cache" }, "worker" = {}}' + '{ "polling_agent" : { "image" : "' + image_polling_agent + '", "shared_socket" : "c:/cache", "shared_data" : "c:/comm" }, "worker" = {}}' } else { '{ "polling_agent" : { "image" : "' + image_polling_agent + '" }, "worker" = {}}' } From b460bf9e03c812fef39b17a4da368cd8a205a281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Fonseca?= Date: Fri, 17 Jan 2025 14:29:13 +0100 Subject: [PATCH 8/8] swich to release version 3.23.0 of API --- Common/src/ArmoniK.Core.Common.csproj | 2 +- Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj | 2 +- Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj | 2 +- Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj | 2 +- .../Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj | 2 +- .../Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj | 2 +- Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj | 2 +- .../ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj | 2 +- .../ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Common/src/ArmoniK.Core.Common.csproj b/Common/src/ArmoniK.Core.Common.csproj index b543ade4f..1a03d09a9 100644 --- a/Common/src/ArmoniK.Core.Common.csproj +++ b/Common/src/ArmoniK.Core.Common.csproj @@ -26,7 +26,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj index 8bdab9277..92108a86c 100644 --- a/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj +++ b/Tests/Bench/Server/src/ArmoniK.Samples.Bench.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj index a8eca505b..d30f74320 100644 --- a/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj +++ b/Tests/Common/Client/src/ArmoniK.Core.Common.Client.csproj @@ -25,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj index f5795b56d..4a5db1975 100644 --- a/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj +++ b/Tests/Connectivity/src/ArmoniK.Core.Tests.Connectivity.csproj @@ -25,7 +25,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj index c439c37ef..20e9592c9 100644 --- a/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj +++ b/Tests/CrashingWorker/Client/src/ArmoniK.Samples.CrashingWorker.Client.csproj @@ -34,7 +34,7 @@ - + diff --git a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj index 8bdab9277..92108a86c 100644 --- a/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj +++ b/Tests/CrashingWorker/Server/src/ArmoniK.Samples.CrashingWorker.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj index c9ef115fb..8568d2875 100644 --- a/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj +++ b/Tests/HtcMock/Server/src/ArmoniK.Samples.HtcMock.Server.csproj @@ -23,7 +23,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj index a8c3551b7..b40a0fa80 100644 --- a/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj +++ b/Tests/Stream/Client/ArmoniK.Extensions.Common.StreamWrapper.Tests.Client.csproj @@ -24,7 +24,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj index d9abbf278..d0e001770 100644 --- a/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj +++ b/Tests/Stream/Server/ArmoniK.Extensions.Common.StreamWrapper.Tests.Server.csproj @@ -22,7 +22,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive