load-test: windows images should still be built in each test

Signed-off-by: Wesley Pettit <[email protected]>

buildspec_integ.yml

@@ -10,8 +10,8 @@ phases:
     commands:
       # Enforce STS regional endpoints
       - export AWS_STS_REGIONAL_ENDPOINTS=regional
-      - export CW_INTEG_VALIDATOR_IMAGE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator
-      - export S3_INTEG_VALIDATOR_IMAGE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator
+      - export CW_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator
+      - export S3_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator
       # Get the default credentials and set as environment variables
       - 'CREDS=`curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`'
       - 'export AWS_ACCESS_KEY_ID=`echo $CREDS | jq -r .AccessKeyId`'

integ/integ.sh

@@ -2,9 +2,17 @@
 export AWS_REGION="us-west-2"
 export PROJECT_ROOT="$(pwd)"
 export VOLUME_MOUNT_CONTAINER="/out"
+export ARCHITECTURE=$(uname -m)
+# For arm, uname evaluates to 'aarch64' but everywhere else in the pipline
+# we use 'arm64'
+if [ "$ARCHITECTURE" = "aarch64" ]; then
+    ARCHITECTURE="arm64"
+fi
+
+export CW_INTEG_VALIDATOR_IMAGE="${CW_INTEG_VALIDATOR_IMAGE_BASE}-${ARCHITECTURE}"
+export S3_INTEG_VALIDATOR_IMAGE="${S3_INTEG_VALIDATOR_IMAGE_BASE}-${ARCHITECTURE}"
 
 test_cloudwatch() {
-	export ARCHITECTURE=$(uname -m)
 	export LOG_GROUP_NAME="fluent-bit-integ-test-${ARCHITECTURE}"
 	# Tag is used to name the log stream; each test run has a unique (random) log stream name
 	export TAG=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 10)
@@ -126,7 +134,6 @@ test_kinesis_firehose() {
 
 test_s3() {
 	# different S3 prefix for each test
-	export ARCHITECTURE=$(uname -m)
 	export S3_PREFIX_PUT_OBJECT="logs/${ARCHITECTURE}/putobject"
 	export S3_PREFIX_MULTIPART="logs/${ARCHITECTURE}/multipart"
 	# Tag is used in the s3 keys; each test run has a unique (random) tag

integ/run-integ.ps1

@@ -75,8 +75,8 @@ $env:FLUENT_BIT_IMAGE = $FluentBitImage
 $env:TAG = -join ((65..90) + (97..122) | Get-Random -Count 10 | % {[char]$_})
 # AWS_FOR_FLUENT_BIT_CONTAINER_NAME is the name for the fluent-bit container ran for each service.
 $env:AWS_FOR_FLUENT_BIT_CONTAINER_NAME = "aws-for-fluent-bit-$($env:TAG)"
-$env:CW_INTEG_VALIDATOR_IMAGE = "906394416424.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator-windows"
-$env:S3_INTEG_VALIDATOR_IMAGE = "906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator-windows"
+$env:CW_INTEG_VALIDATOR_IMAGE = "cw-integ-validator-windows:latest"
+$env:S3_INTEG_VALIDATOR_IMAGE = "s3-integ-validator-windows:latest"
 
 # Windows specific settings.
 $env:ARCHITECTURE= "x86-64"
@@ -114,6 +114,18 @@ Function Install-Package {
         Invoke-WebRequest "https://github.com/docker/compose/releases/download/v${DockerComposeVersion}/docker-compose-Windows-x86_64.exe" -UseBasicParsing -OutFile $Env:ProgramFiles\Docker\docker-compose.exe
     }
     Write-Host $( docker-compose version )
+
+    # build validator images
+    docker build -t $env:CW_INTEG_VALIDATOR_IMAGE --file "${IntegTestRoot}/validate_cloudwatch/Dockerfile" "${IntegTestRoot}/validate_cloudwatch/"
+    if ($LASTEXITCODE)
+    {
+        throw ("Failed to build $env:CW_INTEG_VALIDATOR_IMAGE image")
+    }
+    docker build -t $env:S3_INTEG_VALIDATOR_IMAGE --file "${IntegTestRoot}/s3/Dockerfile.windows" "${IntegTestRoot}/s3/"
+    if ($LASTEXITCODE)
+    {
+        throw ("Failed to build $env:S3_INTEG_VALIDATOR_IMAGE image")
+    }
 }
 
 Function Test-Command {

integ/s3/updating_validator_image.md

@@ -1,19 +1,21 @@
 ## S3 Validator Image
 
-The S3 Validator image exists in our account as:
+The S3 Validator image exists in our account for linux as:
 
 ```
 906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator:arm64
 906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator:x86_64
-906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator-windows:x86_64
 ```
 
 If we need to perform a change or dependency update, build the image and push it to the account with that name.
 
 The base image URI is set as an env var `S3_INTEG_VALIDATOR_IMAGE` in:
 - Linux: buildspec_integ.yml
-- Windows: run-integ.ps1
 
 The tag is the value of `${ARCHITECTURE}`
 
 The env var is used in the `docker-compose.validate-and-clean.yml` files for firehose, kinesis, and S3 tests.
+
+Storing the images in the account is more efficient than re-building them for each integration test execution. It ensures builds can not be blocked by external dependencies like pypi or go. 
+
+For windows, we currently still build the images locally. This is partly because for windows, the base image changes with every monthly patch tuesday. The current windows base image is the largest part of the image, and is present on the AMI. Therefore, building it locally each test run can use that base without a large download. In addition, windows requires separate images for server version 2022 vs 2019 (and others if we support them in the future). The images are built in `run-integ.ps1`.

integ/test_cloudwatch/docker-compose.validate.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     cloudwatch_validator:
-        image: ${CW_INTEG_VALIDATOR_IMAGE}:${ARCHITECTURE}
+        image: ${CW_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_firehose/docker-compose.validate-and-clean-s3.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     validate-s3:
-        image: ${S3_INTEG_VALIDATOR_IMAGE}:${ARCHITECTURE}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_kinesis/docker-compose.validate-and-clean-s3.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     validate-s3:
-        image: ${S3_INTEG_VALIDATOR_IMAGE}:${ARCHITECTURE}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_s3/docker-compose.validate-s3-multipart.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     validate-s3-multipart:
-        image: ${S3_INTEG_VALIDATOR_IMAGE}:${ARCHITECTURE}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_s3/docker-compose.validate-s3-putobject.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     validate-s3-put-object:
-        image: ${S3_INTEG_VALIDATOR_IMAGE}:${ARCHITECTURE}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/validate_cloudwatch/updating_validator_image.md

@@ -1,19 +1,21 @@
 ## CW Validator Image
 
-The CW Validator image exists in our account as:
+The CW Validator image exists in our account for linux as:
 
 ```
 906394416424.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator:arm64
 906394416424.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator:x86_64
-906394416424.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator-windows:x86_64
 ```
 
 If we need to perform a change or dependency update, build the image and push it to the account with that name.
 
 The base image URI is set as an env var `CW_INTEG_VALIDATOR_IMAGE` in:
 - Linux: buildspec_integ.yml
-- Windows: run-integ.ps1
 
 The tag is the value of `${ARCHITECTURE}`
 
-The env var is used in the `docker-compose.validate-and-clean.yml` files for firehose, kinesis, and S3 tests.
+The env var is used in the `docker-compose.validate.yml` files for firehose, kinesis, and S3 tests.
+
+Storing the images in the account is more efficient than re-building them for each integration test execution. It ensures builds can not be blocked by external dependencies like pypi or go. 
+
+For windows, we currently still build the images locally. This is partly because for windows, the base image changes with every monthly patch tuesday. The current windows base image is the largest part of the image, and is present on the AMI. Therefore, building it locally each test run can use that base without a large download. In addition, windows requires separate images for server version 2022 vs 2019 (and others if we support them in the future). The images are built in `run-integ.ps1`.