Add Lightsail terraform deployment module (#149)

.env.template

@@ -0,0 +1,3 @@
+WHITEFOX_TEST_AWS_REGION=eu-west-1
+WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY=weashdfgvkeuajysmjtsrq2384we
+WHITEFOX_TEST_AWS_ACCESS_KEY_ID=AKIAYTRD345SDUY

.github/workflows/compile.yaml

@@ -32,7 +32,7 @@ jobs:
               if [ "$RUNNER_OS" == "Windows" ]; then
                 export HADOOP_HOME="$(pwd)/.github/workflows/hadoop3-win-binaries"
               fi
-              ./gradlew build testNative --no-daemon
+              ./gradlew build testNative --no-daemon --rerun-tasks
               ./gradlew server:app:printVersion --no-daemon -q
         shell: bash
       - name: Run integration test
@@ -42,6 +42,8 @@ jobs:
           WHITEFOX_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.WHITEFOX_AWS_ACCESS_KEY_ID }}
           WHITEFOX_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.WHITEFOX_AWS_SECRET_ACCESS_KEY }}
         run: |
+          WHITEFOX_SERVER_AUTHENTICATION_ENABLED=TRUE \
+          WHITEFOX_SERVER_AUTHENTICATION_BEARERTOKEN=token \
           java -jar server/app/build/quarkus-app/quarkus-run.jar &
           ./gradlew :client-spark:clientSparkTest --no-daemon
           kill -9 %1

client-spark/build.gradle.kts

@@ -38,7 +38,7 @@ tasks.getByName<Test>("test") {
 }
 
 tasks.withType<Test> {
-    environment = env.allVariables
+    environment = env.allVariables()
     systemProperty ("java.util.logging.manager", "java.util.logging.LogManager") //TODO modularize the whitefox-conventions plugin
 }
 

client-spark/src/test/java/io/whitefox/api/client/ApiUtils.java

@@ -0,0 +1,112 @@
+package io.whitefox.api.client;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.whitefox.api.utils.ApiClient;
+import io.whitefox.api.utils.ApiException;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.function.Supplier;
+
+public class ApiUtils {
+    /**
+     * Returns the result of the call of the first argument (f) unless it throws an ApiException with HTTP status code 409,
+     * in that case, returns the result of the call of the second argument (defaultValue).
+     * If defaultValue is not dynamic, you can use also {@link ApiUtils#recoverConflict recoverConflinct}.
+     */
+    public static <T> T recoverConflictLazy(Supplier<T> f, Supplier<T> defaultValue) {
+        try {
+            return f.get();
+        } catch (ApiException e) {
+            if (e.getCode() == 409) {
+                return defaultValue.get();
+            } else {
+                throw e;
+            }
+        }
+    }
+
+    /**
+     * Returns the result of the call of the first argument (f) unless it throws an ApiException with HTTP status code 409,
+     * in that case, returns the second argument (defaultValue).
+     * If defaultValue is dynamic, you can use also {@link ApiUtils#recoverConflictLazy recoverConflictLazy}.
+     */
+    public static <T> T recoverConflict(Supplier<T> f, T defaultValue) {
+        return recoverConflictLazy(f, new Supplier<T>() {
+            @Override
+            public T get() {
+                return defaultValue;
+            }
+        });
+    }
+
+    /**
+     * Calls the first argument (f), if the call throws an ApiException with HTTP status code 409 will swallow the exception.
+     */
+    public static <T> void ignoreConflict(Supplier<T> f) {
+        recoverConflict(f, null);
+    }
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    public static final String ENDPOINT_FIELD_NAME = "endpoint";
+    public static final String BEARER_TOKEN_FIELD_NAME = "bearerToken";
+
+    /**
+     * Reads a resource named as the parameter, parses it following
+     * <a href="https://github.com/delta-io/delta-sharing/blob/main/PROTOCOL.md#profile-file-format">delta sharing specification</a>
+     * and configures an {@link ApiClient ApiClient} accordingly.
+     */
+    public static ApiClient configureApiClientFromResource(String resourceName) {
+        try (InputStream is = ApiUtils.class.getClassLoader().getResourceAsStream(resourceName)) {
+            return configureClientInternal(objectMapper.reader().readTree(is));
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Cannot read %s", resourceName), e);
+        } catch (NullPointerException e) {
+            throw new RuntimeException(String.format("Cannot find resource %s", resourceName), e);
+        }
+    }
+
+    /**
+     * Reads a local file named as the parameter, parses it following
+     * <a href="https://github.com/delta-io/delta-sharing/blob/main/PROTOCOL.md#profile-file-format">delta sharing specification</a>
+     * and configures an {@link ApiClient ApiClient} accordingly.
+     */
+    public static ApiClient configureClientFromFile(File file) {
+        try (InputStream is = new FileInputStream(file)) {
+            return configureClientInternal(objectMapper.reader().readTree(is));
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Cannot read %s", file), e);
+        }
+    }
+
+    private static ApiClient configureClientInternal(JsonNode conf) {
+        var endpointText = getRequiredField(conf, ENDPOINT_FIELD_NAME).asText();
+        var token = getRequiredField(conf, BEARER_TOKEN_FIELD_NAME).asText();
+        try {
+            var endpoint = new URI(endpointText);
+            var apiClient = new ApiClient();
+            apiClient.setHost(endpoint.getHost());
+            apiClient.setPort(endpoint.getPort());
+            apiClient.setScheme(endpoint.getScheme());
+            apiClient.setRequestInterceptor(
+                    builder -> builder.header("Authorization", String.format("Bearer %s", token)));
+            return apiClient;
+        } catch (URISyntaxException u) {
+            throw new RuntimeException(String.format("Invalid endpoint syntax %s", endpointText), u);
+        }
+    }
+
+    private static JsonNode getRequiredField(JsonNode node, String fieldName) {
+        if (node.has(fieldName)) {
+            return node.get(fieldName);
+        } else {
+            throw new RuntimeException(
+                    String.format("Cannot find required field %s in %s", fieldName, node));
+        }
+    }
+}

client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java

@@ -1,5 +1,8 @@
 package io.whitefox.api.utils;
 
+import static io.whitefox.api.client.ApiUtils.configureApiClientFromResource;
+import static io.whitefox.api.client.ApiUtils.ignoreConflict;
+
 import io.whitefox.api.client.*;
 import io.whitefox.api.client.model.*;
 import java.util.List;
@@ -16,7 +19,7 @@ public class StorageManagerInitializer {
   private final SchemaV1Api schemaV1Api;
 
   public StorageManagerInitializer() {
-    var apiClient = new ApiClient();
+    ApiClient apiClient = configureApiClientFromResource("MrFoxProfile.json");
     this.s3TestConfig = S3TestConfig.loadFromEnv();
     this.storageV1Api = new StorageV1Api(apiClient);
     this.providerV1Api = new ProviderV1Api(apiClient);
@@ -27,27 +30,30 @@ public StorageManagerInitializer() {
   }
 
   public void initStorageManager() {
-    storageV1Api.createStorage(createStorageRequest(s3TestConfig));
-    shareV1Api.createShare(createShareRequest());
+    ignoreConflict(() -> storageV1Api.createStorage(createStorageRequest(s3TestConfig)));
+    ignoreConflict(() -> shareV1Api.createShare(createShareRequest()));
   }
 
   public void createS3DeltaTable() {
     var providerRequest = addProviderRequest(Optional.empty(), TableFormat.delta);
-    providerV1Api.addProvider(providerRequest);
+    ignoreConflict(() -> providerV1Api.addProvider(providerRequest));
     var createTableRequest = createDeltaTableRequest();
-    tableV1Api.createTableInProvider(providerRequest.getName(), createTableRequest);
+    ignoreConflict(
+        () -> tableV1Api.createTableInProvider(providerRequest.getName(), createTableRequest));
     var shareRequest = createShareRequest();
     var schemaRequest = createSchemaRequest(TableFormat.delta);
-    schemaV1Api.createSchema(shareRequest.getName(), schemaRequest);
-    schemaV1Api.addTableToSchema(
+    ignoreConflict(() -> schemaV1Api.createSchema(shareRequest.getName(), schemaRequest));
+    ignoreConflict(() -> schemaV1Api.addTableToSchema(
         shareRequest.getName(),
         schemaRequest,
-        addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName()));
+        addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())));
   }
 
   public Metastore createGlueMetastore() {
     var metastoreRequest = createMetastoreRequest(s3TestConfig, CreateMetastore.TypeEnum.GLUE);
-    return metastoreV1Api.createMetastore(metastoreRequest);
+    return ApiUtils.recoverConflictLazy(
+        () -> metastoreV1Api.createMetastore(metastoreRequest),
+        () -> metastoreV1Api.describeMetastore(metastoreRequest.getName()));
   }
 
   private String createSchemaRequest(TableFormat tableFormat) {

client-spark/src/test/resources/MrFoxProfile.json

@@ -1,6 +1,6 @@
 {
   "shareCredentialsVersion": 1,
   "endpoint": "http://localhost:8080/delta-api/v1/",
-  "bearerToken": "fakeToken",
+  "bearerToken": "token",
   "expirationTime": null
 }

deployment/lightsail/demo.tfvars

@@ -0,0 +1,2 @@
+region = "eu-west-1"
+whitefox_token = "aSuperSecretToken"

deployment/lightsail/main.tf

@@ -0,0 +1,52 @@
+provider "aws" {
+  region = var.region
+}
+variable "region" {
+  type = string
+}
+variable "whitefox_token" {
+  type = string
+}
+resource "aws_lightsail_container_service" "whitefox-container-service" {
+  name        = "whitefox-container-service"
+  power       = "nano"
+  scale       = 1
+  is_disabled = false
+}
+
+resource "aws_lightsail_container_service_deployment_version" "whitefox-server" {
+  container {
+    container_name = "whitefox-server"
+    image          = "ghcr.io/agile-lab-dev/io.whitefox.server:latest"
+
+    command = []
+
+    environment = {
+      "WHITEFOX_SERVER_AUTHENTICATION_BEARERTOKEN" = var.whitefox_token
+      "WHITEFOX_SERVER_AUTHENTICATION_ENABLED" = true
+    }
+
+    ports = {
+      8080 = "HTTP"
+    }
+  }
+
+  public_endpoint {
+    container_name = "whitefox-server"
+    container_port = 8080
+
+    health_check {
+      healthy_threshold   = 2
+      unhealthy_threshold = 2
+      timeout_seconds     = 2
+      interval_seconds    = 5
+      path                = "/q/health/live"
+      success_codes       = "200"
+    }
+  }
+
+  service_name = aws_lightsail_container_service.whitefox-container-service.name
+}
+output "public_endpoint" {
+  value = aws_lightsail_container_service.whitefox-container-service.url
+}

docsite/docs/deployment/README.md

@@ -0,0 +1,15 @@
+# Deployment
+
+Whitefox is developed using Quarkus framework, therefore its natural habitat is a containerized environment.
+
+Due to dependencies upon Apache Hadoop, it's not possible to create GrallVM native images of whitefox (for now) 
+so right now, you need a JVM.  
+
+For your convenience, we publish on [ghcr](https://github.com/orgs/agile-lab-dev/packages?repo_name=whitefox) container
+images at each push on the main branch, therefore you can pick the runtime of your choice,
+pull the image from the registry, and you're good to go.
+
+To make things even easier we will collect ready to go guides on deploying whitefox in this section.
+
+Right now we feature the following platforms:
+- [Amazon lightsail](lightsail.md)

docsite/docs/deployment/lightsail.md

@@ -0,0 +1,90 @@
+# Amazon Lightsail
+
+[Amazon Lightsail](https://aws.amazon.com/lightsail) lets you build applications and websites fast with low-cost, 
+pre-configured cloud resources.
+
+In order to deploy a minimal and **public** setup of whitefox you'll need:
+
+- [an AWS account](https://aws.amazon.com/free/)
+- [Terraform installed locally](https://developer.hashicorp.com/terraform/install)
+- 10 minutes of your time
+
+:::warning
+
+**This is not a production ready deployment**, but is a pretty cheap one: approximately 8$ a month.
+
+:::
+
+:::danger
+
+Your secret token will be set as an environment variable of the container and will be visible to whoever
+has access to the Amazon Lightsail console. 
+
+:::
+
+----
+
+## Configuration
+
+You need to configure a few things:
+
+### AWS region
+
+head over to `deployment/lightsail/demo.tfvars` and change the value of `region` to your preferred AWS region.
+
+### Secret token
+
+head over to `deployment/lightsail/demo.tfvars` and change the value of `whitefox_token` to a secret string that
+you will need to provide as the bearer token to access  whitefox API.
+
+### AWS credentials
+
+You need to provide terraform a way to authenticate to aws, so head over [terraform docs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs)
+and find the way that suits your needs.
+
+## Show time
+
+Run:
+
+```shell
+cd deployment/lightsail
+terraform init
+terraform apply -var-file="demo.tfvars"
+```
+
+To verify that the deployment was successful, grab the output of the previous command, especially the `public_endpoint`
+and issue the command:
+
+```shell
+curl $public_endpoint/q/health/live
+```
+
+the output should be something similar to:
+
+```json
+{
+    "status": "UP",
+    "checks": [
+    ]
+}
+```
+
+If you log into your aws lightsail account: https://lightsail.aws.amazon.com/ls/webapp/home/instances you should
+see something like this:
+
+![lightsail console](imgs/lightsail_console.png)
+
+
+## Wrap up
+
+Destroy everything running:
+
+```shell
+terraform destroy -var-file="demo.tfvars"
+```
+
+:::warning
+
+**Might not be the best idea to store terraform state locally, [evaluate to configure a different backend](https://developer.hashicorp.com/terraform/language/settings/backends/configuration).**
+
+:::

server/app/build.gradle.kts

@@ -115,7 +115,7 @@ sourceSets {
 // region test running
 
 tasks.withType<Test> {
-    environment = env.allVariables
+    environment = env.allVariables()
 }
 
 // endregion

server/core/build.gradle.kts

@@ -56,7 +56,7 @@ tasks.withType<JavaCompile> {
 // region test running
 
 tasks.withType<Test> {
-    environment = env.allVariables
+    environment = env.allVariables()
 }
 
 // endregion