feat(Storage): Support using AWS S3 as online cache for scan results

Signed-off-by: carlos-ardila2 <[email protected]>

README.md

@@ -546,6 +546,35 @@ ort:
     storageWriters: ["artifactoryStorage"]
 ```
 
+### AWS S3 Storage
+
+An [AWS S3 Bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html) can be used to store scan
+results. You must provide a previously created Bucket, an optional AWS region (if not present, the
+[default](https://docs.aws.amazon.com/sdk-for-kotlin/latest/developer-guide/region-selection.html) will be used),
+credentials (if not provided, system
+[default](https://docs.aws.amazon.com/sdk-for-kotlin/latest/developer-guide/credential-providers.html) values will be
+used), and an optional [custom endpoint](https://docs.aws.amazon.com/sdkref/latest/guide/feature-ss-endpoints.html) to
+support local endpoints, VPC endpoints, and third-party local AWS development environments. Also, you can optionally
+set the data to be compressed before stored.
+
+```yaml
+ort:
+  scanner:
+    storages:
+      awsS3Storage:
+        backend:
+          s3FileStorage:
+            accessKeyId: "aws-access-key (optional)"
+            awsRegion: "us-east-1 (optional)"
+            bucketName: "ort-scan-results"
+            compression: true (optional),
+            customEndpoint: "https://192.145.16.241/aws (optional)",
+            secretAccessKey: "aws-access-key-secret (optional)"
+
+    storageReaders: ["awsS3Storage"]
+    storageWriters: ["awsS3Storage"]
+```
+
 ### PostgreSQL Storage
 
 To use PostgreSQL for storing scan results you need at least version 9.4, create a database with the `client_encoding`

gradle/libs.versions.toml

@@ -50,6 +50,7 @@ postgresEmbedded = "1.0.2"
 reflections = "0.10.2"
 retrofit = "2.9.0"
 retrofitConverterKotlinxSerialization = "1.0.0"
+s3 = "2.20.162"
 saxonHe = "12.3"
 scanoss = "1.1.6"
 semver4j = "5.2.2"
@@ -74,6 +75,7 @@ versions = { id = "com.github.ben-manes.versions", version.ref = "versionsPlugin
 antlr = { module = "org.antlr:antlr4", version.ref = "antlr" }
 asciidoctorj = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" }
 asciidoctorjPdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorjPdf" }
+awsS3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" }
 clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" }
 commonsCompress = { module = "org.apache.commons:commons-compress", version.ref = "commonsCompress" }
 cvssCalculator = { module = "us.springett:cvss-calculator", version.ref = "cvssCalculator" }

integrations/schemas/ort-configuration-schema.json

@@ -263,6 +263,9 @@
                 },
                 "httpFileStorage": {
                     "§ref": "#/definitions/HttpFileStorage"
+                },
+                "s3FileStorage": {
+                    "§ref": "#/definitions/S3FileStorage"
                 }
             },
             "required": [
@@ -303,6 +306,33 @@
                 "url"
             ]
         },
+        "S3FileStorage": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "accessKeyId": {
+                    "type": "string"
+                },
+                "awsRegion": {
+                    "type": "string"
+                },
+                "bucketName": {
+                    "type": "string"
+                },
+                "compression": {
+                    "type": "boolean"
+                },
+                "customEndpoint": {
+                    "type": "string"
+                },
+                "secretAccessKey": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "bucketName"
+            ]
+        },
         "PostgresConfig": {
             "type": "object",
             "additionalProperties": false,
@@ -397,6 +427,7 @@
         },
         "StorageTypes": {
             "enum": [
+                "aws",
                 "clearlyDefined",
                 "http",
                 "local",

model/src/main/kotlin/config/FileStorageConfiguration.kt

@@ -23,6 +23,7 @@ import org.ossreviewtoolkit.utils.common.expandTilde
 import org.ossreviewtoolkit.utils.ort.storage.FileStorage
 import org.ossreviewtoolkit.utils.ort.storage.HttpFileStorage
 import org.ossreviewtoolkit.utils.ort.storage.LocalFileStorage
+import org.ossreviewtoolkit.utils.ort.storage.S3FileStorage
 import org.ossreviewtoolkit.utils.ort.storage.XZCompressedLocalFileStorage
 
 /**
@@ -37,20 +38,32 @@ data class FileStorageConfiguration(
     /**
      * The configuration of a [LocalFileStorage].
      */
-    val localFileStorage: LocalFileStorageConfiguration? = null
+    val localFileStorage: LocalFileStorageConfiguration? = null,
+
+    /**
+     * The configuration of a [S3FileStorage].
+     */
+    val s3FileStorage: S3FileStorageConfiguration? = null
 ) {
     /**
      * Create a [FileStorage] based on this configuration.
      */
     fun createFileStorage(): FileStorage {
-        val storage = requireNotNull(listOfNotNull(httpFileStorage, localFileStorage).singleOrNull()) {
+        val storage = requireNotNull(listOfNotNull(httpFileStorage, localFileStorage, s3FileStorage).singleOrNull()) {
             "Exactly one implementation must be configured for a FileStorage."
         }
 
         if (storage is HttpFileStorageConfiguration) {
             return HttpFileStorage(storage.url, storage.query, storage.headers)
         }
 
+        if (storage is S3FileStorageConfiguration) {
+            return S3FileStorage(
+                storage.accessKeyId, storage.awsRegion, storage.bucketName, storage.compression,
+                storage.customEndpoint, storage.secretAccessKey
+            )
+        }
+
         check(storage is LocalFileStorageConfiguration)
 
         val directory = storage.directory.expandTilde()

model/src/main/kotlin/config/S3FileStorageConfiguration.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2023 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.model.config
+
+/**
+ * A class to hold the configuration for using an AWS S3 bucket as a storage.
+ */
+data class S3FileStorageConfiguration(
+    /** The AWS access key */
+    val accessKeyId: String? = null,
+
+    /** The AWS region to be used. */
+    val awsRegion: String? = null,
+
+    /** The name of the S3 bucket used to store files in. */
+    val bucketName: String,
+
+    /** Whether to use compression for storing files or not. Defaults to true. */
+    val compression: Boolean = false,
+
+    /** Custom endpoint to perform AWS API Requests */
+    val customEndpoint: String? = null,
+
+    /** The AWS secret for the access key. */
+    val secretAccessKey: String? = null
+)

model/src/main/resources/reference.yml

@@ -273,6 +273,16 @@ ort:
               key1: value1
               key2: value2
 
+      aws:
+        backend:
+          s3FileStorage:
+            accessKeyId: "accessKey"
+            awsRegion: "us-east-1"
+            bucketName: "ort-scan-results"
+            compression: false
+            customEndpoint: "http://localhost:4567"
+            secretAccessKey: "secret"
+
       clearlyDefined:
         serverUrl: 'https://api.clearlydefined.io'
 
@@ -298,7 +308,7 @@ ort:
         token: token
 
     # Storage readers are listed from highest to lower priority, i.e. the first match wins.
-    storageReaders: [local, postgres, http, clearlyDefined]
+    storageReaders: [local, postgres, http, aws, clearlyDefined]
 
     # For storage writers no priority is implied by the order; scan results are stored for all writers.
     storageWriters: [postgres]

model/src/test/kotlin/config/OrtConfigurationTest.kt

@@ -193,6 +193,7 @@ class OrtConfigurationTest : WordSpec({
                     enabled shouldBe true
 
                     fileStorage shouldNotBeNull {
+                        s3FileStorage should beNull()
                         httpFileStorage should beNull()
                         localFileStorage shouldNotBeNull {
                             directory shouldBe File("~/.ort/scanner/archive")
@@ -224,6 +225,7 @@ class OrtConfigurationTest : WordSpec({
 
                 fileListStorage shouldNotBeNull {
                     fileStorage shouldNotBeNull {
+                        s3FileStorage should beNull()
                         httpFileStorage should beNull()
                         localFileStorage shouldNotBeNull {
                             directory shouldBe File("~/.ort/scanner/file-lists")
@@ -291,7 +293,7 @@ class OrtConfigurationTest : WordSpec({
 
                 storages shouldNotBeNull {
                     keys shouldContainExactlyInAnyOrder setOf(
-                        "local", "http", "clearlyDefined", "postgres", "sw360Configuration"
+                        "local", "http", "aws", "clearlyDefined", "postgres", "sw360Configuration"
                     )
 
                     val localStorage = this["local"]
@@ -309,6 +311,16 @@ class OrtConfigurationTest : WordSpec({
                         headers should containExactlyEntries("key1" to "value1", "key2" to "value2")
                     }
 
+                    val s3Storage = this["aws"]
+                    s3Storage.shouldBeInstanceOf<FileBasedStorageConfiguration>()
+                    s3Storage.backend.s3FileStorage shouldNotBeNull {
+                        bucketName shouldBe "ort-scan-results"
+                        awsRegion shouldBe "us-east-1"
+                        accessKeyId shouldBe "accessKey"
+                        secretAccessKey shouldBe "secret"
+                        compression shouldBe false
+                    }
+
                     val cdStorage = this["clearlyDefined"]
                     cdStorage.shouldBeInstanceOf<ClearlyDefinedStorageConfiguration>()
                     cdStorage.serverUrl shouldBe "https://api.clearlydefined.io"
@@ -338,14 +350,15 @@ class OrtConfigurationTest : WordSpec({
                     sw360Storage.token shouldBe "token"
                 }
 
-                storageReaders shouldContainExactly listOf("local", "postgres", "http", "clearlyDefined")
+                storageReaders shouldContainExactly listOf("local", "postgres", "http", "aws", "clearlyDefined")
                 storageWriters shouldContainExactly listOf("postgres")
 
                 ignorePatterns shouldContainExactly listOf("**/META-INF/DEPENDENCIES")
 
                 provenanceStorage shouldNotBeNull {
                     fileStorage shouldNotBeNull {
                         httpFileStorage should beNull()
+                        s3FileStorage should beNull()
                         localFileStorage shouldNotBeNull {
                             directory shouldBe File("~/.ort/scanner/provenance")
                             compression shouldBe false

model/src/test/kotlin/config/ScannerConfigurationTest.kt

@@ -51,6 +51,7 @@ class ScannerConfigurationTest : WordSpec({
             actualScannerConfig.storageReaders shouldBe expectedScannerConfig.storageReaders
             actualScannerConfig.storageWriters shouldBe expectedScannerConfig.storageWriters
             actualScannerConfig.archive?.fileStorage?.httpFileStorage should beNull()
+            actualScannerConfig.archive?.fileStorage?.s3FileStorage should beNull()
 
             actualStorages.keys shouldContainExactly expectedStorages.keys
             actualStorages.entries.forAll { (storageKey, storage) ->

utils/ort/build.gradle.kts

@@ -28,6 +28,7 @@ dependencies {
 
     api(libs.okhttp)
 
+    implementation(libs.awsS3)
     implementation(libs.commonsCompress)
     implementation(libs.kotlinxCoroutines)