Upgrade to latest Scanamo version (1.0.0-M28)

Here we're moving up from Scanamo 1.0.0-M9 to 1.0.0-M28, which involves
several changes to requirements:

* AWS SDK v1 → v2 - Scanamo 1.0-M13 and above require AWS SDK v2
* Scanamo API changes:
  - When we `exec()` a Scanamo call, we now access an _instance_ of
    Scanamo that _wraps_ an AWS DynamoDB client, rather than supplying
    an AWS DynamoDB client to the `Scanamo` singleton. This means most
    of the `DataStore` classes in `media-atom-maker` now accept a
    `Scanamo` instance rather than a `AmazonDynamoDBClient`.
  - Auto-generated `DynamoFormat` handlers are now created by
    `org.scanamo.generic.auto._` rather than `org.scanamo.auto._`
  - Query on entity keys by name as a String (`"id"`), rather than
    using the old 'Symbol' apostrophe syntax (`'id`), and use `===`
    rather than `->` to specify the required key value.

The AWS SDK v1 → v2 requirement of Scanamo is the tricky one to handle-
a lot of code `media-atom-maker` also makes AWS SDK calls, and I didn't
want to have to fix all of it to use AWS SDK v2 in this change. Instead,
this change introduces `com.gu.media.aws.CredentialsForBothSdkVersions`,
which makes a choice of AWS SDK v1 or v2 credentials available in the
already-existing `com.gu.media.aws.AwsCredentials` class. This allows
a partial move to AWS SDK v2, while other parts of the code remain on
AWS SDK v1 for now. Notably, we need both v1 _and_ v2 versions of the
DynamoDB client, as the atom-maker libraries in https://github.com/guardian/atom-maker
still use AWS SDK v1.

app/data/DataStores.scala

@@ -14,7 +14,7 @@ class DataStores(aws: AWSConfig with SNSAccess, capi: CapiAccess)  {
   val preview = new PreviewDynamoDataStore(aws.dynamoDB, aws.dynamoTableName)
   val published = new PublishedDynamoDataStore(aws.dynamoDB, aws.publishedDynamoTableName)
 
-  val pluto: PlutoDataStore = new PlutoDataStore(aws.dynamoDB, aws.manualPlutoDynamo)
+  val pluto: PlutoDataStore = new PlutoDataStore(aws.scanamo, aws.manualPlutoDynamo)
 
   val livePublisher: AtomPublisher = aws.capiContentEventsTopicName match {
     case Some(capiContentEventsTopicName) =>

app/di.scala

@@ -73,7 +73,7 @@ class MediaAtomMaker(context: Context)
   private val capi = new Capi(config)
 
   private val stores = new DataStores(aws, capi)
-  private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.credentials.instance)
+  private val permissions = new MediaAtomMakerPermissionsProvider(aws.stage, aws.credentials.instance.awsV1Creds)
 
   private val reindexer = buildReindexer()
 

app/util/AWS.scala

@@ -26,7 +26,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden
   lazy val ec2Client = AmazonEC2ClientBuilder
     .standard()
     .withRegion(region.getName)
-    .withCredentials(credentials.instance)
+    .withCredentials(credentials.instance.awsV1Creds)
     .build()
 
   lazy val pinboardLoaderUrl = getString("panda.domain").map(domain => s"https://pinboard.$domain/pinboard.loader.js")
@@ -43,7 +43,7 @@ class AWSConfig(override val config: Config, override val credentials: AwsCreden
   lazy val expiryPollerName = "Expiry"
   lazy val expiryPollerLastName = "Poller"
 
-  final override def regionName = getString("aws.region")
+  final override def region = AwsAccess.regionFrom(this)
 
   final override def readTag(tagName: String) = {
     val tagsResult = ec2Client.describeTags(

app/util/UploadDecorator.scala

@@ -3,9 +3,9 @@ package util
 import com.gu.media.aws.{DynamoAccess, UploadAccess}
 import com.gu.media.model.{ClientAsset, ClientAssetMetadata}
 import com.gu.media.upload.model.Upload
-import org.scanamo.{Scanamo, Table}
+import org.scanamo.Table
 import org.scanamo.syntax._
-import org.scanamo.auto._
+import org.scanamo.generic.auto._
 
 class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFunctions) {
   private val table = Table[Upload](aws.cacheTableName)
@@ -28,10 +28,6 @@ class UploadDecorator(aws: DynamoAccess with UploadAccess, stepFunctions: StepFu
     }
   }
 
-  private def getUpload(id: String): Option[Upload] = {
-    val op = table.get('id -> id)
-    val result = Scanamo.exec(aws.dynamoDB)(op).flatMap(_.right.toOption)
-
-    result orElse { stepFunctions.getById(id) }
-  }
+  private def getUpload(id: String): Option[Upload] =
+    aws.scanamo.exec(table.get("id" === id)).flatMap(_.toOption) orElse { stepFunctions.getById(id) }
 }

build.sbt

@@ -4,11 +4,12 @@ import scala.sys.process._
 
 val scroogeVersion = "4.12.0"
 val awsVersion = "1.11.678"
+val awsV2Version = "2.21.17"
 val pandaVersion = "1.2.0"
 val pandaHmacVersion = "2.1.0"
 val atomMakerVersion = "1.3.4"
 val typesafeConfigVersion = "1.4.0" // to match what we get from Play transitively
-val scanamoVersion = "1.0.0-M9"
+val scanamoVersion = "1.0.0-M28"
 
 val playJsonExtensionsVersion = "0.42.0"
 val okHttpVersion = "2.4.0"
@@ -104,10 +105,12 @@ lazy val common = (project in file("common"))
       "com.amazonaws" % "aws-lambda-java-core" % awsLambdaCoreVersion,
       "com.amazonaws" % "aws-java-sdk-s3" % awsVersion,
       "com.amazonaws" % "aws-java-sdk-dynamodb" % awsVersion,
+      "software.amazon.awssdk" % "dynamodb" % awsV2Version,
       "com.amazonaws" % "aws-java-sdk-kinesis" % awsVersion,
       "ai.x" %% "play-json-extensions" % playJsonExtensionsVersion,
       "ch.qos.logback" % "logback-classic" % logbackClassicVersion,
       "com.amazonaws" % "aws-java-sdk-sts" % awsVersion,
+      "software.amazon.awssdk" % "sts" % awsV2Version,
       "com.amazonaws" % "aws-java-sdk-elastictranscoder" % awsVersion,
       "org.scanamo" %% "scanamo" % scanamoVersion,
       "com.squareup.okhttp" % "okhttp" % okHttpVersion,
@@ -134,6 +137,7 @@ lazy val app = (project in file("."))
       ehcache,
       "com.fasterxml.jackson.core" % "jackson-databind" % jacksonDatabindVersion,
       "com.amazonaws" % "aws-java-sdk-sts" % awsVersion,
+      "software.amazon.awssdk" % "sts" % awsV2Version,
       "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion,
       "org.scalatestplus.play" %% "scalatestplus-play" % scalaTestPlusPlayVersion % "test",
       "org.mockito" %  "mockito-core" % mockitoVersion % "test",

common/src/main/scala/com/gu/media/PlutoDataStore.scala

@@ -1,20 +1,17 @@
 package com.gu.media
 
-import com.amazonaws.services.dynamodbv2.AmazonDynamoDB
-import com.amazonaws.services.dynamodbv2.model.DeleteItemResult
 import com.gu.media.model.PlutoSyncMetadataMessage
-import org.scanamo.error.DynamoReadError
+import org.scanamo.generic.auto._
 import org.scanamo.syntax._
 import org.scanamo.{Scanamo, Table}
-import org.scanamo.auto._
 
-class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) {
+class PlutoDataStore(scanamo: Scanamo, dynamoTableName: String) {
 
   val table = Table[PlutoSyncMetadataMessage](dynamoTableName)
 
   def getUploadsWithAtomId(id: String): List[PlutoSyncMetadataMessage] = {
     val atomIdIndex = table.index("atom-id")
-    val results = Scanamo.exec(client)(atomIdIndex.query('atomId -> id))
+    val results = scanamo.exec(atomIdIndex.query("atomId" === id))
 
     val errors = results.collect { case Left(err) => err }
     if (errors.nonEmpty) {
@@ -25,21 +22,21 @@ class PlutoDataStore(client: AmazonDynamoDB, dynamoTableName: String) {
   }
 
   def get(id: String): Option[PlutoSyncMetadataMessage] = {
-    val operation = table.get('id -> id)
-    val result = Scanamo.exec(client)(operation)
+    val operation = table.get("id" === id)
+    val result = scanamo.exec(operation)
 
     result.map {
       case Right(item) => item
       case Left(err) => throw DynamoPlutoTableException(err.toString)
     }
   }
 
-  def put(item: PlutoSyncMetadataMessage): Option[Either[DynamoReadError, PlutoSyncMetadataMessage]] = {
-    Scanamo.exec(client)(table.put(item))
+  def put(item: PlutoSyncMetadataMessage): Unit = {
+    scanamo.exec(table.put(item))
   }
 
-  def delete(id: String): DeleteItemResult = {
-    Scanamo.exec(client)(table.delete('id -> id))
+  def delete(id: String): Unit = {
+    scanamo.exec(table.delete("id" === id))
   }
 
   case class DynamoPlutoTableException(err: String) extends RuntimeException(err)

common/src/main/scala/com/gu/media/aws/AwsAccess.scala

@@ -5,17 +5,17 @@ import com.amazonaws.regions.{Region, Regions}
 import com.gu.media.Settings
 
 trait AwsAccess { this: Settings =>
-  def regionName: Option[String]
   def readTag(tag: String): Option[String]
 
   val credentials: AwsCredentials
+
   // To avoid renaming references everywhere
-  def credsProvider: AWSCredentialsProvider = credentials.instance
+  def credsProvider: AWSCredentialsProvider = credentials.instance.awsV1Creds
 
-  final def defaultRegion: Region = Region.getRegion(Regions.EU_WEST_1)
-  final def region: Region = regionName
-    .map { name => Region.getRegion(Regions.fromName(name)) }
-    .getOrElse(defaultRegion)
+  def region: Region
+
+  final def awsV2Region: software.amazon.awssdk.regions.Region =
+    software.amazon.awssdk.regions.Region.of(region.getName)
 
   // These are injected as environment variables when running in a Lambda (unfortunately they cannot be tagged)
   final val stage = sys.env.getOrElse("STAGE", readTag("Stage").getOrElse("DEV"))
@@ -24,3 +24,11 @@ trait AwsAccess { this: Settings =>
   final val stack: Option[String] = if (isDev) Some("media-atom-maker") else readTag("Stack")
   final val app: String = if (isDev) "media-atom-maker" else readTag("App").getOrElse("media-atom-maker")
 }
+
+object AwsAccess {
+  def regionFrom(maybeName: Option[String]): Region = maybeName
+    .map { name => Region.getRegion(Regions.fromName(name)) }
+    .getOrElse(Region.getRegion(Regions.EU_WEST_1))
+
+  def regionFrom(settings: Settings): Region = regionFrom(settings.getString("aws.region"))
+}

common/src/main/scala/com/gu/media/aws/AwsCredentials.scala

@@ -1,20 +1,20 @@
 package com.gu.media.aws
 
-import com.amazonaws.auth._
-import com.amazonaws.auth.profile.ProfileCredentialsProvider
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder
 import com.gu.media.Settings
 
-case class AwsCredentials(instance: AWSCredentialsProvider, crossAccount: AWSCredentialsProvider,
-                          upload: AWSCredentialsProvider)
+case class AwsCredentials(
+  instance: CredentialsForBothSdkVersions,
+  crossAccount: CredentialsForBothSdkVersions,
+  upload: CredentialsForBothSdkVersions
+)
 
 object AwsCredentials {
   def dev(settings: Settings): AwsCredentials = {
     val profile = settings.getMandatoryString("aws.profile")
-    val instance = new ProfileCredentialsProvider(profile)
+    val instance = CredentialsForBothSdkVersions.profile(profile)
 
     // To enable publishing to CAPI code from DEV, update the kinesis streams in config and uncomment below:
-    //   val crossAccount = new ProfileCredentialsProvider("composer")
+    //   val crossAccount = AwsCredentialsProvidersForBothSdkVersions.profile("composer")
     val crossAccount = instance
 
     val upload = devUpload(settings)
@@ -23,38 +23,31 @@ object AwsCredentials {
   }
 
   def app(settings: Settings): AwsCredentials = {
-    val instance = InstanceProfileCredentialsProvider.getInstance()
+    val instance = CredentialsForBothSdkVersions.instance()
 
     val crossAccount = assumeCrossAccountRole(instance, settings)
 
     AwsCredentials(instance, crossAccount, upload = instance)
   }
 
   def lambda(): AwsCredentials = {
-    val instance = new EnvironmentVariableCredentialsProvider()
+    val instance = CredentialsForBothSdkVersions.environmentVariables()
     AwsCredentials(instance, crossAccount = instance, upload = instance)
   }
 
-  private def devUpload(settings: Settings): AWSCredentialsProvider = {
+  private def devUpload(settings: Settings): CredentialsForBothSdkVersions = {
     // Only required in dev (because federated credentials such as those from Janus cannot do STS requests).
     // Instance profile credentials are sufficient when deployed.
     val accessKey = settings.getMandatoryString("aws.upload.accessKey", "This is the AwsId output of the dev cloudformation")
     val secretKey = settings.getMandatoryString("aws.upload.secretKey", "This is the AwsSecret output of the dev cloudformation")
 
-    new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))
+    CredentialsForBothSdkVersions.static(accessKey, secretKey)
   }
 
-  private def assumeCrossAccountRole(instance: AWSCredentialsProvider, settings: Settings) = {
+  private def assumeCrossAccountRole(instance: CredentialsForBothSdkVersions, settings: Settings): CredentialsForBothSdkVersions = {
     val crossAccountRoleArn = settings.getMandatoryString("aws.kinesis.stsCapiRoleToAssume",
       "Role to assume to access CAPI streams (in format arn:aws:iam::<account>:role/<role_name>)")
 
-    assumeAccountRole(instance, crossAccountRoleArn, "capi")
-  }
-
-  private def assumeAccountRole(instance: AWSCredentialsProvider, roleArn: String, sessionNameSuffix: String): AWSCredentialsProvider = {
-    val securityTokens = AWSSecurityTokenServiceClientBuilder.standard().withCredentials(instance).build()
-
-    new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, s"media-atom-maker-${sessionNameSuffix}")
-      .withStsClient(securityTokens).build()
+    instance.assumeAccountRole(crossAccountRoleArn, "capi", AwsAccess.regionFrom(settings).getName)
   }
 }

common/src/main/scala/com/gu/media/aws/AwsV2Util.scala

@@ -0,0 +1,16 @@
+package com.gu.media.aws
+
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+import software.amazon.awssdk.awscore.client.builder.{AwsClientBuilder, AwsSyncClientBuilder}
+import software.amazon.awssdk.http.apache.ApacheHttpClient
+import software.amazon.awssdk.regions.Region
+
+object AwsV2Util {
+  def buildSync[T, B <: AwsClientBuilder[B, T] with AwsSyncClientBuilder[B, T]](
+    builder: B, creds: AwsCredentialsProvider, region: Region
+  ): T = builder
+    .httpClientBuilder(ApacheHttpClient.builder())
+    .credentialsProvider(creds)
+    .region(region)
+    .build()
+}

common/src/main/scala/com/gu/media/aws/CredentialsForBothSdkVersions.scala

@@ -0,0 +1,48 @@
+package com.gu.media.aws
+
+import com.amazonaws.auth._
+import com.amazonaws.auth.profile.ProfileCredentialsProvider
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder
+import software.amazon.awssdk.auth.{credentials => awsv2}
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.sts.{StsClient, StsClientBuilder}
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest
+
+case class CredentialsForBothSdkVersions(
+  awsV1Creds: com.amazonaws.auth.AWSCredentialsProvider,
+  awsV2Creds: software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
+) {
+  def assumeAccountRole(roleArn: String, sessionNameSuffix: String, regionName: String): CredentialsForBothSdkVersions = {
+    val roleSessionName = s"media-atom-maker-$sessionNameSuffix"
+    CredentialsForBothSdkVersions(
+      new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, roleSessionName)
+        .withStsClient(AWSSecurityTokenServiceClientBuilder.standard().withCredentials(awsV1Creds).build()).build(),
+      software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider.builder()
+        .stsClient(AwsV2Util.buildSync[StsClient, StsClientBuilder](StsClient.builder(), awsV2Creds, Region.of(regionName)))
+        .refreshRequest(AssumeRoleRequest.builder.roleSessionName(roleSessionName).roleArn(roleArn).build)
+        .build()
+    )
+  }
+}
+
+object CredentialsForBothSdkVersions {
+  def profile(name: String): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions(
+    new ProfileCredentialsProvider(name),
+    awsv2.ProfileCredentialsProvider.create(name)
+  )
+
+  def instance(): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions(
+    InstanceProfileCredentialsProvider.getInstance(),
+    awsv2.InstanceProfileCredentialsProvider.create()
+  )
+
+  def environmentVariables(): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions(
+    new EnvironmentVariableCredentialsProvider(),
+    awsv2.EnvironmentVariableCredentialsProvider.create()
+  )
+
+  def static(accessKey: String, secretKey: String): CredentialsForBothSdkVersions = CredentialsForBothSdkVersions(
+    new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)),
+    awsv2.StaticCredentialsProvider.create(awsv2.AwsBasicCredentials.create(accessKey, secretKey))
+  )
+}

common/src/main/scala/com/gu/media/aws/DynamoAccess.scala

@@ -2,6 +2,9 @@ package com.gu.media.aws
 
 import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder
 import com.gu.media.Settings
+import com.gu.media.aws.AwsV2Util.buildSync
+import org.scanamo.Scanamo
+import software.amazon.awssdk.services.dynamodb.{DynamoDbClient, DynamoDbClientBuilder}
 
 trait DynamoAccess { this: Settings with AwsAccess =>
   lazy val dynamoTableName: String = sys.env.getOrElse("ATOM_TABLE_NAME",
@@ -24,4 +27,9 @@ trait DynamoAccess { this: Settings with AwsAccess =>
     .withCredentials(credsProvider)
     .withRegion(region.getName)
     .build()
+
+  lazy val dynamoDbSdkV2: DynamoDbClient =
+    buildSync[DynamoDbClient, DynamoDbClientBuilder](DynamoDbClient.builder(), credentials.instance.awsV2Creds, awsV2Region)
+
+  lazy val scanamo: Scanamo = Scanamo(dynamoDbSdkV2)
 }

common/src/main/scala/com/gu/media/aws/KinesisAccess.scala

@@ -21,12 +21,12 @@ trait KinesisAccess { this: Settings with AwsAccess with Logging =>
   val syncWithPluto: Boolean = getBoolean("pluto.sync").getOrElse(false)
 
   lazy val crossAccountKinesisClient = AmazonKinesisClientBuilder.standard()
-    .withCredentials(credentials.crossAccount)
+    .withCredentials(credentials.crossAccount.awsV1Creds)
     .withRegion(region.getName)
     .build()
 
   lazy val kinesisClient = AmazonKinesisClientBuilder.standard()
-    .withCredentials(credentials.instance)
+    .withCredentials(credentials.instance.awsV1Creds)
     .withRegion(region.getName)
     .build()
 

common/src/main/scala/com/gu/media/aws/SNSAccess.scala

@@ -11,6 +11,6 @@ trait SNSAccess { this: Settings with AwsAccess =>
   lazy val snsClient =
     AmazonSNSClientBuilder.standard()
       .withRegion(Regions.fromName(region.getName))
-      .withCredentials(credentials.instance)
+      .withCredentials(credentials.instance.awsV1Creds)
       .build()
 }

common/src/main/scala/com/gu/media/aws/UploadAccess.scala

@@ -30,7 +30,7 @@ trait UploadAccess { this: Settings with AwsAccess =>
       throw new IllegalArgumentException("aws.upload.role must be in ARN format: arn:aws:iam::<account>:role/<role_name>")
     }
 
-    AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload).withRegion(region.getName).build()
+    AWSSecurityTokenServiceClientBuilder.standard().withCredentials(credentials.upload.awsV1Creds).withRegion(region.getName).build()
   }
 
   private def getPipelineArn() = {

common/src/main/scala/com/gu/media/lambda/LambdaBase.scala

@@ -9,10 +9,10 @@ import com.gu.media.aws.{AwsAccess, AwsCredentials, HMACSettings}
 import com.typesafe.config.{Config, ConfigFactory}
 
 trait LambdaBase extends Settings with AwsAccess with HMACSettings {
-  final override def regionName = sys.env.get("REGION")
+  final override def region = AwsAccess.regionFrom(sys.env.get("REGION"))
   final override def readTag(tag: String) = sys.env.get(tag.toUpperCase(Locale.ENGLISH))
 
-  final override val credentials = AwsCredentials.lambda()
+  final override val credentials: AwsCredentials = AwsCredentials.lambda()
 
   private val remoteConfig = downloadConfig()
   private val mergedConfig = remoteConfig.withFallback(ConfigFactory.load())
@@ -24,7 +24,7 @@ trait LambdaBase extends Settings with AwsAccess with HMACSettings {
       case (Some(bucket), Some(key)) =>
         val defaultRegionS3 = AmazonS3ClientBuilder
           .standard()
-          .withCredentials(credentials.instance)
+          .withCredentials(credentials.instance.awsV1Creds)
           .withRegion(region.getName)
           .build()