diff --git a/build.sbt b/build.sbt index d8b40da8ba..7ccef47a3c 100644 --- a/build.sbt +++ b/build.sbt @@ -169,7 +169,6 @@ lazy val `hmda-platform` = (project in file("hmda")) oldStrategy(x) }, reStart / envVars ++= Map("CASSANDRA_CLUSTER_HOSTS" -> "localhost", "APP_PORT" -> "2551"), - addCompilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full) ), dockerSettings, packageSettings diff --git a/common/src/main/scala/hmda/auth/KeycloakTokenVerifier.scala b/common/src/main/scala/hmda/auth/KeycloakTokenVerifier.scala index b4799adac2..83dc62669b 100644 --- a/common/src/main/scala/hmda/auth/KeycloakTokenVerifier.scala +++ b/common/src/main/scala/hmda/auth/KeycloakTokenVerifier.scala @@ -1,21 +1,15 @@ package hmda.auth -import akka.util.ByteString import com.typesafe.config.ConfigFactory -import io.circe.generic.auto._ -import io.circe.parser.decode import org.keycloak.adapters.KeycloakDeployment import org.keycloak.representations.AccessToken import java.security.KeyFactory import org.keycloak.jose.jws.AlgorithmType -import org.keycloak.adapters.KeycloakDeployment import java.math.BigInteger import java.security.spec.RSAPublicKeySpec -import java.security.KeyFactory import java.util.Base64 -import org.keycloak.TokenVerifier +import org.keycloak.{TokenVerifier => keycloakTV} import scala.util.Try - import scala.concurrent.duration._ // $COVERAGE-OFF$ @@ -34,18 +28,11 @@ class KeycloakTokenVerifier(keycloakDeployment: KeycloakDeployment) extends Toke lazy val publicKey = keyFactory.generatePublic(new RSAPublicKeySpec(modulus, publicExponent)) def verifyToken(token: String): Try[AccessToken] = { - val tokenVerifier = TokenVerifier.create(token, classOf[AccessToken]) + val tokenVerifier = keycloakTV.create(token, classOf[AccessToken]) Try { tokenVerifier.withDefaultChecks().realmUrl(realmUrl) tokenVerifier.publicKey(publicKey).verify().getToken } } - - private def parseAuthKey(line: ByteString): AuthKey = { - val str = line.utf8String - val authKeys = decode[AuthKeys](str).getOrElse(AuthKeys()) - if (authKeys.keys.nonEmpty) authKeys.keys.head - else AuthKey() - } } // $COVERAGE-ON$ \ No newline at end of file diff --git a/common/src/main/scala/hmda/auth/OAuth2Authorization.scala b/common/src/main/scala/hmda/auth/OAuth2Authorization.scala index 2132d620b0..667bcc4f0e 100644 --- a/common/src/main/scala/hmda/auth/OAuth2Authorization.scala +++ b/common/src/main/scala/hmda/auth/OAuth2Authorization.scala @@ -17,7 +17,7 @@ import org.keycloak.common.crypto.CryptoIntegration import java.util.concurrent.atomic.AtomicReference import scala.collection.JavaConverters._ import scala.util.{Failure, Success} -import org.keycloak.common.crypto.CryptoIntegration + // $COVERAGE-OFF$ class OAuth2Authorization(logger: Logger, tokenVerifier: TokenVerifier) { CryptoIntegration.init(this.getClass.getClassLoader) diff --git a/common/src/main/scala/hmda/census/records/CountyLoanLimitrecords.scala b/common/src/main/scala/hmda/census/records/CountyLoanLimitrecords.scala index ae4248c5ff..09a02de12c 100644 --- a/common/src/main/scala/hmda/census/records/CountyLoanLimitrecords.scala +++ b/common/src/main/scala/hmda/census/records/CountyLoanLimitrecords.scala @@ -45,7 +45,7 @@ object CountyLoanLimitRecords { } def countyLoansLimitByState(countyLoanLimits: Seq[CountyLoanLimit]): Map[String, StateBoundries] = { - countyLoanLimits.groupBy(county => county.stateAbbrv).mapValues { countyList => + countyLoanLimits.groupBy(county => county.stateAbbrv).view.mapValues { countyList => val oneUnit = countyList.map(county => county.oneUnitLimit) val twoUnit = countyList.map(county => county.twoUnitLimit) val threeUnit = countyList.map(county => county.threeUnitLimit) @@ -60,7 +60,7 @@ object CountyLoanLimitRecords { fourUnitMax = fourUnit.max, fourUnitMin = fourUnit.min ) - } + }.toMap } def overallLoanLimits(countyLoanLimits: Seq[CountyLoanLimit]): OverallLoanLimit = { diff --git a/common/src/main/scala/hmda/parser/derivedFields/EthnicityCategorization.scala b/common/src/main/scala/hmda/parser/derivedFields/EthnicityCategorization.scala index fdc03cf925..79f90ced00 100644 --- a/common/src/main/scala/hmda/parser/derivedFields/EthnicityCategorization.scala +++ b/common/src/main/scala/hmda/parser/derivedFields/EthnicityCategorization.scala @@ -14,7 +14,7 @@ object EthnicityCategorization { val coethnicityFields = Array(coEthnicity.ethnicity1, coEthnicity.ethnicity2, coEthnicity.ethnicity3, coEthnicity.ethnicity4, coEthnicity.ethnicity5) - val hispanicEnums = Array(HispanicOrLatino, Mexican, PuertoRican, Cuban, OtherHispanicOrLatino) + val hispanicEnums = Seq(HispanicOrLatino, Mexican, PuertoRican, Cuban, OtherHispanicOrLatino) def checkEthnicityTwoToFiveEmpty(ethnicity: Ethnicity): Boolean = ethnicity.ethnicity2 == EmptyEthnicityValue && @@ -30,7 +30,7 @@ object EthnicityCategorization { ethnicity.ethnicity5 == EmptyEthnicityValue def OnlyHispanicFlag(ethnicityFields: Array[EthnicityEnum], - ethnicityEnum: Array[EthnicityEnum with Product with Serializable]): Boolean = + ethnicityEnum: Seq[EthnicityEnum with Product]): Boolean = ethnicityFields.exists(ethnicityEnum.contains) && !ethnicityFields.contains(NotHispanicOrLatino) diff --git a/common/src/main/scala/hmda/parser/derivedFields/RaceCategorization.scala b/common/src/main/scala/hmda/parser/derivedFields/RaceCategorization.scala index ac71a25979..542be325ba 100644 --- a/common/src/main/scala/hmda/parser/derivedFields/RaceCategorization.scala +++ b/common/src/main/scala/hmda/parser/derivedFields/RaceCategorization.scala @@ -14,9 +14,9 @@ object RaceCategorization { Array(race.race1, race.race2, race.race3, race.race4, race.race5) val coRaceFields = Array(coRace.race1, coRace.race2, coRace.race3, coRace.race4, coRace.race5) - val asianEnums = Array(Asian, AsianIndian, Chinese, Filipino, Japanese, Korean, Vietnamese, OtherAsian) + val asianEnums = Seq(Asian, AsianIndian, Chinese, Filipino, Japanese, Korean, Vietnamese, OtherAsian) val hawaiianIslanderEnums = - Array(NativeHawaiianOrOtherPacificIslander, NativeHawaiian, GuamanianOrChamorro, Samoan, OtherPacificIslander) + Seq(NativeHawaiianOrOtherPacificIslander, NativeHawaiian, GuamanianOrChamorro, Samoan, OtherPacificIslander) if (race.race1 == EmptyRaceValue) { "Free Form Text Only" @@ -115,32 +115,32 @@ object RaceCategorization { } private def AnyApplicantAMinority(raceFields: Array[RaceEnum], - asianEnums: Array[RaceEnum with Product with Serializable], - hawaiianIslanderEnums: Array[RaceEnum with Product with Serializable]): Boolean = + asianEnums: Seq[RaceEnum with Product], + hawaiianIslanderEnums: Seq[RaceEnum with Product]): Boolean = raceFields.exists(asianEnums.contains) || raceFields.exists(hawaiianIslanderEnums.contains) || raceFields.contains(BlackOrAfricanAmerican) | raceFields.contains(AmericanIndianOrAlaskaNative) private def OnlyNativeHawaiianOrOtherPacificIslander(raceFields: Array[RaceEnum], - asianEnums: Array[RaceEnum with Product with Serializable], - hawaiianIslanderEnums: Array[RaceEnum with Product with Serializable]): Boolean = + asianEnums: Seq[RaceEnum with Product], + hawaiianIslanderEnums: Seq[RaceEnum with Product]): Boolean = !raceFields.exists(asianEnums.contains) && raceFields.exists(hawaiianIslanderEnums.contains) && !raceFields.contains(BlackOrAfricanAmerican) && !raceFields.contains(AmericanIndianOrAlaskaNative) private def OnlyAsian(raceFields: Array[RaceEnum], - asianEnums: Array[RaceEnum with Product with Serializable], - hawaiianIslanderEnums: Array[RaceEnum with Product with Serializable]): Boolean = + asianEnums: Seq[RaceEnum with Product], + hawaiianIslanderEnums: Seq[RaceEnum with Product]): Boolean = raceFields.exists(asianEnums.contains) && !raceFields.exists(hawaiianIslanderEnums.contains) && !raceFields.contains(BlackOrAfricanAmerican) && !raceFields.contains(AmericanIndianOrAlaskaNative) private def moreThanOneMinority(raceFields: Array[RaceEnum], - asianEnums: Array[RaceEnum with Product with Serializable], - hawaiianIslanderEnums: Array[RaceEnum with Product with Serializable]): Boolean = { + asianEnums: Seq[RaceEnum with Product], + hawaiianIslanderEnums: Seq[RaceEnum with Product]): Boolean = { val numberOfMinorityFields = List( raceFields.exists(hawaiianIslanderEnums.contains), raceFields.exists(asianEnums.contains), diff --git a/hmda-data-publisher/src/main/scala/hmda/publisher/util/PublishingReporter.scala b/hmda-data-publisher/src/main/scala/hmda/publisher/util/PublishingReporter.scala index 30304be993..38205abe59 100644 --- a/hmda-data-publisher/src/main/scala/hmda/publisher/util/PublishingReporter.scala +++ b/hmda-data-publisher/src/main/scala/hmda/publisher/util/PublishingReporter.scala @@ -136,7 +136,7 @@ object PublishingReporter { val expectations: Map[ScheduleGroupId, List[Expectation]] = Schedules.values .groupBy(forSchedule) - .mapValues(_.map(Expectation.baseForSchedule).toList) + .view.mapValues(_.map(Expectation.baseForSchedule).toList) .toMap .withDefault(_ => List()) def newForId(id: ScheduleGroupId): Group = diff --git a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2018Spec.scala b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2018Spec.scala index e81fdf07c2..7dfe6996aa 100644 --- a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2018Spec.scala +++ b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2018Spec.scala @@ -115,7 +115,7 @@ class PublisherComponent2018Spec result <- count() _ = result shouldBe 1 result <- Source.fromPublisher(getAllLARs(Array.empty)).runWith(Sink.collection) - _ = result should have length 1 + _ = result should have size 1 _ = result.head shouldBe data result <- deleteByLei("EXAMPLE-LEI") _ = result shouldBe 1 @@ -143,7 +143,7 @@ class PublisherComponent2018Spec result <- count() _ = result shouldBe 1 result <- Source.fromPublisher(getAllLARs(Array.empty)).runWith(Sink.collection) - _ = result should have length 1 + _ = result should have size 1 _ = result.head shouldBe data result <- deleteByLei("EXAMPLE-LEI") _ = result shouldBe 1 diff --git a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2019Spec.scala b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2019Spec.scala index f8432ab619..5b467d7718 100644 --- a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2019Spec.scala +++ b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2019Spec.scala @@ -115,7 +115,7 @@ class PublisherComponent2019Spec result <- count() _ = result shouldBe 1 result <- Source.fromPublisher(getAllLARs(Array.empty)).runWith(Sink.collection) - _ = result should have length 1 + _ = result should have size 1 _ = result.head shouldBe data result <- deleteByLei("EXAMPLE-LEI") _ = result shouldBe 1 diff --git a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2020Spec.scala b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2020Spec.scala index 5fd47b3a56..84f5c11215 100644 --- a/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2020Spec.scala +++ b/hmda-data-publisher/src/test/scala/hmda/publisher/query/component/PublisherComponent2020Spec.scala @@ -115,10 +115,10 @@ class PublisherComponent2020Spec result <- count() _ = result shouldBe 1 result <- Source.fromPublisher(getAllLARs(Array.empty)).runWith(Sink.collection) - _ = result should have length 1 + _ = result should have size 1 _ = result.head shouldBe data result <- Source.fromPublisher(getAllLARs(Array.empty)).runWith(Sink.collection) - _ = result should have length 1 + _ = result should have size 1 _ = result.head shouldBe data result <- deleteByLei("EXAMPLE-LEI") _ = result shouldBe 1 diff --git a/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala index 55f72e5d5e..da32ceb675 100644 --- a/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala +++ b/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala @@ -168,7 +168,7 @@ private class LeiSubmissionSummary(log: Logger, clusterSharding: ClusterSharding leiFilter match { case Some(leis) => val leisSet = leis.toSet - all.map(_.filterKeys(leisSet.contains)) + all.map(_.view.filterKeys(leisSet.contains).toMap) case None => all } } diff --git a/hmda/src/main/scala/hmda/persistence/submission/HmdaValidationError.scala b/hmda/src/main/scala/hmda/persistence/submission/HmdaValidationError.scala index 1c5c2a8d19..57496782aa 100644 --- a/hmda/src/main/scala/hmda/persistence/submission/HmdaValidationError.scala +++ b/hmda/src/main/scala/hmda/persistence/submission/HmdaValidationError.scala @@ -530,9 +530,9 @@ object HmdaValidationError larsDistinctCount = rawLineResult.distinctCount, uniqueLarsSpecificFields = -1, distinctUliCount = -1, - duplicateUliToLineNumbers = rawLineResult.uliToDuplicateLineNumbers.mapValues(_.toList), + duplicateUliToLineNumbers = rawLineResult.uliToDuplicateLineNumbers.view.mapValues(_.toList).toMap, distinctActionTakenUliCount = s306Result.distinctCount, - duplicateUliToLineNumbersUliActionType = s306Result.uliToDuplicateLineNumbers.mapValues(_.toList), + duplicateUliToLineNumbersUliActionType = s306Result.uliToDuplicateLineNumbers.view.mapValues(_.toList).toMap, actionTakenDatesWithinRange = actionTakenWithinRangeResults.totalCount, actionTakenDatesGreaterThanRange = actionTakenGreaterThanRangeResults.totalCount ), @@ -554,7 +554,7 @@ object HmdaValidationError larsDistinctCount = -1, uniqueLarsSpecificFields = uniqueLarResult.distinctCount, distinctUliCount = uliResult.distinctCount, - duplicateUliToLineNumbers = uliResult.uliToDuplicateLineNumbers.mapValues(_.toList)), + duplicateUliToLineNumbers = uliResult.uliToDuplicateLineNumbers.view.mapValues(_.toList).toMap), editType, validationContext ) diff --git a/institutions-api/src/test/scala/hmda/institution/projection/InstitutionDBProjectionSpec.scala b/institutions-api/src/test/scala/hmda/institution/projection/InstitutionDBProjectionSpec.scala index 60f15d2090..0c7eadf879 100644 --- a/institutions-api/src/test/scala/hmda/institution/projection/InstitutionDBProjectionSpec.scala +++ b/institutions-api/src/test/scala/hmda/institution/projection/InstitutionDBProjectionSpec.scala @@ -29,23 +29,21 @@ class InstitutionDBProjectionSpec val emailRepository = new InstitutionEmailsRepository(dbConfig) - override def beforeAll(): Unit = - whenReady( - Future.sequence( - institutionRepositories.values.map(_.createSchema()).toSeq ++ - tsRepositories.values.map(_.createSchema()).toSeq :+ - emailRepository.createSchema() - ) - )(_ => ()) + override def beforeAll(): Unit = { + val futures = Future.sequence(institutionRepositories.values.map(_.createSchema()).toSeq ++ + tsRepositories.values.map(_.createSchema()).toSeq :+ + emailRepository.createSchema()) + whenReady(futures)(_ => ()) + } - override def afterAll(): Unit = - whenReady( - Future.sequence( - institutionRepositories.values.map(_.dropSchema()).toSeq ++ - tsRepositories.values.map(_.dropSchema()) :+ - emailRepository.dropSchema() - ) - )(_ => ()) + override def afterAll(): Unit = { + val futures = Future.sequence( + institutionRepositories.values.map(_.dropSchema()).toSeq ++ + tsRepositories.values.map(_.dropSchema()) :+ + emailRepository.dropSchema() + ) + whenReady(futures)(_ => ()) + } "InstitutionDBProjection stores, modifies and deletes events" in { val actor = system.spawn(InstitutionDBProjection.behavior, InstitutionDBProjection.name) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index e2e3a4aa6a..e492458808 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -5,7 +5,7 @@ import sbtassembly.AssemblyPlugin.autoImport._ object BuildSettings { val buildOrganization = "cfpb" val buildVersion = "2.0.0" - val buildScalaVersion = "2.12.15" + val buildScalaVersion = "2.13.12" lazy val dockerPublishLocalSkipTestsCommand = Command.command("dockerPublishLocalSkipTests") { state => var s = state @@ -19,10 +19,10 @@ object BuildSettings { organization := buildOrganization, version := buildVersion, scalaVersion := buildScalaVersion, - scalacOptions ++= Seq("-Xlint", "-deprecation", "-unchecked", "-feature", "-Ypartial-unification"), - aggregate in assembly := false, - parallelExecution in Test := true, - fork in Test := true, + scalacOptions ++= Seq("-Xlint", "-deprecation", "-unchecked", "-feature", "-Ymacro-annotations"), + assembly / aggregate := false, + Test / parallelExecution:= true, + Test / fork := true, resolvers += Resolver.bintrayRepo("tanukkii007", "maven"), commands += dockerPublishLocalSkipTestsCommand ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e882e8bdd7..2e443ed5ec 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -65,7 +65,7 @@ object Dependencies { lazy val resteasyClient = "org.jboss.resteasy" % "resteasy-client" % Version.resteasy % "provided" lazy val resteasyJackson = "org.jboss.resteasy" % "resteasy-jackson2-provider" % Version.resteasy % "provided" lazy val resteasyMulti = "org.jboss.resteasy" % "resteasy-multipart-provider" % Version.resteasy % "provided" - lazy val jbossLogging = "org.jboss.logging" % "jboss-logging" % Version.jboss + lazy val jbossLogging = "org.jboss.logging" % "jboss-logging" % Version.jbossLogging lazy val httpClient = "org.apache.httpcomponents" % "httpclient" % Version.httpcomponents lazy val akkaKafkaStreams = "com.typesafe.akka" %% "akka-stream-kafka" % Version.akkaKafka lazy val embeddedKafka = "io.github.embeddedkafka" %% "embedded-kafka" % Version.embeddedKafka diff --git a/project/Version.scala b/project/Version.scala index d30bcabe1d..2217f1191e 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -3,7 +3,7 @@ object Version { val logback = "1.2.1" val scalaTest = "3.0.8" val scalaCheck = "1.17.0" - val akka = "2.8.2" + val akka = "2.8.5" val akkaHttp = "10.5.2" val akkaHttpJson = "1.39.2" val akkaClusterManagement = "1.4.1" @@ -18,19 +18,19 @@ object Version { val embeddedPg = "2.10" val keycloak = "22.0.1" val resteasy = "6.2.4.Final" - val jboss = "3.5.1.Final" + val jbossLogging = "3.5.1.Final" val httpcomponents = "4.5.14" val alpakka = "2.0.2" val alpakkaS3 = "2.0.1" - val embeddedKafka = "2.4.1" + val embeddedKafka = "3.0.0" val akkaQuartzScheduler = "1.9.3-akka-2.6.x" val sprayresolver = "0.10.0" val enumeratum = "1.7.2" val enumeratumCirce = "1.7.2" val monix = "3.4.1" val lettuce = "6.2.4.RELEASE" - val java8Compat = "0.9.0" - val scalaMock = "4.2.0" + val java8Compat = "1.0.2" + val scalaMock = "4.3.0" val guava = "32.0.0-jre" val awsSesSdk = "1.12.484" val zeroAllocation = "0.16"