Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dont show invalid sbt and scalajs versions #614

Merged
merged 10 commits into from
Apr 9, 2020
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 11 additions & 12 deletions data/src/main/scala/ch.epfl.scala.index.data/Main.scala
Original file line number Diff line number Diff line change
@@ -1,22 +1,21 @@
package ch.epfl.scala.index.data

import bintray.{BintrayDownloadPoms, BintrayListPoms, BintrayDownloadSbtPlugins}
import cleanup.{NonStandardLib, GithubRepoExtractor}
import elastic.SeedElasticSearch
import github.GithubDownload
import maven.DownloadParentPoms
import util.PidLock

import java.nio.file.Path

import java.time.LocalDateTime
import java.time.format.DateTimeFormatter

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer

import ch.epfl.scala.index.data.bintray.{
BintrayDownloadPoms,
BintrayDownloadSbtPlugins,
BintrayListPoms
}
import ch.epfl.scala.index.data.central.CentralMissing
import ch.epfl.scala.index.data.cleanup.{GithubRepoExtractor, NonStandardLib}
import ch.epfl.scala.index.data.elastic.SeedElasticSearch
import ch.epfl.scala.index.data.github.GithubDownload
import ch.epfl.scala.index.data.maven.DownloadParentPoms
import ch.epfl.scala.index.data.util.PidLock
import com.typesafe.config.ConfigFactory

import org.slf4j.LoggerFactory

import scala.sys.process.Process
Expand Down
Original file line number Diff line number Diff line change
@@ -1,49 +1,37 @@
package ch.epfl.scala.index
package data
package central

import ch.epfl.scala.index.model.Release
import ch.epfl.scala.index.model.release.ScalaTarget
import ch.epfl.scala.index.model.SemanticVersion
import ch.epfl.scala.index.data.github.GithubDownload
import ch.epfl.scala.index.data.project.{ProjectConvert, ArtifactMetaExtractor}
import ch.epfl.scala.index.data.maven.PomsReader
import ch.epfl.scala.index.model.misc.Sha1

import scala.util.{Failure, Success}
import scala.concurrent.{Future, Promise, Await}
import scala.concurrent.duration._
import scala.util.Try
import java.nio.charset.StandardCharsets
import java.nio.file.Files

import akka.NotUsed
import akka.actor.ActorSystem

import akka.http.scaladsl.Http
import akka.http.scaladsl.model.Uri.Query
import akka.http.scaladsl.model._
import akka.http.scaladsl.model.ContentTypes._
import akka.http.scaladsl.model.headers._

import akka.http.scaladsl.model.Uri.Query
import akka.http.scaladsl.unmarshalling._

import akka.stream.Materializer
import akka.stream.ThrottleMode
import akka.stream.{Materializer, ThrottleMode}
import akka.stream.scaladsl._

import akka.stream.{OverflowStrategy, QueueOfferResult}

import ch.epfl.scala.index.data.maven.PomsReader
import ch.epfl.scala.index.data.project.ArtifactMetaExtractor
import ch.epfl.scala.index.model.MinorBinary
import ch.epfl.scala.index.model.misc.Sha1
import ch.epfl.scala.index.model.release.{
SbtPlugin,
ScalaJs,
ScalaJvm,
ScalaNative
}
import de.heikoseeberger.akkahttpjson4s.Json4sSupport._
import org.joda.time.DateTime
import org.json4s._
import org.json4s.native.JsonMethods._

import org.slf4j.LoggerFactory

import scala.collection.immutable.Seq

import java.nio.file.Files
import java.nio.charset.StandardCharsets

import java.lang.reflect.InvocationTargetException
import scala.concurrent.duration._
import scala.concurrent.{Await, Future}
import scala.util.{Failure, Success, Try}

object CentralMissing {
// q = g:"com.47deg" AND a:"sbt-microsites"
Expand Down Expand Up @@ -263,27 +251,27 @@ class CentralMissing(paths: DataPaths)(implicit val materializer: Materializer,
.flatten
.toSet

val scala213 = SemanticVersion("2.13").get
val scala212 = SemanticVersion("2.12").get
val scala211 = SemanticVersion("2.11").get
val scala210 = SemanticVersion("2.10").get
val sbt013 = SemanticVersion("0.13").get
val sbt10 = SemanticVersion("1.0").get
val scalaJs06 = SemanticVersion("0.6").get
val native03 = SemanticVersion("0.3").get
val scala213 = MinorBinary(2, 13)
val scala212 = MinorBinary(2, 12)
val scala211 = MinorBinary(2, 11)
val scala210 = MinorBinary(2, 10)
val sbt013 = MinorBinary(0, 13)
val sbt10 = MinorBinary(1, 0)
val scalaJs06 = MinorBinary(0, 6)
val native03 = MinorBinary(0, 3)

val allTargets = List(
ScalaTarget.scala(scala213),
ScalaTarget.scala(scala212),
ScalaTarget.scala(scala211),
ScalaTarget.scala(scala210),
ScalaTarget.sbt(scala210, sbt013),
ScalaTarget.sbt(scala212, sbt10),
ScalaTarget.scalaJs(scala213, scalaJs06),
ScalaTarget.scalaJs(scala212, scalaJs06),
ScalaTarget.scalaJs(scala211, scalaJs06),
ScalaTarget.scalaJs(scala210, scalaJs06),
ScalaTarget.scalaNative(scala211, native03)
ScalaJvm(scala213),
ScalaJvm(scala212),
ScalaJvm(scala211),
ScalaJvm(scala210),
SbtPlugin(scala210, sbt013),
SbtPlugin(scala212, sbt10),
ScalaJs(scala213, scalaJs06),
ScalaJs(scala212, scalaJs06),
ScalaJs(scala211, scalaJs06),
ScalaJs(scala210, scalaJs06),
ScalaNative(scala211, native03)
)

val releasesDownloads =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ class SeedElasticSearch(paths: DataPaths, githubDownload: GithubDownload)(
),
keywordField("version"),
keywordField("targetType") normalizer "lowercase",
keywordField("fullScalaVersion") normalizer "lowercase",
keywordField("scalaVersion"),
keywordField("scalaJsVersion"),
keywordField("scalaNativeVersion"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,15 @@ trait ProjectProtocol {
classOf[Milestone],
classOf[ReleaseCandidate],
classOf[OtherPreRelease],
classOf[BintrayResolver]
classOf[BintrayResolver],
classOf[ScalaJvm],
classOf[ScalaJs],
classOf[ScalaNative],
classOf[SbtPlugin],
classOf[MajorBinary],
classOf[MinorBinary],
classOf[PatchBinary],
classOf[PreReleaseBinary]
)
)
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@ package data
package project

import cleanup._
import model.release.ScalaTarget
import model.{Artifact, SemanticVersion}
import model.release.{SbtPlugin, ScalaJvm, ScalaTarget}
import model.{Artifact, BinaryVersion, SemanticVersion}
import maven.{ReleaseModel, SbtPluginTarget}

import org.slf4j.LoggerFactory

case class ArtifactMeta(
Expand Down Expand Up @@ -37,89 +36,81 @@ class ArtifactMetaExtractor(paths: DataPaths) {
.map(_.lookup)

nonStandardLookup match {
case None => {
case None =>
pom.sbtPluginTarget match {

// This is a usual Scala library (whose artifact name is suffixed by the Scala binary version)
// For example: akka-actors_2.12
case None => {
Artifact(pom.artifactId).map {
case (artifactName, target) =>
case None =>
Artifact.parse(pom.artifactId).map {
case Artifact(artifactName, target) =>
ArtifactMeta(
artifactName = artifactName,
scalaTarget = Some(target),
isNonStandard = false
)
}
}

// Or it can be an sbt-plugin published as a maven style. In such a case the Scala target
// is not suffixed to the artifact name but can be found in the model’s `sbtPluginTarget` member.
case Some(SbtPluginTarget(rawScalaVersion, rawSbtVersion)) => {
SemanticVersion(rawScalaVersion).zip(
SemanticVersion(rawSbtVersion)
) match {
case Some(SbtPluginTarget(rawScalaVersion, rawSbtVersion)) =>
BinaryVersion
.parse(rawScalaVersion)
.zip(
BinaryVersion.parse(rawSbtVersion)
) match {
case List((scalaVersion, sbtVersion)) =>
Some(
ArtifactMeta(
artifactName = pom.artifactId,
scalaTarget =
Some(ScalaTarget.sbt(scalaVersion, sbtVersion)),
scalaTarget = Some(SbtPlugin(scalaVersion, sbtVersion)),
isNonStandard = false
)
)
case _ => {
case _ =>
log.error("Unable to decode the Scala target")
None
}
}
}
}
}

// For example: io.gatling
case Some(ScalaTargetFromPom) => {
case Some(ScalaTargetFromPom) =>
pom.dependencies
.find(
dep =>
dep.groupId == "org.scala-lang" &&
dep.artifactId == "scala-library"
)
.flatMap(dep => SemanticVersion(dep.version))
.map(
version =>
// we assume binary compatibility
ArtifactMeta(
artifactName = pom.artifactId,
scalaTarget =
Some(ScalaTarget.scala(version.copy(patch = None))),
isNonStandard = true
.flatMap(dep => SemanticVersion.parse(dep.version))
.map { version =>
// we assume binary compatibility
ArtifactMeta(
artifactName = pom.artifactId,
scalaTarget = Some(ScalaJvm.fromFullVersion(version)),
isNonStandard = true
)
)
}
}

// For example: typesafe config
case Some(NoScalaTargetPureJavaDependency) => {
case Some(NoScalaTargetPureJavaDependency) =>
Some(
ArtifactMeta(
artifactName = pom.artifactId,
scalaTarget = None,
isNonStandard = true
)
)
}

// For example: scala-compiler
case Some(ScalaTargetFromVersion) => {
case Some(ScalaTargetFromVersion) =>
SemanticVersion(pom.version).map(
version =>
ArtifactMeta(
artifactName = pom.artifactId,
scalaTarget = Some(ScalaTarget.scala(version)),
scalaTarget = Some(ScalaJvm.fromFullVersion(version)),
isNonStandard = true
)
)
}
}
}
}
Loading