From 8c3605c54d334b26c8b28dcdabf6f58b08a7a037 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 27 Oct 2020 13:27:33 +0100 Subject: [PATCH] Remove the bootstrapping mechanism for scanners The bootstrapping mechanism was sub-optimal anyway (see [1]) and added complexity to the code that was barely needed, so just remove it. Instead, prefer mechanisms like our Dockerfile to add any required scanners a before runtime of ORT. [1] https://github.com/oss-review-toolkit/ort/issues/1580 Signed-off-by: Sebastian Schuberth --- .../kotlin/commands/RequirementsCommand.kt | 3 +- scanner/src/main/kotlin/LocalScanner.kt | 37 ++------------- scanner/src/main/kotlin/scanners/Askalono.kt | 35 -------------- scanner/src/main/kotlin/scanners/BoyterLc.kt | 35 -------------- scanner/src/main/kotlin/scanners/Licensee.kt | 26 ----------- scanner/src/main/kotlin/scanners/ScanCode.kt | 46 ------------------- 6 files changed, 5 insertions(+), 177 deletions(-) diff --git a/cli/src/main/kotlin/commands/RequirementsCommand.kt b/cli/src/main/kotlin/commands/RequirementsCommand.kt index 1015594c9c1b1..e8af6d082852a 100644 --- a/cli/src/main/kotlin/commands/RequirementsCommand.kt +++ b/cli/src/main/kotlin/commands/RequirementsCommand.kt @@ -111,7 +111,6 @@ class RequirementsCommand : CliktCommand(help = "List the required command line println("${category}s:") tools.forEach { tool -> - // TODO: State whether a tool can be bootstrapped, but that requires refactoring of CommandLineTool. val message = buildString { val (prefix, suffix) = if (tool.isInPath()) { try { @@ -136,7 +135,7 @@ class RequirementsCommand : CliktCommand(help = "List the required command line Pair("\t+ ", "Could not determine the version.") } } else { - // Tolerate scanners to be missing as they can be bootstrapped. + // Tolerate scanners to be missing as we only need a single one basically. if (category != "Scanner") { statusCode = statusCode or 4 } diff --git a/scanner/src/main/kotlin/LocalScanner.kt b/scanner/src/main/kotlin/LocalScanner.kt index f1c0bbeb45f49..c76e9540a30f5 100644 --- a/scanner/src/main/kotlin/LocalScanner.kt +++ b/scanner/src/main/kotlin/LocalScanner.kt @@ -86,38 +86,16 @@ abstract class LocalScanner(name: String, config: ScannerConfiguration) : Scanne abstract val resultFileExt: String /** - * The directory the scanner was bootstrapped to, if so. + * The directory the scanner is installed to. */ private val scannerDir by lazy { - val scannerExe = command() - - getPathFromEnvironment(scannerExe)?.parentFile?.takeIf { + getPathFromEnvironment(command())?.parentFile?.takeIf { getVersion(it) == scannerVersion - } ?: run { - if (scannerExe.isNotEmpty()) { - log.info { - "Bootstrapping scanner '$scannerName' as required version $scannerVersion was not found in PATH." - } - - bootstrap().also { - val actualScannerVersion = getVersion(it) - if (actualScannerVersion != scannerVersion) { - throw IOException( - "Bootstrapped scanner version $actualScannerVersion " + - "does not match expected version $scannerVersion." - ) - } - } - } else { - log.info { "Skipping to bootstrap scanner '$scannerName' as it has no executable." } - - File("") - } - } + } ?: File("") } /** - * The required version of the scanner. This is also the version that would get bootstrapped. + * The required version of the scanner. */ protected abstract val scannerVersion: String @@ -133,13 +111,6 @@ abstract class LocalScanner(name: String, config: ScannerConfiguration) : Scanne */ open fun getVersion() = getVersion(scannerDir) - /** - * Bootstrap the scanner to be ready for use, like downloading and / or configuring it. - * - * @return The directory the scanner is installed in. - */ - protected open fun bootstrap(): File = throw NotImplementedError() - /** * Return the configuration of this [LocalScanner]. */ diff --git a/scanner/src/main/kotlin/scanners/Askalono.kt b/scanner/src/main/kotlin/scanners/Askalono.kt index de0d9297fe875..e60d30b8e90c9 100644 --- a/scanner/src/main/kotlin/scanners/Askalono.kt +++ b/scanner/src/main/kotlin/scanners/Askalono.kt @@ -62,41 +62,6 @@ class Askalono(name: String, config: ScannerConfiguration) : LocalScanner(name, // "askalono --version" returns a string like "askalono 0.2.0-beta.1", so simply remove the prefix. output.removePrefix("askalono ") - override fun bootstrap(): File { - val platform = when { - Os.isLinux -> "Linux" - Os.isMac -> "macOS" - Os.isWindows -> "Windows" - else -> throw IllegalArgumentException("Unsupported operating system.") - } - - val archive = "askalono-$platform.zip" - val url = "https://github.com/amzn/askalono/releases/download/$scannerVersion/$archive" - - log.info { "Downloading $scannerName from $url... " } - - val request = Request.Builder().get().url(url).build() - - return OkHttpClientHelper.execute(request).use { response -> - val body = response.body - - if (response.code != HttpURLConnection.HTTP_OK || body == null) { - throw IOException("Failed to download $scannerName from $url.") - } - - if (response.cacheResponse != null) { - log.info { "Retrieved $scannerName from local cache." } - } - - val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() } - - log.info { "Unpacking '$archive' to '$unpackDir'... " } - body.bytes().unpackZip(unpackDir) - - unpackDir - } - } - override fun getConfiguration() = "" override fun scanPathInternal(path: File, resultsFile: File): ScanResult { diff --git a/scanner/src/main/kotlin/scanners/BoyterLc.kt b/scanner/src/main/kotlin/scanners/BoyterLc.kt index ebae1b14386f3..69713b9e0c114 100644 --- a/scanner/src/main/kotlin/scanners/BoyterLc.kt +++ b/scanner/src/main/kotlin/scanners/BoyterLc.kt @@ -69,41 +69,6 @@ class BoyterLc(name: String, config: ScannerConfiguration) : LocalScanner(name, // "lc --version" returns a string like "licensechecker version 1.1.1", so simply remove the prefix. output.removePrefix("licensechecker version ") - override fun bootstrap(): File { - val platform = when { - Os.isLinux -> "x86_64-unknown-linux" - Os.isMac -> "x86_64-apple-darwin" - Os.isWindows -> "x86_64-pc-windows" - else -> throw IllegalArgumentException("Unsupported operating system.") - } - - val archive = "lc-$scannerVersion-$platform.zip" - val url = "https://github.com/boyter/lc/releases/download/v$scannerVersion/$archive" - - log.info { "Downloading $scannerName from $url... " } - - val request = Request.Builder().get().url(url).build() - - return OkHttpClientHelper.execute(request).use { response -> - val body = response.body - - if (response.code != HttpURLConnection.HTTP_OK || body == null) { - throw IOException("Failed to download $scannerName from $url.") - } - - if (response.cacheResponse != null) { - log.info { "Retrieved $scannerName from local cache." } - } - - val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() } - - log.info { "Unpacking '$archive' to '$unpackDir'... " } - body.bytes().unpackZip(unpackDir) - - unpackDir - } - } - override fun getConfiguration() = CONFIGURATION_OPTIONS.joinToString(" ") override fun scanPathInternal(path: File, resultsFile: File): ScanResult { diff --git a/scanner/src/main/kotlin/scanners/Licensee.kt b/scanner/src/main/kotlin/scanners/Licensee.kt index 98f529283f841..a6e03e407f8c0 100644 --- a/scanner/src/main/kotlin/scanners/Licensee.kt +++ b/scanner/src/main/kotlin/scanners/Licensee.kt @@ -60,32 +60,6 @@ class Licensee(name: String, config: ScannerConfiguration) : LocalScanner(name, override fun getVersionArguments() = "version" - override fun bootstrap(): File { - val gem = if (Os.isWindows) "gem.cmd" else "gem" - - if (Os.isWindows) { - // Version 0.28.0 of rugged broke building on Windows and the fix is unreleased yet, see - // https://github.com/libgit2/rugged/commit/2f5a8f6c8f4ae9b94a2d1f6ffabc315f2592868d. So install the latest - // version < 0.28.0 (and => 0.24.0) manually to satisfy Licensee's needs. - ProcessCapture(gem, "install", "rugged", "-v", "0.27.10.1").requireSuccess() - } - - // Work around Travis CI not being able to handle gem user installs, see - // https://github.com/travis-ci/travis-ci/issues/9412. - return if (Ci.isTravis) { - ProcessCapture(gem, "install", "licensee", "-v", scannerVersion).requireSuccess() - getPathFromEnvironment(command())?.parentFile - ?: throw IOException("Install directory for licensee not found.") - } else { - ProcessCapture(gem, "install", "--user-install", "licensee", "-v", scannerVersion).requireSuccess() - - val ruby = ProcessCapture("ruby", "-r", "rubygems", "-e", "puts Gem.user_dir").requireSuccess() - val userDir = ruby.stdout.trimEnd() - - File(userDir, "bin") - } - } - override fun getConfiguration() = CONFIGURATION_OPTIONS.joinToString(" ") override fun scanPathInternal(path: File, resultsFile: File): ScanResult { diff --git a/scanner/src/main/kotlin/scanners/ScanCode.kt b/scanner/src/main/kotlin/scanners/ScanCode.kt index f9520a7e069f0..50342c42b1ccb 100644 --- a/scanner/src/main/kotlin/scanners/ScanCode.kt +++ b/scanner/src/main/kotlin/scanners/ScanCode.kt @@ -154,52 +154,6 @@ class ScanCode( return output.lineSequence().first { it.startsWith(prefix) }.substring(prefix.length) } - override fun bootstrap(): File { - val versionWithoutHypen = scannerVersion.replace("-", "") - - val archive = when { - // Use the .zip file despite it being slightly larger than the .tar.gz file here as the latter for some - // reason does not complete to unpack on Windows. - Os.isWindows -> "v$versionWithoutHypen.zip" - else -> "v$versionWithoutHypen.tar.gz" - } - - // Use the source code archive instead of the release artifact from S3 to enable OkHttp to cache the download - // locally. For details see https://github.com/square/okhttp/issues/4355#issuecomment-435679393. - val url = "https://github.com/nexB/scancode-toolkit/archive/$archive" - - log.info { "Downloading $scannerName from $url... " } - - val request = Request.Builder().get().url(url).build() - - return OkHttpClientHelper.execute(request).use { response -> - val body = response.body - - if (response.code != HttpURLConnection.HTTP_OK || body == null) { - throw IOException("Failed to download $scannerName from $url.") - } - - if (response.cacheResponse != null) { - log.info { "Retrieved $scannerName from local cache." } - } - - val scannerArchive = createTempFile(ORT_NAME, "$scannerName-${url.substringAfterLast("/")}") - scannerArchive.sink().buffer().use { it.writeAll(body.source()) } - - val unpackDir = createTempDir(ORT_NAME, "$scannerName-$scannerVersion").apply { deleteOnExit() } - - log.info { "Unpacking '$scannerArchive' to '$unpackDir'... " } - scannerArchive.unpack(unpackDir) - if (!scannerArchive.delete()) { - log.warn { "Unable to delete temporary file '$scannerArchive'." } - } - - val scannerDir = unpackDir.resolve("scancode-toolkit-$versionWithoutHypen") - - scannerDir - } - } - override fun getConfiguration() = configurationOptions.toMutableList().run { add(OUTPUT_FORMAT_OPTION)