From 27724336df24607ea82aa14dac79953b3b210814 Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Mon, 28 Mar 2022 11:21:49 -0400 Subject: [PATCH 1/5] tests pass --- .../lib/bazel/repository/ArFunction.java | 79 ++++++++++++++++ .../bazel/repository/DecompressorValue.java | 4 +- .../lib/bazel/repository/ArFunctionTest.java | 91 +++++++++++++++++++ .../devtools/build/lib/bazel/repository/BUILD | 1 + .../build/lib/bazel/repository/test_files.ar | 9 ++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java new file mode 100644 index 00000000000000..af365d16c171dc --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java @@ -0,0 +1,79 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.*; +import java.util.Date; +import org.apache.commons.compress.archivers.ar.ArArchiveEntry; +import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; + +/** + * Opens a .ar archive file. It ignores the prefix setting because these archives cannot contain + * directories. + */ +public class ArFunction implements Decompressor { + + public static final Decompressor INSTANCE = new ArFunction(); + + // This is the same value as picked for .tar files, which appears to have worked well. + private static final int BUFFER_SIZE = 32 * 1024; + + private InputStream getDecompressorStream(DecompressorDescriptor descriptor) + throws IOException { + return new BufferedInputStream( + new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE); + }; + + @Override + public Path decompress(DecompressorDescriptor descriptor) + throws InterruptedException, IOException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + try (InputStream decompressorStream = getDecompressorStream(descriptor)) { + ArArchiveInputStream arStream = new ArArchiveInputStream(decompressorStream); + ArArchiveEntry entry; + while ((entry = arStream.getNextArEntry()) != null) { + Path filePath = descriptor.repositoryPath().getRelative(entry.getName()); + filePath.getParentDirectory().createDirectoryAndParents(); + if (entry.isDirectory()) { + // ar archives don't contain any directory information, so this should never + // happen + continue; + } else { + // We do not have to worry about symlinks in .ar files - it's not supported + // by the .ar file format. + try (OutputStream out = filePath.getOutputStream()) { + ByteStreams.copy(arStream, out); + } + filePath.chmod(entry.getMode()); + + Date lastModified = entry.getLastModifiedDate(); + filePath.setLastModifiedTime(lastModified.getTime()); + } + if (Thread.interrupted()) { + throw new InterruptedException(); + } + } + } + + return descriptor.repositoryPath(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java index 49ff20b73b3498..a651c2cd921325 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java @@ -107,11 +107,13 @@ static Decompressor getDecompressor(Path archivePath) return TarZstFunction.INSTANCE; } else if (baseName.endsWith(".tar.bz2")) { return TarBz2Function.INSTANCE; + } else if (baseName.endsWith(".ar") || baseName.endsWith(".deb")) { + return ArFunction.INSTANCE; } else { throw new RepositoryFunctionException( Starlark.errorf( "Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz," - + " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)", + + " .tar.zst, .tzst, .tar.bz2, .ar or .deb suffix (got %s)", archivePath), Transience.PERSISTENT); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java new file mode 100644 index 00000000000000..bfd12f0c08bfb9 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/ArFunctionTest.java @@ -0,0 +1,91 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.testutil.TestUtils; +import com.google.devtools.build.lib.unix.UnixFileSystem; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.File; +import java.io.IOException; + +import com.google.devtools.build.runfiles.Runfiles; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests decompressing archives. */ +@RunWith(JUnit4.class) +public class ArFunctionTest { + /* + * .ar archive created with ar cr test_files.ar archived_first.txt archived_second.md + * The files contain short UTF-8 encoded strings. + */ + private static final String ARCHIVE_NAME = "test_files.ar"; + private static final String PATH_TO_TEST_ARCHIVE = + "/com/google/devtools/build/lib/bazel/repository/"; + private static final String FIRST_FILE_NAME = "archived_first.txt"; + private static final String SECOND_FILE_NAME = "archived_second.md"; + + + @Test + public void testDecompress() throws Exception { + Path outputDir = decompress(createDescriptorBuilder()); + + assertThat(outputDir.exists()).isTrue(); + Path firstFile = outputDir.getRelative(FIRST_FILE_NAME); + assertThat(firstFile.exists()).isTrue(); + // There are 20 bytes in the content "this is test file 1" + assertThat(firstFile.getFileSize()).isEqualTo(20); + assertThat(firstFile.isSymbolicLink()).isFalse(); + + Path secondFile = outputDir.getRelative(SECOND_FILE_NAME); + assertThat(secondFile.exists()).isTrue(); + // There are 20 bytes in the content "this is the second test file" + assertThat(secondFile.getFileSize()).isEqualTo(29); + assertThat(secondFile.isSymbolicLink()).isFalse(); + } + + private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception { + descriptorBuilder.setDecompressor(ArFunction.INSTANCE); + return new ArFunction().decompress(descriptorBuilder.build()); + } + + private DecompressorDescriptor.Builder createDescriptorBuilder() throws IOException { + // This was cribbed from TestArchiveDescriptor + FileSystem testFS = + OS.getCurrent() == OS.WINDOWS + ? new JavaIoFileSystem(DigestHashFunction.SHA256) + : new UnixFileSystem(DigestHashFunction.SHA256, /*hashAttributeName=*/ ""); + + // do not rely on TestConstants.JAVATESTS_ROOT end with slash, but ensure separators + // are not duplicated + String path = + (TestConstants.JAVATESTS_ROOT + PATH_TO_TEST_ARCHIVE + ARCHIVE_NAME).replace("//", "/"); + Path tarballPath = testFS.getPath(Runfiles.create().rlocation(path)); + + Path workingDir = testFS.getPath(new File(TestUtils.tmpDir()).getCanonicalPath()); + Path outDir = workingDir.getRelative("out"); + + return DecompressorDescriptor.builder().setRepositoryPath(outDir).setArchivePath(tarballPath); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD index 2397b27e4bdfba..f9cf05bb792511 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD @@ -27,6 +27,7 @@ java_library( data = [ "test_decompress_archive.tar.gz", "test_decompress_archive.zip", + "test_files.ar", ], deps = [ "//src/main/java/com/google/devtools/build/lib/bazel/repository", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar new file mode 100644 index 00000000000000..8aaddbe34b89de --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/test_files.ar @@ -0,0 +1,9 @@ +! +// 40 ` +archived_first.txt/ +archived_second.md/ +/0 0 0 0 644 20 ` +this is test file 1 +/20 0 0 0 644 29 ` +this is the second test file + From 4d2d9143e8a107dce29aba8de7747a31a7a0e3de Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Mon, 28 Mar 2022 11:44:41 -0400 Subject: [PATCH 2/5] update docs --- .../repository/starlark/StarlarkRepositoryContext.java | 3 ++- .../build/lib/bazel/repository/DecompressorValueTest.java | 4 ++++ tools/build_defs/repo/http.bzl | 7 ++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java index 217f8b0c6661a9..1ef1d77b31fdae 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContext.java @@ -825,7 +825,8 @@ public void extract(Object archive, Object output, String stripPrefix, StarlarkT + " By default, the archive type is determined from the file extension of" + " the URL." + " If the file has no extension, you can explicitly specify either \"zip\"," - + " \"jar\", \"war\", \"aar\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\"" + + " \"jar\", \"war\", \"aar\", \"tar\", \"tar.gz\", \"tgz\", \"tar.xz\"," + + " \"txz\", \".tar.zst\", \".tzst\", \"tar.bz2\", \".ar\", or \".deb\"" + " here."), @Param( name = "stripPrefix", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java index 032da6fa321f39..f457f2b61161d8 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java @@ -56,6 +56,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception { DecompressorDescriptor.builder().setArchivePath(path).build(); path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2"); DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.ar"); + DecompressorDescriptor.builder().setArchivePath(path).build(); + path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.deb"); + DecompressorDescriptor.builder().setArchivePath(path).build(); } @Test diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl index 1bbd2f981a3aed..fd40a5b22afadc 100644 --- a/tools/build_defs/repo/http.bzl +++ b/tools/build_defs/repo/http.bzl @@ -269,7 +269,7 @@ match a directory in the archive, Bazel will return an error.""", By default, the archive type is determined from the file extension of the URL. If the file has no extension, you can explicitly specify one of the following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`, -`"tar.xz"`, or `tar.bz2`.""", +`"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, or `"deb"`.""", ), "patches": attr.label_list( default = [], @@ -357,8 +357,9 @@ http_archive = repository_rule( """Downloads a Bazel repository as a compressed archive file, decompresses it, and makes its targets available for binding. -It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, -`"tar"`, `"tar.gz"`, `"tgz"`, `"tar.xz"`, and `tar.bz2`. +It supports the following file extensions: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, +`"tar.gz"`, `"tgz"`, `"tar.xz"`, `"txz"`, `"tar.zst"`, `"tzst"`, `tar.bz2`, `"ar"`, +or `"deb"`. Examples: Suppose the current repository contains the source code for a chat program, From cdf6f8874c1f8623e3f9a4712624366d7dcd0ad5 Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Tue, 5 Apr 2022 07:39:41 -0400 Subject: [PATCH 3/5] address feedback --- .../build/lib/bazel/repository/ArFunction.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java index af365d16c171dc..cce40356ae3a46 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java @@ -18,7 +18,11 @@ import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor; import com.google.devtools.build.lib.vfs.Path; -import java.io.*; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.Date; import org.apache.commons.compress.archivers.ar.ArArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; @@ -64,10 +68,8 @@ public Path decompress(DecompressorDescriptor descriptor) ByteStreams.copy(arStream, out); } filePath.chmod(entry.getMode()); - - Date lastModified = entry.getLastModifiedDate(); - filePath.setLastModifiedTime(lastModified.getTime()); - } + filePath.setLastModifiedTime(entry.getLastModified()); + } if (Thread.interrupted()) { throw new InterruptedException(); } From 7626755a1f79f0ec3115636d652499ea1e4755bb Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Tue, 5 Apr 2022 07:41:02 -0400 Subject: [PATCH 4/5] unused import --- .../google/devtools/build/lib/bazel/repository/ArFunction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java index cce40356ae3a46..101c4ba05fb205 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.Date; import org.apache.commons.compress.archivers.ar.ArArchiveEntry; import org.apache.commons.compress.archivers.ar.ArArchiveInputStream; From 64bdfc6b6a36e66558629b4f77776f9f646a2025 Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Tue, 5 Apr 2022 07:44:27 -0400 Subject: [PATCH 5/5] seconds to millis --- .../devtools/build/lib/bazel/repository/ArFunction.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java index 101c4ba05fb205..fb19f85d28f2ee 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/ArFunction.java @@ -67,7 +67,9 @@ public Path decompress(DecompressorDescriptor descriptor) ByteStreams.copy(arStream, out); } filePath.chmod(entry.getMode()); - filePath.setLastModifiedTime(entry.getLastModified()); + // entry.getLastModified() appears to be in seconds, so we need to convert + // it into milliseconds for setLastModifiedTime + filePath.setLastModifiedTime(entry.getLastModified() * 1000L); } if (Thread.interrupted()) { throw new InterruptedException();