Merge branch 'main' into add-exclude-files-field-to-deploy-jar-target

CONTRIBUTORS.md

@@ -110,6 +110,7 @@ Created as part of the release process.
 + Grzegorz Kossakowski
 + Guy Marom
 + Harley Cooper
++ HeetVekariya
 + Henry Fuller
 + Herbert Rusznak
 + Huon Wilson
@@ -319,17 +320,21 @@ Created as part of the release process.
 + Win Wang
 + Wolfram Arnold
 + Xaelias
++ Xuanwo
 + Yarden Shoham
 + Yi Cheng
 + Yoav Alon
 + Yuhan GUO
 + Yujie Chen
++ andreaimprovised
 + azban
 + bianca rosa
 + billybecker
 + eugene yokota
 + gyudoza
 + hephex
++ jgartdele/del
++ offsetcyan
 + olivercoleman-switchdin
 + philipp-sontag-by
 + sww

README_de.md

@@ -0,0 +1,29 @@
+# Pants Build System
+
+Pants ist ein skalierbares Build-System für _Monorepos_: Codebasen, die 
+mehrere Projekte, oft mit verschiedenen Programmiersprachen und Frameworks, in einem 
+einzigen vereinheitlichten Code-Repository enthalten.
+
+Einige bemerkenswerte Funktionen umfassen:
+
+* Explizite Abhängigkeitsmodellierung.
+* Feinkörnige Ungültigmachung.
+* Geteilter Ergebniscache.
+* Parallele Ausführung.
+* Remote-Ausführung.
+* Einheitliche Schnittstelle für verschiedene Tools und Sprachen.
+* Erweiterbarkeit und Anpassbarkeit über eine Plugin-API.
+
+Dokumentation: [www.pantsbuild.org](https://www.pantsbuild.org/).
+
+# Erste Schritte
+
+Siehe die [Anleitung zur ersten Verwendung](https://www.pantsbuild.org/docs/getting-started).
+
+# Anerkennungen
+
+Linux ARM64 CI-Ressourcen werden von [Works on ARM](https://www.arm.com/markets/computing-infrastructure/works-on-arm) bereitgestellt.
+
+macOS CI-Ressourcen werden von [MacStadium](https://www.macstadium.com/) bereitgestellt.
+
+<img width="150" height="61" src="https://uploads-ssl.webflow.com/5ac3c046c82724970fc60918/5c019d917bba312af7553b49_MacStadium-developerlogo.png">

src/python/pants/VERSION

@@ -1 +1 @@
-2.19.0.dev2
+2.19.0.dev3

src/python/pants/core/util_rules/external_tool.py

@@ -65,14 +65,20 @@ class ExternalToolVersion:
     platform: str
     sha256: str
     filesize: int
+    url_override: str | None = None
 
     def encode(self) -> str:
-        return "|".join([self.version, self.platform, self.sha256, str(self.filesize)])
+        parts = [self.version, self.platform, self.sha256, str(self.filesize)]
+        if self.url_override:
+            parts.append(self.url_override)
+        return "|".join(parts)
 
     @classmethod
     def decode(cls, version_str: str) -> ExternalToolVersion:
-        version, platform, sha256, filesize = [x.strip() for x in version_str.split("|")]
-        return cls(version, platform, sha256, int(filesize))
+        parts = [x.strip() for x in version_str.split("|")]
+        version, platform, sha256, filesize = parts[:4]
+        url_override = parts[4] if len(parts) > 4 else None
+        return cls(version, platform, sha256, int(filesize), url_override=url_override)
 
 
 class ExternalToolOptionsMixin:
@@ -112,16 +118,20 @@ def name(cls):
             f"""
         Known versions to verify downloads against.
 
-        Each element is a pipe-separated string of `version|platform|sha256|length`, where:
+        Each element is a pipe-separated string of `version|platform|sha256|length` or
+        `version|platform|sha256|length|url_override`, where:
 
           - `version` is the version string
           - `platform` is one of `[{','.join(Platform.__members__.keys())}]`
           - `sha256` is the 64-character hex representation of the expected sha256
             digest of the download file, as emitted by `shasum -a 256`
           - `length` is the expected length of the download file in bytes, as emitted by
             `wc -c`
+          - (Optional) `url_override` is a specific url to use instead of the normally
+            generated url for this version
 
         E.g., `3.1.2|macos_x86_64|6d0f18cd84b918c7b3edd0203e75569e0c7caecb1367bbbe409b44e28514f5be|42813`.
+        and `3.1.2|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|50926|https://example.mac.org/bin/v3.1.2/mac-aarch64-v3.1.2.tgz`.
 
         Values are space-stripped, so pipes can be indented for readability if necessary.
         """
@@ -204,7 +214,12 @@ def get_request(self, plat: Platform) -> ExternalToolRequest:
         for known_version in self.known_versions:
             version = self.decode_known_version(known_version)
             if plat.value == version.platform and version.version == self.version:
-                return self.get_request_for(version.platform, version.sha256, version.filesize)
+                return self.get_request_for(
+                    version.platform,
+                    version.sha256,
+                    version.filesize,
+                    url_override=version.url_override,
+                )
         raise UnknownVersion(
             softwrap(
                 f"""
@@ -228,12 +243,14 @@ def split_known_version_str(cls, known_version: str) -> tuple[str, str, str, int
         version = cls.decode_known_version(known_version)
         return version.version, version.platform, version.sha256, version.filesize
 
-    def get_request_for(self, plat_val: str, sha256: str, length: int) -> ExternalToolRequest:
+    def get_request_for(
+        self, plat_val: str, sha256: str, length: int, url_override: str | None = None
+    ) -> ExternalToolRequest:
         """Generate a request for this tool from the given info."""
         plat = Platform(plat_val)
         digest = FileDigest(fingerprint=sha256, serialized_bytes_length=length)
         try:
-            url = self.generate_url(plat)
+            url = url_override or self.generate_url(plat)
             exe = self.generate_exe(plat)
         except ExternalToolError as e:
             raise ExternalToolError(

src/python/pants/core/util_rules/external_tool_test.py

@@ -32,13 +32,16 @@ class FooBar(ExternalTool):
         "3.2.0|linux_x86_64|c0c667fb679a8221bed01bffeed1f80727c6c7827d0cbd8f162195efb12df9e4|121212",
         "3.4.7|macos_x86_64|9d0e18cd74b918c7b3edd0203e75569e0c8caecb1367b3be409b45e28514f5be|123321",
         "3.4.7|linux_x86_64|a019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5|134213",
+        "3.4.7|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|145678|https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
     ]
 
     def generate_url(self, plat: Platform) -> str:
         if plat == Platform.macos_x86_64:
             plat_str = "osx-x86_64"
         elif plat == Platform.linux_x86_64:
             plat_str = "linux-x86_64"
+        elif plat == Platform.macos_arm64:
+            plat_str = "osx-aarch64"
         else:
             raise ExternalToolError()
         return f"https://foobar.org/bin/v{self.version}/foobar-{self.version}-{plat_str}.tgz"
@@ -57,11 +60,12 @@ class TemplatedFooBar(TemplatedExternalTool):
         "3.2.0|linux_x86_64|c0c667fb679a8221bed01bffeed1f80727c6c7827d0cbd8f162195efb12df9e4|121212",
         "3.4.7|macos_x86_64|9d0e18cd74b918c7b3edd0203e75569e0c8caecb1367b3be409b45e28514f5be|123321",
         "3.4.7|linux_x86_64|a019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5|134213",
+        "3.4.7|macos_arm64 |aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291|145678|https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
     ]
     default_url_template = "https://foobar.org/bin/v{version}/foobar-{version}-{platform}.tgz"
     default_url_platform_mapping = {
         "macos_x86_64": "osx-x86_64",
-        "macos_arm64": "osx-x86_64",
+        "macos_arm64": "osx-aarch64",
         "linux_x86_64": "linux-x86_64",
     }
 
@@ -108,6 +112,13 @@ def do_test(
         Platform.linux_x86_64,
         "3.4.7",
     )
+    do_test(
+        "https://macfoo.org/bin/v3.4.7/mac-m1-v3.4.7.tgz",
+        145678,
+        "aca5c1da0192e2fd46b7b55ab290a92c5f07309e7b0ebf4e45ba95731ae98291",
+        Platform.macos_arm64,
+        "3.4.7",
+    )
 
     with pytest.raises(UnknownVersion):
         create_subsystem(

src/rust/engine/fs/store/src/local.rs

@@ -22,6 +22,7 @@ use task_executor::Executor;
 use tempfile::Builder;
 use tokio::fs::hard_link;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt};
+use tokio::sync::{Semaphore, SemaphorePermit};
 use workunit_store::ObservationMetric;
 
 /// How big a file must be to be stored as a file on disk.
@@ -58,6 +59,7 @@ trait UnderlyingByteStore {
         initial_lease: bool,
         src_is_immutable: bool,
         expected_digest: Digest,
+        file_source: &FileSource,
         src: PathBuf,
     ) -> Result<(), String>;
 
@@ -113,13 +115,18 @@ impl UnderlyingByteStore for ShardedLmdb {
         initial_lease: bool,
         src_is_immutable: bool,
         expected_digest: Digest,
+        _file_source: &FileSource,
         src: PathBuf,
     ) -> Result<(), String> {
         self.store(
             initial_lease,
             src_is_immutable,
             expected_digest,
-            move || std::fs::File::open(&src),
+            move || {
+                // NB: This file access is bounded by the number of blocking threads on the runtime, and
+                // so we don't bother to acquire against the file handle limit in this case.
+                std::fs::File::open(&src)
+            },
         )
         .await
     }
@@ -396,11 +403,13 @@ impl UnderlyingByteStore for ShardedFSDB {
         _initial_lease: bool,
         src_is_immutable: bool,
         expected_digest: Digest,
+        file_source: &FileSource,
         src: PathBuf,
     ) -> Result<(), String> {
         let mut attempts = 0;
         loop {
-            let reader = tokio::fs::File::open(src.clone())
+            let (reader, _permit) = file_source
+                .open_readonly(&src)
                 .await
                 .map_err(|e| format!("Failed to open {src:?}: {e}"))?;
 
@@ -519,6 +528,29 @@ impl UnderlyingByteStore for ShardedFSDB {
     }
 }
 
+/// A best-effort limit on the number of concurrent attempts to open files.
+#[derive(Debug)]
+struct FileSource {
+    open_files: Semaphore,
+}
+
+impl FileSource {
+    async fn open_readonly(
+        &self,
+        path: &Path,
+    ) -> Result<(tokio::fs::File, SemaphorePermit), String> {
+        let permit = self
+            .open_files
+            .acquire()
+            .await
+            .map_err(|e| format!("Failed to acquire permit to open file: {e}"))?;
+        let file = tokio::fs::File::open(path)
+            .await
+            .map_err(|e| e.to_string())?;
+        Ok((file, permit))
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct ByteStore {
     inner: Arc<InnerStore>,
@@ -532,6 +564,7 @@ struct InnerStore {
     file_lmdb: Result<Arc<ShardedLmdb>, String>,
     directory_lmdb: Result<Arc<ShardedLmdb>, String>,
     file_fsdb: ShardedFSDB,
+    file_source: FileSource,
 }
 
 impl ByteStore {
@@ -582,6 +615,13 @@ impl ByteStore {
                     dest_initializer: Arc::new(Mutex::default()),
                     hardlinkable_destinations: Arc::new(Mutex::default()),
                 },
+                // NB: This is much larger than the number of cores on modern machines, but still small
+                // enough to be a "reasonable" number of open files to set in `ulimit`. This is a
+                // best-effort limit (because it does-not/cannot cover all of the places where we open
+                // files).
+                file_source: FileSource {
+                    open_files: Semaphore::new(1024),
+                },
             }),
         })
     }
@@ -790,17 +830,28 @@ impl ByteStore {
         src_is_immutable: bool,
         src: PathBuf,
     ) -> Result<Digest, String> {
-        let mut file = tokio::fs::File::open(src.clone())
-            .await
-            .map_err(|e| format!("Failed to open {src:?}: {e}"))?;
-        let digest = async_copy_and_hash(&mut file, &mut tokio::io::sink())
-            .await
-            .map_err(|e| format!("Failed to hash {src:?}: {e}"))?;
+        let digest = {
+            let (mut file, _permit) = self
+                .inner
+                .file_source
+                .open_readonly(&src)
+                .await
+                .map_err(|e| format!("Failed to open {src:?}: {e}"))?;
+            async_copy_and_hash(&mut file, &mut tokio::io::sink())
+                .await
+                .map_err(|e| format!("Failed to hash {src:?}: {e}"))?
+        };
 
         if ByteStore::should_use_fsdb(entry_type, digest.size_bytes) {
             self.inner
                 .file_fsdb
-                .store(initial_lease, src_is_immutable, digest, src)
+                .store(
+                    initial_lease,
+                    src_is_immutable,
+                    digest,
+                    &self.inner.file_source,
+                    src,
+                )
                 .await?;
         } else {
             let dbs = match entry_type {
@@ -809,6 +860,8 @@ impl ByteStore {
             };
             let _ = dbs
                 .store(initial_lease, src_is_immutable, digest, move || {
+                    // NB: This file access is bounded by the number of blocking threads on the runtime, and
+                    // so we don't bother to acquire against the file handle limit in this case.
                     std::fs::File::open(&src)
                 })
                 .await;