feat(contract-verifier): Download compilers from GH automatically (#3291

)

Resolves #3257

- Adds resolver multiplexer: implementation of resolver that would go
through list of resolvers, trying to find one that works.
- Adds GH resolver: implementation of resolver that is able to fetch
releases from GH dynamically
- Adds a task to periodically (1h) update the list of supported compiler
versions.
- GH resolver has a temporary kill switch: it can be disabled by setting
`DISABLE_GITHUB_RESOLVER` env variable (in case we notice any problems).

⚠️ This PR doesn't contain any new tests, because I'm not sure if we
want to rely on the availability of GH API. Mocking relevant parts (e.g.
creating an alternative mock client for octocrab) would be _very_
troublesome, so idk if it's justified. Happy to discuss.

I've checked the functionality manually though.

Cargo.toml

@@ -124,6 +124,7 @@ ethabi = "18.0.0"
 flate2 = "1.0.28"
 fraction = "0.15.3"
 futures = "0.3"
+futures-util = "0.3"
 glob = "0.3"
 google-cloud-auth = "0.16.0"
 google-cloud-storage = "0.20.0"
@@ -142,6 +143,7 @@ mini-moka = "0.10.0"
 num = "0.4.0"
 num_cpus = "1.13"
 num_enum = "0.7.2"
+octocrab = "0.41"
 once_cell = "1"
 opentelemetry = "0.24.0"
 opentelemetry_sdk = "0.24.0"

core/bin/contract-verifier/src/main.rs

@@ -55,7 +55,9 @@ async fn main() -> anyhow::Result<()> {
     let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool)
         .await
         .context("failed initializing contract verifier")?;
+    let update_task = contract_verifier.sync_compiler_versions_task();
     let tasks = vec![
+        tokio::spawn(update_task),
         tokio::spawn(contract_verifier.run(stop_receiver.clone(), opt.jobs_number)),
         tokio::spawn(
             PrometheusExporterConfig::pull(prometheus_config.listener_port).run(stop_receiver),

core/lib/contract_verifier/Cargo.toml

@@ -28,8 +28,12 @@ hex.workspace = true
 serde = { workspace = true, features = ["derive"] }
 tempfile.workspace = true
 regex.workspace = true
+reqwest.workspace = true
 tracing.workspace = true
 semver.workspace = true
+octocrab = { workspace = true, features = ["stream"] }
+futures-util.workspace = true
+rustls.workspace = true
 
 [dev-dependencies]
 zksync_node_test_utils.workspace = true

core/lib/contract_verifier/src/lib.rs

@@ -9,6 +9,7 @@ use std::{
 use anyhow::Context as _;
 use chrono::Utc;
 use ethabi::{Contract, Token};
+use resolver::{GitHubCompilerResolver, ResolverMultiplexer};
 use tokio::time;
 use zksync_dal::{contract_verification_dal::DeployedContractData, ConnectionPool, Core, CoreDal};
 use zksync_queued_job_processor::{async_trait, JobProcessor};
@@ -121,34 +122,63 @@ impl ContractVerifier {
         compilation_timeout: Duration,
         connection_pool: ConnectionPool<Core>,
     ) -> anyhow::Result<Self> {
-        Self::with_resolver(
-            compilation_timeout,
-            connection_pool,
-            Arc::<EnvCompilerResolver>::default(),
-        )
-        .await
+        let env_resolver = Arc::<EnvCompilerResolver>::default();
+        let gh_resolver = Arc::new(GitHubCompilerResolver::new().await?);
+        let mut resolver = ResolverMultiplexer::new(env_resolver);
+
+        // Killer switch: if anything goes wrong with GH resolver, we can disable it without having to rollback.
+        // TODO: Remove once GH resolver is proven to be stable.
+        let disable_gh_resolver = std::env::var("DISABLE_GITHUB_RESOLVER").is_ok();
+        if !disable_gh_resolver {
+            resolver = resolver.with_resolver(gh_resolver);
+        } else {
+            tracing::warn!("GitHub resolver was disabled via DISABLE_GITHUB_RESOLVER env variable")
+        }
+
+        Self::with_resolver(compilation_timeout, connection_pool, Arc::new(resolver)).await
     }
 
     async fn with_resolver(
         compilation_timeout: Duration,
         connection_pool: ConnectionPool<Core>,
         compiler_resolver: Arc<dyn CompilerResolver>,
     ) -> anyhow::Result<Self> {
-        let this = Self {
+        Self::sync_compiler_versions(compiler_resolver.as_ref(), &connection_pool).await?;
+        Ok(Self {
             compilation_timeout,
             contract_deployer: zksync_contracts::deployer_contract(),
             connection_pool,
             compiler_resolver,
-        };
-        this.sync_compiler_versions().await?;
-        Ok(this)
+        })
+    }
+
+    /// Returns a future that would periodically update the supported compiler versions
+    /// in the database.
+    pub fn sync_compiler_versions_task(
+        &self,
+    ) -> impl std::future::Future<Output = anyhow::Result<()>> {
+        const UPDATE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour.
+
+        let resolver = self.compiler_resolver.clone();
+        let pool = self.connection_pool.clone();
+        async move {
+            loop {
+                tracing::info!("Updating compiler versions");
+                if let Err(err) = Self::sync_compiler_versions(resolver.as_ref(), &pool).await {
+                    tracing::error!("Failed to sync compiler versions: {:?}", err);
+                }
+                tokio::time::sleep(UPDATE_INTERVAL).await;
+            }
+        }
     }
 
     /// Synchronizes compiler versions.
     #[tracing::instrument(level = "debug", skip_all)]
-    async fn sync_compiler_versions(&self) -> anyhow::Result<()> {
-        let supported_versions = self
-            .compiler_resolver
+    async fn sync_compiler_versions(
+        resolver: &dyn CompilerResolver,
+        pool: &ConnectionPool<Core>,
+    ) -> anyhow::Result<()> {
+        let supported_versions = resolver
             .supported_versions()
             .await
             .context("cannot get supported compilers")?;
@@ -163,26 +193,23 @@ impl ContractVerifier {
             "persisting supported compiler versions"
         );
 
-        let mut storage = self
-            .connection_pool
-            .connection_tagged("contract_verifier")
-            .await?;
+        let mut storage = pool.connection_tagged("contract_verifier").await?;
         let mut transaction = storage.start_transaction().await?;
         transaction
             .contract_verification_dal()
-            .set_zksolc_versions(&supported_versions.zksolc)
+            .set_zksolc_versions(&supported_versions.zksolc.into_iter().collect::<Vec<_>>())
             .await?;
         transaction
             .contract_verification_dal()
-            .set_solc_versions(&supported_versions.solc)
+            .set_solc_versions(&supported_versions.solc.into_iter().collect::<Vec<_>>())
             .await?;
         transaction
             .contract_verification_dal()
-            .set_zkvyper_versions(&supported_versions.zkvyper)
+            .set_zkvyper_versions(&supported_versions.zkvyper.into_iter().collect::<Vec<_>>())
             .await?;
         transaction
             .contract_verification_dal()
-            .set_vyper_versions(&supported_versions.vyper)
+            .set_vyper_versions(&supported_versions.vyper.into_iter().collect::<Vec<_>>())
             .await?;
         transaction.commit().await?;
         Ok(())