Merge pull request #345 from ctrlc03/feat/download_ceremony_artifacts

feat(download ceremony artifacts): implemented utilities to download a ceremony artifacts

packages/actions/src/helpers/storage.ts

@@ -1,12 +1,13 @@
 import { Functions } from "firebase/functions"
 import mime from "mime-types"
-import fs from "fs"
+import fs, { createWriteStream } from "fs"
 import fetch from "@adobe/node-fetch-retry"
 import https from "https"
 import { ETagWithPartNumber, ChunkWithUrl, TemporaryParticipantContributionData } from "../types"
 import { commonTerms } from "./constants"
 import {
     completeMultiPartUpload,
+    generateGetObjectPreSignedUrl,
     generatePreSignedUrlsParts,
     openMultiPartUpload,
     temporaryStoreCurrentContributionMultiPartUploadId,
@@ -210,6 +211,41 @@ export const multiPartUpload = async (
     )
 }
 
+/**
+ * Download an artifact from S3 (only for authorized users)
+ * @param cloudFunctions <Functions> Firebase cloud functions instance.
+ * @param bucketName <string> Name of the bucket where the artifact is stored.
+ * @param storagePath <string> Path to the artifact in the bucket.
+ * @param localPath <string> Path to the local file where the artifact will be saved.
+ */
+export const downloadCeremonyArtifact = async (
+    cloudFunctions: Functions,
+    bucketName: string,
+    storagePath: string,
+    localPath: string
+) => {
+    // Request pre-signed url to make GET download request.
+    const getPreSignedUrl = await generateGetObjectPreSignedUrl(cloudFunctions, bucketName, storagePath)
+
+    // Make fetch to get info about the artifact.
+    const response = await fetch(getPreSignedUrl)
+
+    if (response.status !== 200 && !response.ok)
+        throw new Error(
+            `There was an erorr while downloading the object ${storagePath} from the bucket ${bucketName}. Please check the function inputs and try again.`
+        )
+
+    const content = response.body
+    // Prepare stream.
+    const writeStream = createWriteStream(localPath)
+
+    // Write chunk by chunk.
+    for await (const chunk of content) {
+        // Write chunk.
+        writeStream.write(chunk)
+    }
+}
+
 /**
  * Get R1CS file path tied to a particular circuit of a ceremony in the storage.
  * @notice each R1CS file in the storage must be stored in the following path: `circuits/<circuitPrefix>/<completeR1csFilename>`.

packages/actions/src/helpers/verification.ts

@@ -1,5 +1,11 @@
 import { groth16, zKey } from "snarkjs"
 import fs from "fs"
+import { Firestore, where } from "firebase/firestore"
+import { Functions } from "firebase/functions"
+import { downloadCeremonyArtifact, getBucketName, getZkeyStorageFilePath } from "./storage"
+import { fromQueryToFirebaseDocumentInfo, getCeremonyCircuits, queryCollection } from "./database"
+import { commonTerms, finalContributionIndex } from "./constants"
+import { formatZkeyIndex } from "./utils"
 
 /**
  * Verify that a zKey is valid
@@ -126,3 +132,70 @@ export const exportVerifierAndVKey = async (
     const verificationKeyJSONData = await exportVkey(finalZkeyPath)
     fs.writeFileSync(vKeyLocalPath, JSON.stringify(verificationKeyJSONData))
 }
+
+/**
+ * Given a ceremony prefix, download all the ceremony artifacts
+ * @param functions <Functions> firebase functions instance
+ * @param firestore <Firestore> firebase firestore instance
+ * @param ceremonyPrefix <string> ceremony prefix
+ * @param outputDirectory <string> output directory where to
+ */
+export const downloadAllCeremonyArtifacts = async (
+    functions: Functions,
+    firestore: Firestore,
+    ceremonyPrefix: string,
+    outputDirectory: string
+) => {
+    // mkdir if not exists
+    if (!fs.existsSync(outputDirectory)) {
+        fs.mkdirSync(outputDirectory)
+    }
+
+    if (!process.env.CONFIG_CEREMONY_BUCKET_POSTFIX)
+        throw new Error("CONFIG_CEREMONY_BUCKET_POSTFIX not set. Please review your env file and try again.")
+
+    // find the ceremony given the prefix
+    const ceremonyQuery = await queryCollection(firestore, commonTerms.collections.ceremonies.name, [
+        where(commonTerms.collections.ceremonies.fields.prefix, "==", ceremonyPrefix)
+    ])
+    // get the data
+    const ceremonyData = fromQueryToFirebaseDocumentInfo(ceremonyQuery.docs)
+    if (ceremonyData.length === 0)
+        throw new Error("Ceremony not found. Please review your ceremony prefix and try again.")
+    const ceremony = ceremonyData.at(0)!
+    // reconstruct the bucket name
+    const bucketName = getBucketName(ceremonyPrefix, process.env.CONFIG_CEREMONY_BUCKET_POSTFIX!)
+
+    const circuits = await getCeremonyCircuits(firestore, ceremony.id)
+    if (circuits.length === 0)
+        throw new Error("No circuits found for this ceremony. Please review your ceremony prefix and try again.")
+
+    // for each circuit we have to download artifacts
+    for (const circuit of circuits) {
+        // make a directory for storing the circuit artifacts
+        const circuitDir = `${outputDirectory}/${ceremony.data.prefix}/${circuit.data.prefix}`
+        fs.mkdirSync(circuitDir, { recursive: true })
+
+        // get all required file names in storage and for local storage
+        const {potStoragePath} = circuit.data.files
+        const potLocalPath = `${circuitDir}/${circuit.data.files.potFilename}`
+        const {r1csStoragePath} = circuit.data.files
+        const r1csLocalPath = `${circuitDir}/${circuit.data.files.r1csFilename}`
+        const contributions = circuit.data.waitingQueue.completedContributions
+        const zkeyIndex = formatZkeyIndex(contributions)
+        const lastZKeyStoragePath = getZkeyStorageFilePath(
+            circuit.data.prefix,
+            `${circuit.data.prefix}_${zkeyIndex}.zkey`
+        )
+        const lastZKeyLocalPath = `${circuitDir}/${circuit.data.prefix}_${zkeyIndex}.zkey`
+        const finalZKeyName = `${circuit.data.prefix}_${finalContributionIndex}.zkey`
+        const finalZkeyPath = getZkeyStorageFilePath(circuit.data.prefix, finalZKeyName)
+        const finalZKeyLocalPath = `${circuitDir}/${finalZKeyName}`
+
+        // download everything
+        await downloadCeremonyArtifact(functions, bucketName, potStoragePath, potLocalPath)
+        await downloadCeremonyArtifact(functions, bucketName, r1csStoragePath, r1csLocalPath)
+        await downloadCeremonyArtifact(functions, bucketName, lastZKeyStoragePath, lastZKeyLocalPath)
+        await downloadCeremonyArtifact(functions, bucketName, finalZkeyPath, finalZKeyLocalPath)
+    }
+}

packages/actions/src/index.ts

@@ -24,6 +24,7 @@ export {
     getCeremonyCircuits
 } from "./helpers/database"
 export {
+    downloadAllCeremonyArtifacts,
     exportVerifierAndVKey,
     exportVerifierContract,
     exportVkey,