diff --git a/packages/actions/src/helpers/storage.ts b/packages/actions/src/helpers/storage.ts index df2c4b2f..476ad238 100644 --- a/packages/actions/src/helpers/storage.ts +++ b/packages/actions/src/helpers/storage.ts @@ -1,12 +1,13 @@ import { Functions } from "firebase/functions" import mime from "mime-types" -import fs from "fs" +import fs, { createWriteStream } from "fs" import fetch from "@adobe/node-fetch-retry" import https from "https" import { ETagWithPartNumber, ChunkWithUrl, TemporaryParticipantContributionData } from "../types" import { commonTerms } from "./constants" import { completeMultiPartUpload, + generateGetObjectPreSignedUrl, generatePreSignedUrlsParts, openMultiPartUpload, temporaryStoreCurrentContributionMultiPartUploadId, @@ -210,6 +211,41 @@ export const multiPartUpload = async ( ) } +/** + * Download an artifact from S3 (only for authorized users) + * @param cloudFunctions Firebase cloud functions instance. + * @param bucketName Name of the bucket where the artifact is stored. + * @param storagePath Path to the artifact in the bucket. + * @param localPath Path to the local file where the artifact will be saved. + */ +export const downloadCeremonyArtifact = async ( + cloudFunctions: Functions, + bucketName: string, + storagePath: string, + localPath: string +) => { + // Request pre-signed url to make GET download request. + const getPreSignedUrl = await generateGetObjectPreSignedUrl(cloudFunctions, bucketName, storagePath) + + // Make fetch to get info about the artifact. + const response = await fetch(getPreSignedUrl) + + if (response.status !== 200 && !response.ok) + throw new Error( + `There was an erorr while downloading the object ${storagePath} from the bucket ${bucketName}. Please check the function inputs and try again.` + ) + + const content = response.body + // Prepare stream. + const writeStream = createWriteStream(localPath) + + // Write chunk by chunk. + for await (const chunk of content) { + // Write chunk. + writeStream.write(chunk) + } +} + /** * Get R1CS file path tied to a particular circuit of a ceremony in the storage. * @notice each R1CS file in the storage must be stored in the following path: `circuits//`. diff --git a/packages/actions/src/helpers/verification.ts b/packages/actions/src/helpers/verification.ts index 53924485..6403d801 100644 --- a/packages/actions/src/helpers/verification.ts +++ b/packages/actions/src/helpers/verification.ts @@ -1,5 +1,11 @@ import { groth16, zKey } from "snarkjs" import fs from "fs" +import { Firestore, where } from "firebase/firestore" +import { Functions } from "firebase/functions" +import { downloadCeremonyArtifact, getBucketName, getZkeyStorageFilePath } from "./storage" +import { fromQueryToFirebaseDocumentInfo, getCeremonyCircuits, queryCollection } from "./database" +import { commonTerms, finalContributionIndex } from "./constants" +import { formatZkeyIndex } from "./utils" /** * Verify that a zKey is valid @@ -126,3 +132,70 @@ export const exportVerifierAndVKey = async ( const verificationKeyJSONData = await exportVkey(finalZkeyPath) fs.writeFileSync(vKeyLocalPath, JSON.stringify(verificationKeyJSONData)) } + +/** + * Given a ceremony prefix, download all the ceremony artifacts + * @param functions firebase functions instance + * @param firestore firebase firestore instance + * @param ceremonyPrefix ceremony prefix + * @param outputDirectory output directory where to + */ +export const downloadAllCeremonyArtifacts = async ( + functions: Functions, + firestore: Firestore, + ceremonyPrefix: string, + outputDirectory: string +) => { + // mkdir if not exists + if (!fs.existsSync(outputDirectory)) { + fs.mkdirSync(outputDirectory) + } + + if (!process.env.CONFIG_CEREMONY_BUCKET_POSTFIX) + throw new Error("CONFIG_CEREMONY_BUCKET_POSTFIX not set. Please review your env file and try again.") + + // find the ceremony given the prefix + const ceremonyQuery = await queryCollection(firestore, commonTerms.collections.ceremonies.name, [ + where(commonTerms.collections.ceremonies.fields.prefix, "==", ceremonyPrefix) + ]) + // get the data + const ceremonyData = fromQueryToFirebaseDocumentInfo(ceremonyQuery.docs) + if (ceremonyData.length === 0) + throw new Error("Ceremony not found. Please review your ceremony prefix and try again.") + const ceremony = ceremonyData.at(0)! + // reconstruct the bucket name + const bucketName = getBucketName(ceremonyPrefix, process.env.CONFIG_CEREMONY_BUCKET_POSTFIX!) + + const circuits = await getCeremonyCircuits(firestore, ceremony.id) + if (circuits.length === 0) + throw new Error("No circuits found for this ceremony. Please review your ceremony prefix and try again.") + + // for each circuit we have to download artifacts + for (const circuit of circuits) { + // make a directory for storing the circuit artifacts + const circuitDir = `${outputDirectory}/${ceremony.data.prefix}/${circuit.data.prefix}` + fs.mkdirSync(circuitDir, { recursive: true }) + + // get all required file names in storage and for local storage + const {potStoragePath} = circuit.data.files + const potLocalPath = `${circuitDir}/${circuit.data.files.potFilename}` + const {r1csStoragePath} = circuit.data.files + const r1csLocalPath = `${circuitDir}/${circuit.data.files.r1csFilename}` + const contributions = circuit.data.waitingQueue.completedContributions + const zkeyIndex = formatZkeyIndex(contributions) + const lastZKeyStoragePath = getZkeyStorageFilePath( + circuit.data.prefix, + `${circuit.data.prefix}_${zkeyIndex}.zkey` + ) + const lastZKeyLocalPath = `${circuitDir}/${circuit.data.prefix}_${zkeyIndex}.zkey` + const finalZKeyName = `${circuit.data.prefix}_${finalContributionIndex}.zkey` + const finalZkeyPath = getZkeyStorageFilePath(circuit.data.prefix, finalZKeyName) + const finalZKeyLocalPath = `${circuitDir}/${finalZKeyName}` + + // download everything + await downloadCeremonyArtifact(functions, bucketName, potStoragePath, potLocalPath) + await downloadCeremonyArtifact(functions, bucketName, r1csStoragePath, r1csLocalPath) + await downloadCeremonyArtifact(functions, bucketName, lastZKeyStoragePath, lastZKeyLocalPath) + await downloadCeremonyArtifact(functions, bucketName, finalZkeyPath, finalZKeyLocalPath) + } +} diff --git a/packages/actions/src/index.ts b/packages/actions/src/index.ts index 37306c62..01dab9d3 100644 --- a/packages/actions/src/index.ts +++ b/packages/actions/src/index.ts @@ -24,6 +24,7 @@ export { getCeremonyCircuits } from "./helpers/database" export { + downloadAllCeremonyArtifacts, exportVerifierAndVKey, exportVerifierContract, exportVkey, diff --git a/packages/actions/test/unit/verification.test.ts b/packages/actions/test/unit/verification.test.ts index 7cab8842..23dc729a 100644 --- a/packages/actions/test/unit/verification.test.ts +++ b/packages/actions/test/unit/verification.test.ts @@ -3,16 +3,40 @@ import chaiAsPromised from "chai-as-promised" import dotenv from "dotenv" import { cwd } from "process" import fs from "fs" +import { getAuth, signInWithEmailAndPassword } from "firebase/auth" import { + createS3Bucket, + downloadAllCeremonyArtifacts, exportVerifierAndVKey, exportVerifierContract, exportVkey, generateGROTH16Proof, + getBucketName, + getPotStorageFilePath, + getR1csStorageFilePath, + getZkeyStorageFilePath, verifyGROTH16Proof, verifyZKey } from "../../src" -import { envType } from "../utils" +import { + cleanUpMockCeremony, + cleanUpMockUsers, + createMockCeremony, + createMockUser, + deleteAdminApp, + deleteBucket, + deleteObjectFromS3, + envType, + generateUserPasswords, + getStorageConfiguration, + initializeAdminServices, + initializeUserServices, + sleep, + uploadFileToS3 +} from "../utils" import { TestingEnvironment } from "../../src/types/enums" +import { fakeCeremoniesData, fakeUsersData } from "../data/samples" +import { generateFakeCircuit } from "../data/generators" chai.use(chaiAsPromised) dotenv.config() @@ -25,17 +49,56 @@ describe("Verification utilities", () => { let wasmPath: string = "" let zkeyPath: string = "" let vkeyPath: string = "" + let finalZkeyPath: string = "" + let verifierExportPath: string = "" + let vKeyExportPath: string = "" + let r1csPath: string = "" + let potPath: string = "" + let badzkeyPath: string = "" + let wrongZkeyPath: string = "" if (envType === TestingEnvironment.DEVELOPMENT) { wasmPath = `${cwd()}/../actions/test/data/artifacts/circuit.wasm` zkeyPath = `${cwd()}/../actions/test/data/artifacts/circuit_0000.zkey` vkeyPath = `${cwd()}/../actions/test/data/artifacts/verification_key_circuit.json` + finalZkeyPath = `${cwd()}/../actions/test/data/artifacts/circuit-small_00001.zkey` + verifierExportPath = `${cwd()}/../actions/test/data/artifacts/verifier.sol` + vKeyExportPath = `${cwd()}/../actions/test/data/artifacts/vkey.json` + r1csPath = `${cwd()}/../actions/test/data/artifacts/circuit.r1cs` + potPath = `${cwd()}/../actions/test/data/artifacts/powersOfTau28_hez_final_02.ptau` + badzkeyPath = `${cwd()}/../actions/test/data/artifacts/bad_circuit_0000.zkey` + wrongZkeyPath = `${cwd()}/../actions/test/data/artifacts/notcircuit_0000.zkey` } else { wasmPath = `${cwd()}/packages/actions/test/data/artifacts/circuit.wasm` zkeyPath = `${cwd()}/packages/actions/test/data/artifacts/circuit_0000.zkey` vkeyPath = `${cwd()}/packages/actions/test/data/artifacts/verification_key_circuit.json` + finalZkeyPath = `${cwd()}/packages/actions/test/data/artifacts/circuit-small_00001.zkey` + verifierExportPath = `${cwd()}/packages/actions/test/data/artifacts/verifier.sol` + vKeyExportPath = `${cwd()}/packages/actions/test/data/artifacts/vkey.json` + r1csPath = `${cwd()}/packages/actions/test/data/artifacts/circuit.r1cs` + potPath = `${cwd()}/packages/actions/test/data/artifacts/powersOfTau28_hez_final_02.ptau` + badzkeyPath = `${cwd()}/packages/actions/test/data/artifacts/bad_circuit_0000.zkey` + wrongZkeyPath = `${cwd()}/packages/actions/test/data/artifacts/notcircuit_0000.zkey` } + const solidityVersion = "0.8.10" + + const { ceremonyBucketPostfix } = getStorageConfiguration() + + // Initialize admin and user services. + const { adminFirestore, adminAuth } = initializeAdminServices() + const { userApp, userFirestore, userFunctions } = initializeUserServices() + const userAuth = getAuth(userApp) + + const users = [fakeUsersData.fakeUser1] + const passwords = generateUserPasswords(users.length) + + beforeAll(async () => { + for (let i = 0; i < users.length; i++) { + users[i].uid = await createMockUser(userApp, users[i].data.email, passwords[i], true, adminAuth) + } + }) + describe("generateGROTH16Proof", () => { it("should generate a GROTH16 proof", async () => { const inputs = { @@ -87,12 +150,6 @@ describe("Verification utilities", () => { ).to.be.rejected }) }) - - const finalZkeyPath = `${cwd()}/packages/actions/test/data/artifacts/circuit-small_00001.zkey` - const verifierExportPath = `${cwd()}/packages/actions/test/data/artifacts/verifier.sol` - const vKeyExportPath = `${cwd()}/packages/actions/test/data/artifacts/vkey.json` - const solidityVersion = "0.8.10" - describe("exportVerifierContract", () => { if (envType === TestingEnvironment.PRODUCTION) { it("should export the verifier contract", async () => { @@ -152,25 +209,6 @@ describe("Verification utilities", () => { }) }) describe("verifyzKey", () => { - let badzkeyPath: string = "" - let wrongZkeyPath: string = "" - let potPath: string = "" - let r1csPath: string = "" - - if (envType === TestingEnvironment.DEVELOPMENT) { - zkeyPath = `${cwd()}/../actions/test/data/artifacts/circuit_0000.zkey` - badzkeyPath = `${cwd()}/../actions/test/data/artifacts/bad_circuit_0000.zkey` - wrongZkeyPath = `${cwd()}/../actions/test/data/artifacts/notcircuit_0000.zkey` - potPath = `${cwd()}/../actions/test/data/artifacts/powersOfTau28_hez_final_02.ptau` - r1csPath = `${cwd()}/../actions/test/data/artifacts/circuit.r1cs` - } else { - zkeyPath = `${cwd()}/packages/actions/test/data/artifacts/circuit_0000.zkey` - badzkeyPath = `${cwd()}/packages/actions/test/data/artifacts/bad_circuit_0000.zkey` - wrongZkeyPath = `${cwd()}/packages/actions/test/data/artifacts/notcircuit_0000.zkey` - potPath = `${cwd()}/packages/actions/test/data/artifacts/powersOfTau28_hez_final_02.ptau` - r1csPath = `${cwd()}/packages/actions/test/data/artifacts/circuit.r1cs` - } - it("should return true for a valid zkey", async () => { expect(await verifyZKey(r1csPath, zkeyPath, potPath)).to.be.true }) @@ -199,12 +237,120 @@ describe("Verification utilities", () => { ) }) }) - afterAll(() => { + if (envType === TestingEnvironment.PRODUCTION) { + describe("downloadAllCeremonyArtifacts", () => { + const ceremony = fakeCeremoniesData.fakeCeremonyOpenedFixed + + // create a circuit object that suits our needs + const circuits = generateFakeCircuit({ + uid: "000000000000000000A3", + data: { + name: "Circuit", + description: "Short description of Circuit", + prefix: "circuit", + sequencePosition: 1, + fixedTimeWindow: 10, + zKeySizeInBytes: 45020, + lastUpdated: Date.now(), + metadata: { + constraints: 65, + curve: "bn-128", + labels: 79, + outputs: 1, + pot: 2, + privateInputs: 0, + publicInputs: 2, + wires: 67 + }, + template: { + commitHash: "295d995802b152a1dc73b5d0690ce3f8ca5d9b23", + paramsConfiguration: ["2"], + source: "https://github.com/0xjei/circom-starter/blob/dev/circuits/exercise/checkAscendingOrder.circom" + }, + waitingQueue: { + completedContributions: 1, + contributors: [fakeUsersData.fakeUser1.uid, fakeUsersData.fakeUser2.uid], + currentContributor: fakeUsersData.fakeUser1.uid, + failedContributions: 0 + }, + files: { + initialZkeyBlake2bHash: + "eea0a468524a984908bff6de1de09867ac5d5b0caed92c3332fd5ec61004f79505a784df9d23f69f33efbfef016ad3138871fa8ad63b6e8124a9d0721b0e9e32", + initialZkeyFilename: "circuit_00000.zkey", + initialZkeyStoragePath: "circuits/circuit/contributions/circuit_00000.zkey", + potBlake2bHash: + "34379653611c22a7647da22893c606f9840b38d1cb6da3368df85c2e0b709cfdb03a8efe91ce621a424a39fe4d5f5451266d91d21203148c2d7d61cf5298d119", + potFilename: "powersOfTau28_hez_final_02.ptau", + potStoragePath: "pot/powersOfTau28_hez_final_02.ptau", + r1csBlake2bHash: + "0739198d5578a4bdaeb2fa2a1043a1d9cac988472f97337a0a60c296052b82d6cecb6ae7ce503ab9864bc86a38cdb583f2d33877c41543cbf19049510bca7472", + r1csFilename: "circuit.r1cs", + r1csStoragePath: "circuits/circuit/circuit.r1cs" + }, + avgTimings: { + contributionComputation: 0, + fullContribution: 0, + verifyCloudFunction: 0 + }, + compiler: { + commitHash: "ed807764a17ce06d8307cd611ab6b917247914f5", + version: "2.0.5" + } + } + }) + + const bucketName = getBucketName(ceremony.data.prefix!, ceremonyBucketPostfix) + + // the r1cs + const r1csStorageFilePath = getR1csStorageFilePath(circuits.data.prefix!, "circuit.r1cs") + // the last zkey + const zkeyStorageFilePath = getZkeyStorageFilePath(circuits.data.prefix!, "circuit_00000.zkey") + // the final zkey + const finalZkeyStorageFilePath = getZkeyStorageFilePath(circuits.data.prefix!, `circuit_final.zkey`) + // the pot + const potStorageFilePath = getPotStorageFilePath("powersOfTau28_hez_final_02.ptau") + + const outputDirectory = `${cwd()}/packages/actions/test/data/artifacts/verification` + + beforeAll(async () => { + await createMockCeremony(adminFirestore, ceremony, circuits) + await signInWithEmailAndPassword(userAuth, users[0].data.email, passwords[0]) + await createS3Bucket(userFunctions, bucketName) + await sleep(1000) + // upload all files to S3 + await uploadFileToS3(bucketName, r1csStorageFilePath, r1csPath) + await uploadFileToS3(bucketName, zkeyStorageFilePath, zkeyPath) + await uploadFileToS3(bucketName, finalZkeyStorageFilePath, finalZkeyPath) + await uploadFileToS3(bucketName, potStorageFilePath, potPath) + }) + it("should download all artifacts for a ceremony", async () => { + await downloadAllCeremonyArtifacts(userFunctions, userFirestore, ceremony.data.prefix!, outputDirectory) + }) + it("should fail to download all artifacts for a ceremony that does not exist", async () => { + await expect( + downloadAllCeremonyArtifacts(userFunctions, userFirestore, "invalid", outputDirectory) + ).to.be.rejectedWith("Ceremony not found. Please review your ceremony prefix and try again.") + }) + afterAll(async () => { + await cleanUpMockCeremony(adminFirestore, ceremony.uid, circuits.uid) + await deleteObjectFromS3(bucketName, r1csStorageFilePath) + await deleteObjectFromS3(bucketName, zkeyStorageFilePath) + await deleteObjectFromS3(bucketName, finalZkeyStorageFilePath) + await deleteObjectFromS3(bucketName, potStorageFilePath) + await deleteBucket(bucketName) + // remove dir with output + if (fs.existsSync(outputDirectory)) fs.rmSync(outputDirectory, { recursive: true, force: true }) + }) + }) + } + afterAll(async () => { if (fs.existsSync(verifierExportPath)) { fs.unlinkSync(verifierExportPath) } if (fs.existsSync(vKeyExportPath)) { fs.unlinkSync(vKeyExportPath) } + await cleanUpMockUsers(adminAuth, adminFirestore, users) + await deleteAdminApp() }) })