feat(cli): add extractor witness gen (#90)

* feat(cli): add extractor witness gen * fix: select subarray param bug * feat: add circuit config to `circuits.json`. Pending: http extraction * feat: add http extractor support * fix: tests * refactor files a bit * fix tests again, arghhhhhhhgstgst * simplify cli a bit more * more cli cleanup * add json proof docs * add http build docs --------- Co-authored-by: Colin Roberts <[email protected]>
pluto · Sep 13, 2024 · fc67ae7 · fc67ae7
1 parent dfd7e62
commit fc67ae7
Show file tree

Hide file tree

Showing 15 changed files with 696 additions and 224 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -6,3 +6,4 @@ edition = "2021"
 serde = { version = "1.0.209", features = ["derive"] }
 serde_json = "1.0.127"
 clap = { version = "4.5.16", features = ["derive"] }
+regex = "1.10.6"
diff --git a/README.md b/README.md
@@ -3,13 +3,13 @@
 </h1>
 
 <div align="center">
-  <a href="https://github.com/pluto/extractor/graphs/contributors">
+  <a href="https://github.com/pluto/parser-attestor/graphs/contributors">
     <img src="https://img.shields.io/github/contributors/pluto/spark?style=flat-square&logo=github&logoColor=8b949e&labelColor=282f3b&color=32c955" alt="Contributors" />
   </a>
-  <a href="https://github.com/pluto/extractor/actions/workflows/test.yaml">
+  <a href="https://github.com/pluto/parser-attestor/actions/workflows/test.yaml">
     <img src="https://img.shields.io/badge/tests-passing-32c955?style=flat-square&logo=github-actions&logoColor=8b949e&labelColor=282f3b" alt="Tests" />
   </a>
-  <a href="https://github.com/pluto/extractor/actions/workflows/lint.yaml">
+  <a href="https://github.com/pluto/parser-attestor/actions/workflows/lint.yaml">
     <img src="https://img.shields.io/badge/lint-passing-32c955?style=flat-square&logo=github-actions&logoColor=8b949e&labelColor=282f3b" alt="Lint" />
   </a>
 </div>
@@ -120,10 +120,10 @@ From the root of this repository, run:
 ```sh
 cargo install --path .
 ```
-to install the `wpbuild` binary.
+to install the `pabuild` binary.
 You can see a help menu with the subcommands by:
 ```sh
-wpbuild --help
+pabuild --help
 ```
 This is our local Rust command line application.
 Please see the [documentation](docs/pabuild.md) for how to use this alongside the other tools.

diff --git a/circuits/http/extractor.circom b/circuits/http/extractor.circom
@@ -78,7 +78,7 @@ template ExtractResponse(DATA_BYTES, maxContentLength) {
         valueStartingIndex[i] <== valueStartingIndex[i-1] + i * (1-isZeroMask[i]) * isPrevStartingIndex[i];
     }
 
-    response <== SelectSubArray(DATA_BYTES, maxContentLength)(dataMask, valueStartingIndex[DATA_BYTES-1]+1, DATA_BYTES - valueStartingIndex[DATA_BYTES-1]);
+    response <== SelectSubArray(DATA_BYTES, maxContentLength)(dataMask, valueStartingIndex[DATA_BYTES-1]+1, maxContentLength);
 }
 
 template ExtractHeaderValue(DATA_BYTES, headerNameLength, maxValueLength) {

diff --git a/circuits/test/http/codegen.test.ts b/circuits/test/http/codegen.test.ts
@@ -41,11 +41,12 @@ interface Response {
 }
 
 
-function executeCodegen(inputFilename: string, outputFilename: string) {
+function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string) {
     return new Promise((resolve, reject) => {
-        const inputPath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", inputFilename);
+        const inputFilePath = join(__dirname, "..", "..", "..", "examples", "http", inputFileName);
+        const lockfilePath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", lockfileName);
 
-        const codegen = spawn("cargo", ["run", "http", "--lockfile", inputPath, "--output-filename", outputFilename]);
+        const codegen = spawn("cargo", ["run", "codegen", "http", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath]);
 
         codegen.stdout.on('data', (data) => {
             console.log(`stdout: ${data}`);
@@ -71,9 +72,10 @@ describe("HTTP :: Codegen :: Request", async () => {
     it("(valid) GET:", async () => {
         let lockfile = "request.lock";
         let inputfile = "get_request.http";
+        let circuitName = "get_request_test";
 
         // generate extractor circuit using codegen
-        await executeCodegen(`${lockfile}.json`, lockfile);
+        await executeCodegen(circuitName, inputfile, `${lockfile}.json`);
 
         const lockData = readLockFile<Request>(`${lockfile}.json`);
         console.log("lockData: ", JSON.stringify(lockData));
@@ -89,7 +91,7 @@ describe("HTTP :: Codegen :: Request", async () => {
 
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${lockfile}`,
+            file: `main/http_${circuitName}`,
             template: "LockHTTPRequest",
             params: params,
         });
@@ -113,9 +115,10 @@ describe("HTTP :: Codegen :: Request", async () => {
     it("(invalid) GET:", async () => {
         let lockfile = "request.lock";
         let inputfile = "get_request.http";
+        let circuitName = "get_request_test";
 
         // generate extractor circuit using codegen
-        await executeCodegen(`${lockfile}.json`, lockfile);
+        await executeCodegen(circuitName, inputfile, `${lockfile}.json`);
 
         const lockData = readLockFile<Request>(`${lockfile}.json`);
 
@@ -130,7 +133,7 @@ describe("HTTP :: Codegen :: Request", async () => {
 
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${lockfile}`,
+            file: `main/http_${circuitName}`,
             template: "LockHTTPRequest",
             params: params,
         });
@@ -159,9 +162,10 @@ describe("HTTP :: Codegen :: Response", async () => {
     it("(valid) GET:", async () => {
         let lockfile = "response.lock";
         let inputfile = "get_response.http";
+        let circuitName = "get_response_test";
 
         // generate extractor circuit using codegen
-        await executeCodegen(`${lockfile}.json`, lockfile);
+        await executeCodegen(circuitName, inputfile, `${lockfile}.json`);
 
         const lockData = readLockFile<Response>(`${lockfile}.json`);
         console.log("lockData: ", JSON.stringify(lockData));
@@ -179,7 +183,7 @@ describe("HTTP :: Codegen :: Response", async () => {
 
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${lockfile}`,
+            file: `main/http_${circuitName}`,
             template: "LockHTTPResponse",
             params: params,
         });
@@ -205,9 +209,10 @@ describe("HTTP :: Codegen :: Response", async () => {
     it("(invalid) GET:", async () => {
         let lockfile = "response.lock";
         let inputfile = "get_response.http";
+        let circuitName = "get_response_test";
 
         // generate extractor circuit using codegen
-        await executeCodegen(`${lockfile}.json`, lockfile);
+        await executeCodegen(circuitName, inputfile, `${lockfile}.json`);
 
         const lockData = readLockFile<Response>(`${lockfile}.json`);
 
@@ -224,7 +229,7 @@ describe("HTTP :: Codegen :: Response", async () => {
 
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${lockfile}`,
+            file: `main/http_${circuitName}`,
             template: "LockHTTPResponse",
             params: params,
         });

diff --git a/circuits/test/http/extractor.test.ts b/circuits/test/http/extractor.test.ts
@@ -31,7 +31,7 @@ describe("HTTP :: body Extractor", async () => {
 
         let output3 = parsedHttp.bodyBytes.slice(0);
         output3.pop();
-        // output3.pop(); // TODO: fails due to shift subarray bug
+        output3.pop();
         generatePassCase(parsedHttp.input, output3, "output length less than actual length");
     });
 
@@ -75,15 +75,6 @@ describe("HTTP :: header Extractor", async () => {
         let parsedHttp = readHTTPInputFile("get_response.http");
 
         generatePassCase(parsedHttp.input, toByte("Content-Length"), toByte(parsedHttp.headers["Content-Length"]), "");
-
-        // let output2 = parsedHttp.bodyBytes.slice(0);
-        // output2.push(0, 0, 0, 0);
-        // generatePassCase(parsedHttp.input, output2, "output length more than actual length");
-
-        // let output3 = parsedHttp.bodyBytes.slice(0);
-        // output3.pop();
-        // // output3.pop(); // TODO: fails due to shift subarray bug
-        // generatePassCase(parsedHttp.input, output3, "output length less than actual length");
     });
 });
 
diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts
@@ -3,11 +3,12 @@ import { join } from "path";
 import { spawn } from "child_process";
 
 
-function executeCodegen(inputFilename: string, outputFilename: string) {
+function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string) {
     return new Promise((resolve, reject) => {
-        const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", inputFilename);
+        const inputFilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "test", inputFileName);
+        const lockfilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", lockfileName);
 
-        const codegen = spawn("cargo", ["run", "json", "--template", inputPath, "--output-filename", outputFilename]);
+        const codegen = spawn("cargo", ["run", "codegen", "json", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath]);
 
         codegen.stdout.on('data', (data) => {
             console.log(`stdout: ${data}`);
@@ -34,13 +35,13 @@ describe("ExtractValue", async () => {
         let filename = "value_string";
 
         // generate extractor circuit using codegen
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
 
         // read JSON input file into bytes
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractStringValue",
             params: [input.length, 1, 1, 0, 1],
         });
@@ -56,11 +57,11 @@ describe("ExtractValue", async () => {
 
     it("two_keys: {\"key1\": \"abc\", \"key2\": \"def\" }", async () => {
         let filename = "two_keys"
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["key2"]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractStringValue",
             params: [input.length, 1, 4, 0, 3],
         });
@@ -71,11 +72,11 @@ describe("ExtractValue", async () => {
 
     it("value_number: {\"k\": 69 }", async () => {
         let filename = "value_number";
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractNumValue",
             params: [input.length, 1, 1, 0, 2],
         });
@@ -88,13 +89,14 @@ describe("ExtractValue", async () => {
 
     it("value_array_string: { \"k\" : [   420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" ,  \"ba\",  \"ccc\", \"d\" ] }", async () => {
         let filename = "value_array_string";
-        await executeCodegen(`${filename}.json`, filename);
+        let inputFileName = "value_array.json";
+        await executeCodegen(`${filename}_test`, inputFileName, `${filename}.json`);
 
         for (let i = 0; i < 4; i++) {
-            let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["b", i]);
+            let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["b", i]);
 
             circuit = await circomkit.WitnessTester(`Extract`, {
-                file: `main/${filename}`,
+                file: `main/json_${filename}_test`,
                 template: "ExtractStringValue",
                 params: [input.length, 2, 1, 0, i, 1, output.length],
             });
@@ -106,13 +108,15 @@ describe("ExtractValue", async () => {
 
     it("value_array_number: { \"k\" : [   420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" ,  \"ba\",  \"ccc\", \"d\" ] }", async () => {
         let filename = "value_array_number";
-        await executeCodegen(`${filename}.json`, filename);
+        let inputFileName = "value_array.json";
+
+        await executeCodegen(`${filename}_test`, inputFileName, `${filename}.json`);
 
         for (let i = 0; i < 4; i++) {
-            let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["k", i]);
+            let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["k", i]);
 
             circuit = await circomkit.WitnessTester(`Extract`, {
-                file: `main/${filename}`,
+                file: `main/json_${filename}_test`,
                 template: "ExtractNumValue",
                 params: [input.length, 2, 1, 0, i, 1, output.length],
             });
@@ -125,13 +129,13 @@ describe("ExtractValue", async () => {
 
     it("value_array_nested: { \"a\": [[1,0],[0,1,3]] }", async () => {
         let filename = "value_array_nested";
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
         let index_0 = 1;
         let index_1 = 0;
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, index_1]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractNumValue",
             params: [input.length, 3, 1, 0, index_0, 1, index_1, 2, 1],
         });
@@ -150,12 +154,12 @@ describe("ExtractValueMultiDepth", () => {
     it("value_object: { \"a\": { \"d\" : \"e\", \"e\": \"c\" }, \"e\": { \"f\": \"a\", \"e\": \"2\" } }", async () => {
         let filename = "value_object";
 
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
 
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["e", "e"]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractStringValue",
             params: [input.length, 3, 1, 0, 1, 1, 1],
         });
@@ -176,14 +180,14 @@ describe("ExtractValueArrayObject", () => {
     it("value_array_object: {\"a\":[{\"b\":[1,4]},{\"c\":\"b\"}]}", async () => {
         let filename = "value_array_object";
 
-        await executeCodegen(`${filename}.json`, filename);
+        await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`);
 
         let index_0 = 0;
         let index_1 = 0;
         let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, "b", index_1]);
 
         circuit = await circomkit.WitnessTester(`Extract`, {
-            file: `main/${filename}`,
+            file: `main/json_${filename}_test`,
             template: "ExtractNumValue",
             params: [input.length, 4, 1, 0, index_0, 1, 1, 2, index_1, 3, 1],
         });