From 08fc6386922d3135c11ede97fe115a809108b1a4 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 18:49:30 +0000 Subject: [PATCH 01/21] wip --- render.yaml | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 render.yaml diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000000..69a66e5980 --- /dev/null +++ b/render.yaml @@ -0,0 +1,44 @@ +services: + - type: worker + name: skystrife-indexer + plan: standard + region: ohio + runtime: docker + dockerfilePath: ./Dockerfile + dockerContext: ./ + envVars: + - key: DATABASE_URL + fromDatabase: + name: skystrife-indexer-db + property: connectionString + - key: RPC_HTTP_URL + value: https://rpc.holesky.redstone.xyz + - key: DEBUG + value: mud:store-sync:createStoreSync + - key: STORE_ADDRESS + value: "0x7203e7ADfDF38519e1ff4f8Da7DCdC969371f377" + - key: START_BLOCK + value: 895629 + + - type: web + name: skystrife-indexer-api + plan: pro + region: ohio + runtime: docker + domains: + - indexer.skystrife.xyz + dockerfilePath: ./Dockerfile + dockerContext: ./ + dockerCommand: pnpm tsx bin/postgres-frontend + envVars: + - key: DATABASE_URL + fromDatabase: + name: skystrife-indexer-db + property: connectionString + - key: DEBUG + value: mud:* + +databases: + - name: skystrife-indexer-db + plan: pro + region: ohio From 3d871b7635e81447f86f65b35a303abe2a6f59af Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:20:30 +0000 Subject: [PATCH 02/21] set up render yaml --- render.yaml | 54 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/render.yaml b/render.yaml index 69a66e5980..ca2eb38244 100644 --- a/render.yaml +++ b/render.yaml @@ -1,6 +1,6 @@ services: - type: worker - name: skystrife-indexer + name: base-mainnet-indexer plan: standard region: ohio runtime: docker @@ -9,36 +9,68 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: skystrife-indexer-db + name: base-mainnet-indexer-db property: connectionString + - key: DEBUG + value: mud:store-sync:createStoreSync - key: RPC_HTTP_URL - value: https://rpc.holesky.redstone.xyz + sync: false # prompt for it + - type: worker + name: redstone-holesky-indexer + plan: standard + region: ohio + runtime: docker + dockerfilePath: ./Dockerfile + dockerContext: ./ + envVars: + - key: DATABASE_URL + fromDatabase: + name: redstone-holesky-indexer-db + property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync - - key: STORE_ADDRESS - value: "0x7203e7ADfDF38519e1ff4f8Da7DCdC969371f377" - - key: START_BLOCK - value: 895629 + - key: RPC_HTTP_URL + sync: false # prompt for it - type: web - name: skystrife-indexer-api + name: base-mainnet-indexer-api plan: pro region: ohio runtime: docker domains: - - indexer.skystrife.xyz + - base-mainnet-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: skystrife-indexer-db + name: base-mainnet-indexer-db + property: connectionString + - key: DEBUG + value: mud:* + - type: web + name: redstone-holesky-indexer-api + plan: pro + region: ohio + runtime: docker + domains: + - redstone-holesky-indexer.temp.offchain.dev + dockerfilePath: ./Dockerfile + dockerContext: ./ + dockerCommand: pnpm tsx bin/postgres-frontend + envVars: + - key: DATABASE_URL + fromDatabase: + name: redstone-holesky-indexer-db property: connectionString - key: DEBUG value: mud:* databases: - - name: skystrife-indexer-db + - name: base-mainnet-indexer-db + plan: pro + region: ohio + - name: redstone-holesky-indexer-db plan: pro region: ohio From e60b4db00837dab784ebf9d9b5d5e3d4f26660a3 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:25:30 +0000 Subject: [PATCH 03/21] remove package specific layers --- Dockerfile | 8 -------- render.yaml | 10 ++++++---- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f87cd851f..93cdb2dc35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,11 +39,3 @@ WORKDIR /app RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile RUN pnpm run -r build - -FROM mud AS store-indexer -WORKDIR /app/packages/store-indexer -EXPOSE 3001 - -FROM mud AS faucet -WORKDIR /app/packages/faucet -EXPOSE 3002 diff --git a/render.yaml b/render.yaml index ca2eb38244..35f95fa1c1 100644 --- a/render.yaml +++ b/render.yaml @@ -5,7 +5,8 @@ services: region: ohio runtime: docker dockerfilePath: ./Dockerfile - dockerContext: ./ + dockerContext: ./packages/store-indexer + dockerCommand: pnpm tsx bin/postgres envVars: - key: DATABASE_URL fromDatabase: @@ -21,7 +22,8 @@ services: region: ohio runtime: docker dockerfilePath: ./Dockerfile - dockerContext: ./ + dockerContext: ./packages/store-indexer + dockerCommand: pnpm tsx bin/postgres envVars: - key: DATABASE_URL fromDatabase: @@ -40,7 +42,7 @@ services: domains: - base-mainnet-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile - dockerContext: ./ + dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL @@ -57,7 +59,7 @@ services: domains: - redstone-holesky-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile - dockerContext: ./ + dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL From 754eee64592733fd1633d5f205d623fe446a7415 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:25:59 +0000 Subject: [PATCH 04/21] fix indexer command --- render.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index 35f95fa1c1..7c37d3810f 100644 --- a/render.yaml +++ b/render.yaml @@ -6,7 +6,7 @@ services: runtime: docker dockerfilePath: ./Dockerfile dockerContext: ./packages/store-indexer - dockerCommand: pnpm tsx bin/postgres + dockerCommand: pnpm tsx bin/postgres-indexer envVars: - key: DATABASE_URL fromDatabase: @@ -23,7 +23,7 @@ services: runtime: docker dockerfilePath: ./Dockerfile dockerContext: ./packages/store-indexer - dockerCommand: pnpm tsx bin/postgres + dockerCommand: pnpm tsx bin/postgres-indexer envVars: - key: DATABASE_URL fromDatabase: From 2beafc5a281a1774d73870937a3e2ac198319d72 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:33:38 +0000 Subject: [PATCH 05/21] try prebuilt dockerfile --- Dockerfile | 15 +++++++++++++++ Dockerfile.render | 4 ++++ render.yaml | 8 ++++---- 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.render diff --git a/Dockerfile b/Dockerfile index 93cdb2dc35..31e8acc9eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,13 @@ RUN mkdir -p /etc/apt/keyrings && \ node --version && \ npm --version +# go +ENV PATH="${PATH}:/usr/local/go/bin" +RUN wget https://dl.google.com/go/go1.20.4.linux-amd64.tar.gz && \ + # -C to move to given directory + tar -C /usr/local/ -xzf go1.20.4.linux-amd64.tar.gz && \ + go version + # foundry ENV PATH="${PATH}:/root/.foundry/bin" RUN curl -L https://foundry.paradigm.xyz/ | bash && \ @@ -39,3 +46,11 @@ WORKDIR /app RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile RUN pnpm run -r build + +FROM mud AS store-indexer +WORKDIR /app/packages/store-indexer +EXPOSE 3001 + +FROM mud AS faucet +WORKDIR /app/packages/faucet +EXPOSE 3002 diff --git a/Dockerfile.render b/Dockerfile.render new file mode 100644 index 0000000000..8ce6c67ec8 --- /dev/null +++ b/Dockerfile.render @@ -0,0 +1,4 @@ +FROM ghcr.io/latticexyz/mud:sha-8193136 + +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install +RUN pnpm run -r build diff --git a/render.yaml b/render.yaml index 7c37d3810f..6a823588af 100644 --- a/render.yaml +++ b/render.yaml @@ -4,7 +4,7 @@ services: plan: standard region: ohio runtime: docker - dockerfilePath: ./Dockerfile + dockerfilePath: ./Dockerfile.render dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-indexer envVars: @@ -21,7 +21,7 @@ services: plan: standard region: ohio runtime: docker - dockerfilePath: ./Dockerfile + dockerfilePath: ./Dockerfile.render dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-indexer envVars: @@ -41,7 +41,7 @@ services: runtime: docker domains: - base-mainnet-indexer.temp.offchain.dev - dockerfilePath: ./Dockerfile + dockerfilePath: ./Dockerfile.render dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-frontend envVars: @@ -58,7 +58,7 @@ services: runtime: docker domains: - redstone-holesky-indexer.temp.offchain.dev - dockerfilePath: ./Dockerfile + dockerfilePath: ./Dockerfile.render dockerContext: ./packages/store-indexer dockerCommand: pnpm tsx bin/postgres-frontend envVars: From da962121102977de23bb9ad27c8712b1e736b5ce Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:37:01 +0000 Subject: [PATCH 06/21] try not building --- Dockerfile.render | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.render b/Dockerfile.render index 8ce6c67ec8..405ee22999 100644 --- a/Dockerfile.render +++ b/Dockerfile.render @@ -1,4 +1,4 @@ FROM ghcr.io/latticexyz/mud:sha-8193136 -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install -RUN pnpm run -r build +# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install +# RUN pnpm run -r build From d79c8d6e2673a6759d5468ae5bd4513006cf2490 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:38:51 +0000 Subject: [PATCH 07/21] Revert "try not building" This reverts commit da962121102977de23bb9ad27c8712b1e736b5ce. --- Dockerfile.render | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.render b/Dockerfile.render index 405ee22999..8ce6c67ec8 100644 --- a/Dockerfile.render +++ b/Dockerfile.render @@ -1,4 +1,4 @@ FROM ghcr.io/latticexyz/mud:sha-8193136 -# RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install -# RUN pnpm run -r build +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install +RUN pnpm run -r build From cb88d8d81221670655f9b209d336b082a10b443e Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:45:59 +0000 Subject: [PATCH 08/21] remove go --- Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 31e8acc9eb..0f87cd851f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,13 +19,6 @@ RUN mkdir -p /etc/apt/keyrings && \ node --version && \ npm --version -# go -ENV PATH="${PATH}:/usr/local/go/bin" -RUN wget https://dl.google.com/go/go1.20.4.linux-amd64.tar.gz && \ - # -C to move to given directory - tar -C /usr/local/ -xzf go1.20.4.linux-amd64.tar.gz && \ - go version - # foundry ENV PATH="${PATH}:/root/.foundry/bin" RUN curl -L https://foundry.paradigm.xyz/ | bash && \ From c74fb48b73b5d46bf7f8a29049d2500541bcf948 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 19:47:42 +0000 Subject: [PATCH 09/21] try diff workdir --- Dockerfile.render | 2 ++ render.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile.render b/Dockerfile.render index 8ce6c67ec8..b8afae0d03 100644 --- a/Dockerfile.render +++ b/Dockerfile.render @@ -2,3 +2,5 @@ FROM ghcr.io/latticexyz/mud:sha-8193136 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install RUN pnpm run -r build + +WORKDIR /app/packages/store-indexer diff --git a/render.yaml b/render.yaml index 6a823588af..911106880b 100644 --- a/render.yaml +++ b/render.yaml @@ -5,7 +5,7 @@ services: region: ohio runtime: docker dockerfilePath: ./Dockerfile.render - dockerContext: ./packages/store-indexer + dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-indexer envVars: - key: DATABASE_URL @@ -22,7 +22,7 @@ services: region: ohio runtime: docker dockerfilePath: ./Dockerfile.render - dockerContext: ./packages/store-indexer + dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-indexer envVars: - key: DATABASE_URL @@ -42,7 +42,7 @@ services: domains: - base-mainnet-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile.render - dockerContext: ./packages/store-indexer + dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL @@ -59,7 +59,7 @@ services: domains: - redstone-holesky-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile.render - dockerContext: ./packages/store-indexer + dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL From 4e481c1a4a169076a73dd1b97b01e7accf7253ba Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 20:16:06 +0000 Subject: [PATCH 10/21] add in cache headers --- .../store-indexer/src/postgres/apiRoutes.ts | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/store-indexer/src/postgres/apiRoutes.ts b/packages/store-indexer/src/postgres/apiRoutes.ts index 383700fc5f..06c215f899 100644 --- a/packages/store-indexer/src/postgres/apiRoutes.ts +++ b/packages/store-indexer/src/postgres/apiRoutes.ts @@ -21,6 +21,7 @@ export function apiRoutes(database: Sql): Middleware { options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {}); } catch (e) { ctx.status = 400; + ctx.set("Content-Type", "application/json"); ctx.body = JSON.stringify(e); debug(e); return; @@ -45,10 +46,27 @@ export function apiRoutes(database: Sql): Middleware { } const blockNumber = records[0].chainBlockNumber; - ctx.body = JSON.stringify({ blockNumber, logs }); ctx.status = 200; + + // max age is set to several multiples of the uncached response time (currently ~10s, but using 60s for wiggle room) to ensure only ~one origin request at a time + // and stale-while-revalidate below means that the cache is refreshed under the hood while still responding fast (cached) + const maxAgeSeconds = 60 * 5; + // we set stale-while-revalidate to the time elapsed by the number of blocks we can fetch from the RPC in the same amount of time as an uncached response + // meaning it would take ~the same about of time to get an uncached response from the origin as it would to catch up from the currently cached response + // if an uncached response takes ~10 seconds, we have ~10s to catch up, so let's say we can do enough RPC calls to fetch 4000 blocks + // with a block per 2 seconds, that means we can serve a stale/cached response for 8000 seconds before we should require the response be returned by the origin + const staleWhileRevalidateSeconds = 4000 * 2; + + ctx.set( + "CDN-Cache-Control", + `public, max-age=${maxAgeSeconds}, stale-while-revalidate=${staleWhileRevalidateSeconds}` + ); + + ctx.set("Content-Type", "application/json"); + ctx.body = JSON.stringify({ blockNumber, logs }); } catch (e) { ctx.status = 500; + ctx.set("Content-Type", "application/json"); ctx.body = JSON.stringify(e); error(e); } From b3d59795e37281ee8bf5b5aae307ee718c7d2ac4 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 20:17:28 +0000 Subject: [PATCH 11/21] base sepolia --- render.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/render.yaml b/render.yaml index 911106880b..f1934220b4 100644 --- a/render.yaml +++ b/render.yaml @@ -17,7 +17,7 @@ services: - key: RPC_HTTP_URL sync: false # prompt for it - type: worker - name: redstone-holesky-indexer + name: base-sepolia-indexer plan: standard region: ohio runtime: docker @@ -27,7 +27,7 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: redstone-holesky-indexer-db + name: base-sepolia-indexer-db property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync @@ -52,19 +52,19 @@ services: - key: DEBUG value: mud:* - type: web - name: redstone-holesky-indexer-api + name: base-sepolia-indexer-api plan: pro region: ohio runtime: docker domains: - - redstone-holesky-indexer.temp.offchain.dev + - base-sepolia-indexer.temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: redstone-holesky-indexer-db + name: base-sepolia-indexer-db property: connectionString - key: DEBUG value: mud:* @@ -73,6 +73,6 @@ databases: - name: base-mainnet-indexer-db plan: pro region: ohio - - name: redstone-holesky-indexer-db + - name: base-sepolia-indexer-db plan: pro region: ohio From 7e9fa868ae9da607bc1b9ef5883590cf9ff89f6a Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 20:35:56 +0000 Subject: [PATCH 12/21] revert 404 on empty rows --- packages/store-indexer/src/postgres/apiRoutes.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/store-indexer/src/postgres/apiRoutes.ts b/packages/store-indexer/src/postgres/apiRoutes.ts index 06c215f899..a3b511d34b 100644 --- a/packages/store-indexer/src/postgres/apiRoutes.ts +++ b/packages/store-indexer/src/postgres/apiRoutes.ts @@ -34,17 +34,6 @@ export function apiRoutes(database: Sql): Middleware { const logs = records.map(recordToLog); benchmark("map records to logs"); - if (records.length === 0) { - ctx.status = 404; - ctx.body = "no logs found"; - error( - `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify( - options.filters - )}` - ); - return; - } - const blockNumber = records[0].chainBlockNumber; ctx.status = 200; From f50df33ffba3a53453da9f034b9a1dec993fb286 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Wed, 21 Feb 2024 20:40:18 +0000 Subject: [PATCH 13/21] update hostnames --- render.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index f1934220b4..e076ff8967 100644 --- a/render.yaml +++ b/render.yaml @@ -40,7 +40,7 @@ services: region: ohio runtime: docker domains: - - base-mainnet-indexer.temp.offchain.dev + - base-mainnet.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend @@ -57,7 +57,7 @@ services: region: ohio runtime: docker domains: - - base-sepolia-indexer.temp.offchain.dev + - base-sepolia.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend From 30aaf3df32974e174dd01e9010c396f2596bb53c Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 27 Feb 2024 10:29:08 +0000 Subject: [PATCH 14/21] set up new set of services --- render.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/render.yaml b/render.yaml index e076ff8967..3b148bc007 100644 --- a/render.yaml +++ b/render.yaml @@ -1,6 +1,6 @@ services: - type: worker - name: base-mainnet-indexer + name: base-mainnet-indexer-20240227 plan: standard region: ohio runtime: docker @@ -10,14 +10,14 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: base-mainnet-indexer-db + name: base-mainnet-indexer-db-20240227 property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync - key: RPC_HTTP_URL sync: false # prompt for it - type: worker - name: base-sepolia-indexer + name: base-sepolia-indexer-20240227 plan: standard region: ohio runtime: docker @@ -27,7 +27,7 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: base-sepolia-indexer-db + name: base-sepolia-indexer-db-20240227 property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync @@ -35,44 +35,44 @@ services: sync: false # prompt for it - type: web - name: base-mainnet-indexer-api + name: base-mainnet-indexer-api-20240227 plan: pro region: ohio runtime: docker domains: - - base-mainnet.indexer.sbg-temp.offchain.dev + - base-mainnet-20240227.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: base-mainnet-indexer-db + name: base-mainnet-indexer-db-20240227 property: connectionString - key: DEBUG value: mud:* - type: web - name: base-sepolia-indexer-api + name: base-sepolia-indexer-api-20240227 plan: pro region: ohio runtime: docker domains: - - base-sepolia.indexer.sbg-temp.offchain.dev + - base-sepolia-20240227.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: base-sepolia-indexer-db + name: base-sepolia-indexer-db-20240227 property: connectionString - key: DEBUG value: mud:* databases: - - name: base-mainnet-indexer-db + - name: base-mainnet-indexer-db-20240227 plan: pro region: ohio - - name: base-sepolia-indexer-db + - name: base-sepolia-indexer-db-20240227 plan: pro region: ohio From 9b5469dedbf75f47d280b5a0f014f7c75ccf712b Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 27 Feb 2024 10:54:43 +0000 Subject: [PATCH 15/21] fix base image sha --- Dockerfile.render | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.render b/Dockerfile.render index b8afae0d03..e7ba54addb 100644 --- a/Dockerfile.render +++ b/Dockerfile.render @@ -1,4 +1,4 @@ -FROM ghcr.io/latticexyz/mud:sha-8193136 +FROM ghcr.io/latticexyz/mud:sha-df118eb RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install RUN pnpm run -r build From 96aee4e2b67ed562cf600d05aa1703f338b829c8 Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 27 Feb 2024 11:31:27 +0000 Subject: [PATCH 16/21] update image tag --- Dockerfile.render | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.render b/Dockerfile.render index e7ba54addb..a95af5e241 100644 --- a/Dockerfile.render +++ b/Dockerfile.render @@ -1,4 +1,4 @@ -FROM ghcr.io/latticexyz/mud:sha-df118eb +FROM ghcr.io/latticexyz/mud:sha-3622e39 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install RUN pnpm run -r build From 626bde4c22aec9269dc3bfafe6aaf1a24484048b Mon Sep 17 00:00:00 2001 From: Kevin Ingersoll Date: Tue, 27 Feb 2024 11:31:57 +0000 Subject: [PATCH 17/21] Revert "set up new set of services" This reverts commit 30aaf3df32974e174dd01e9010c396f2596bb53c. --- render.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/render.yaml b/render.yaml index 3b148bc007..e076ff8967 100644 --- a/render.yaml +++ b/render.yaml @@ -1,6 +1,6 @@ services: - type: worker - name: base-mainnet-indexer-20240227 + name: base-mainnet-indexer plan: standard region: ohio runtime: docker @@ -10,14 +10,14 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: base-mainnet-indexer-db-20240227 + name: base-mainnet-indexer-db property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync - key: RPC_HTTP_URL sync: false # prompt for it - type: worker - name: base-sepolia-indexer-20240227 + name: base-sepolia-indexer plan: standard region: ohio runtime: docker @@ -27,7 +27,7 @@ services: envVars: - key: DATABASE_URL fromDatabase: - name: base-sepolia-indexer-db-20240227 + name: base-sepolia-indexer-db property: connectionString - key: DEBUG value: mud:store-sync:createStoreSync @@ -35,44 +35,44 @@ services: sync: false # prompt for it - type: web - name: base-mainnet-indexer-api-20240227 + name: base-mainnet-indexer-api plan: pro region: ohio runtime: docker domains: - - base-mainnet-20240227.indexer.sbg-temp.offchain.dev + - base-mainnet.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: base-mainnet-indexer-db-20240227 + name: base-mainnet-indexer-db property: connectionString - key: DEBUG value: mud:* - type: web - name: base-sepolia-indexer-api-20240227 + name: base-sepolia-indexer-api plan: pro region: ohio runtime: docker domains: - - base-sepolia-20240227.indexer.sbg-temp.offchain.dev + - base-sepolia.indexer.sbg-temp.offchain.dev dockerfilePath: ./Dockerfile.render dockerContext: ./ dockerCommand: pnpm tsx bin/postgres-frontend envVars: - key: DATABASE_URL fromDatabase: - name: base-sepolia-indexer-db-20240227 + name: base-sepolia-indexer-db property: connectionString - key: DEBUG value: mud:* databases: - - name: base-mainnet-indexer-db-20240227 + - name: base-mainnet-indexer-db plan: pro region: ohio - - name: base-sepolia-indexer-db-20240227 + - name: base-sepolia-indexer-db plan: pro region: ohio From 1a52134d6fa5804feee5ff80c70db1e2f519d5c9 Mon Sep 17 00:00:00 2001 From: alvrs Date: Tue, 16 Apr 2024 14:27:23 +0100 Subject: [PATCH 18/21] remove render related changes --- Dockerfile.render | 6 ---- render.yaml | 78 ----------------------------------------------- 2 files changed, 84 deletions(-) delete mode 100644 Dockerfile.render delete mode 100644 render.yaml diff --git a/Dockerfile.render b/Dockerfile.render deleted file mode 100644 index a95af5e241..0000000000 --- a/Dockerfile.render +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/latticexyz/mud:sha-3622e39 - -RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install -RUN pnpm run -r build - -WORKDIR /app/packages/store-indexer diff --git a/render.yaml b/render.yaml deleted file mode 100644 index e076ff8967..0000000000 --- a/render.yaml +++ /dev/null @@ -1,78 +0,0 @@ -services: - - type: worker - name: base-mainnet-indexer - plan: standard - region: ohio - runtime: docker - dockerfilePath: ./Dockerfile.render - dockerContext: ./ - dockerCommand: pnpm tsx bin/postgres-indexer - envVars: - - key: DATABASE_URL - fromDatabase: - name: base-mainnet-indexer-db - property: connectionString - - key: DEBUG - value: mud:store-sync:createStoreSync - - key: RPC_HTTP_URL - sync: false # prompt for it - - type: worker - name: base-sepolia-indexer - plan: standard - region: ohio - runtime: docker - dockerfilePath: ./Dockerfile.render - dockerContext: ./ - dockerCommand: pnpm tsx bin/postgres-indexer - envVars: - - key: DATABASE_URL - fromDatabase: - name: base-sepolia-indexer-db - property: connectionString - - key: DEBUG - value: mud:store-sync:createStoreSync - - key: RPC_HTTP_URL - sync: false # prompt for it - - - type: web - name: base-mainnet-indexer-api - plan: pro - region: ohio - runtime: docker - domains: - - base-mainnet.indexer.sbg-temp.offchain.dev - dockerfilePath: ./Dockerfile.render - dockerContext: ./ - dockerCommand: pnpm tsx bin/postgres-frontend - envVars: - - key: DATABASE_URL - fromDatabase: - name: base-mainnet-indexer-db - property: connectionString - - key: DEBUG - value: mud:* - - type: web - name: base-sepolia-indexer-api - plan: pro - region: ohio - runtime: docker - domains: - - base-sepolia.indexer.sbg-temp.offchain.dev - dockerfilePath: ./Dockerfile.render - dockerContext: ./ - dockerCommand: pnpm tsx bin/postgres-frontend - envVars: - - key: DATABASE_URL - fromDatabase: - name: base-sepolia-indexer-db - property: connectionString - - key: DEBUG - value: mud:* - -databases: - - name: base-mainnet-indexer-db - plan: pro - region: ohio - - name: base-sepolia-indexer-db - plan: pro - region: ohio From 8a535ad25fb0ee9c6256718c2a294659157b1c56 Mon Sep 17 00:00:00 2001 From: alvrs Date: Tue, 16 Apr 2024 14:28:07 +0100 Subject: [PATCH 19/21] remove CDN prefix on Cache-Control --- packages/store-indexer/src/postgres/apiRoutes.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/store-indexer/src/postgres/apiRoutes.ts b/packages/store-indexer/src/postgres/apiRoutes.ts index a3b511d34b..a001b23b21 100644 --- a/packages/store-indexer/src/postgres/apiRoutes.ts +++ b/packages/store-indexer/src/postgres/apiRoutes.ts @@ -47,8 +47,8 @@ export function apiRoutes(database: Sql): Middleware { const staleWhileRevalidateSeconds = 4000 * 2; ctx.set( - "CDN-Cache-Control", - `public, max-age=${maxAgeSeconds}, stale-while-revalidate=${staleWhileRevalidateSeconds}` + "Cache-Control", + `public, max-age=${maxAgeSeconds}, stale-while-revalidate=${staleWhileRevalidateSeconds}`, ); ctx.set("Content-Type", "application/json"); From ce0fef601ab4ff0a9da9cc724eb1f3ede852a634 Mon Sep 17 00:00:00 2001 From: alvarius Date: Tue, 16 Apr 2024 14:32:54 +0100 Subject: [PATCH 20/21] Create fifty-dryers-enjoy.md --- .changeset/fifty-dryers-enjoy.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fifty-dryers-enjoy.md diff --git a/.changeset/fifty-dryers-enjoy.md b/.changeset/fifty-dryers-enjoy.md new file mode 100644 index 0000000000..66e9549920 --- /dev/null +++ b/.changeset/fifty-dryers-enjoy.md @@ -0,0 +1,5 @@ +--- +"@latticexyz/store-indexer": patch +--- + +Added `Cache-Control` and `Content-Type` headers to the postgres indexer API. From 1b1ec9eec45550ceb3370fbe30071d3363a93906 Mon Sep 17 00:00:00 2001 From: alvrs Date: Tue, 16 Apr 2024 14:55:15 +0100 Subject: [PATCH 21/21] readd error on no records found --- packages/store-indexer/src/postgres/apiRoutes.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/store-indexer/src/postgres/apiRoutes.ts b/packages/store-indexer/src/postgres/apiRoutes.ts index a001b23b21..01a38f3e0d 100644 --- a/packages/store-indexer/src/postgres/apiRoutes.ts +++ b/packages/store-indexer/src/postgres/apiRoutes.ts @@ -34,6 +34,22 @@ export function apiRoutes(database: Sql): Middleware { const logs = records.map(recordToLog); benchmark("map records to logs"); + // Ideally we would immediately return an error if the request is for a Store that the indexer + // is not configured to index. Since we don't have easy access to this information here, + // we return an error if there are no logs found for a given Store, since that would never + // be the case for a Store that is being indexed (since there would at least be records for the + // Tables table with tables created during Store initialization). + if (records.length === 0) { + ctx.status = 404; + ctx.body = "no logs found"; + error( + `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify( + options.filters, + )}`, + ); + return; + } + const blockNumber = records[0].chainBlockNumber; ctx.status = 200;