http-server: Bring over instructions about running with docker (#156)

* put instructions in the right place
* build everything for tests
* reduce unnecessary matrix redundancy
---------

Co-authored-by: andrew edstrom <[email protected]>

.dockerignore

@@ -0,0 +1,2 @@
+**/node_modules
+**/tmp

.github/workflows/example-tests.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}} ;
-          pnpm build;
+          pnpm run -r build;
 
       - name: Test
         run: |

.github/workflows/http-server-tests.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}} ;
-          pnpm build;
+          pnpm run -r build;
 
       - name: Test
         run: |
@@ -71,7 +71,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.14.0, 18.18.0]
+        node-version: [18.18.0]
 
     # Service containers to run with `runner-job`
     services:
@@ -118,6 +118,11 @@ jobs:
           cd ${{env.PKG_DIR}} ;
           pnpm install;
 
+      - name: Build cross-workspace deps
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm run -r build;
+
       - name: Test
         run: |
           cd ${{env.PKG_DIR}} ;

.github/workflows/spa-tests.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}} ;
-          pnpm build;
+          pnpm run -r build;
 
       - name: Test
         run: |

.github/workflows/ts-etl-tests.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}} ;
-          pnpm build;
+          pnpm run -r build;
 
       - name: Test
         run: |

.github/workflows/ts-sdk-tests.yaml

@@ -54,7 +54,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}} ;
-          pnpm build;
+          pnpm run -r build;
 
       - name: Test
         run: |

.github/workflows/widget-tests.yaml

@@ -56,7 +56,7 @@ jobs:
       - name: Verify builds
         run: |
           cd ${{env.PKG_DIR}}
-          pnpm build;
+          pnpm run -r build;
 
       - name: Install Playwright Browsers
         run: |

.gitignore

@@ -134,4 +134,6 @@ dist
 **/.DS_Store
 
 # IntelliJ files
-.idea/**/*
+.idea/**/*
+
+**/tmp

README.md

@@ -18,6 +18,13 @@ cross-package behavior, such as release automation.
 **Releases and tags:** Github Actions uses [multi-semantic-release](https://github.com/dhoulb/multi-semantic-release) to automatically generate semvers based on commit history for each package in the repository. Multiple tags
 are generated for a single commit if it updates multiple packages.
 
+**Cross-dependencies and typescript:** Several packages depend on `ts-sdk`, and will refer to
+the `dist` directory within that package for their source code when making local changes.
+For that reason, you may need to run `pnpm --filter docmaps-sdk run build` or `pnpm run -r build`
+if you are making local changes to uptream dependencies within this monorepo so that your
+downstream code changes will pull those in.
+
+
 **Dependencies:** The workspace root builds a Docker image for the http-server. In addition to the
 npm package dependencies, to do local development you should have the following tools installed:
 
@@ -64,7 +71,6 @@ This Single-page App (SPA) is a simple demonstration of the above tools in actio
 accessible [live on Github Pages](https://docmaps-project.github.io/docmaps/demo/) where you can
 plug in a DOI and get a best-effort view of a Docmap as inferred from Crossref's API.
 
-
 ## Governance
 
 As stated in [CODE_OF_CONDUCT.md](/CODE_OF_CONDUCT.md):

package.json

@@ -5,8 +5,6 @@
   "description": "Top-level Docmaps specification module -- not intended for direct npm consumption.",
   "scripts": {
     "clean": "manypkg exec $npm_execpath run clean && rm -rf node_modules",
-    "compose:repave": "rm -rf tmp/oxigraph_data/*",
-    "compose:up": "docker compose -f $INIT_CWD/docker-compose.local.yml up --build",
     "test": "node scripts/package-tests.js",
     "test:packages": "manypkg exec $npm_execpath test",
     "test:all": "$npm_execpath run test && $npm_execpath run test:packages"

packages/http-client/package.json

@@ -14,7 +14,6 @@
     "lint": "npx eslint .",
     "lint:fix": "npx eslint --fix .",
     "start": "tsx dist/httpserver/main.js",
-    "prepare": "tsc --declaration",
     "build": "tsc --declaration"
   },
   "keywords": [],

Dockerfile → packages/http-server/Dockerfile

@@ -4,6 +4,9 @@
 # If you need more help, visit the Dockerfile reference guide at
 # https://docs.docker.com/engine/reference/builder/
 
+# NOTE: due to use of pnpm workspace, this dockerfile needs to be built
+# from the repository root context.
+
 ARG NODE_VERSION=18.14.0
 ARG PNPM_VERSION=8.7.6
 
@@ -26,9 +29,10 @@ FROM base AS build
 # "devDependencies" to be installed to build. If you don't need this, remove this step.
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
       pnpm install \
+      --filter "@docmaps/http-server" \
       --frozen-lockfile
 
-RUN pnpm run -r build
+RUN pnpm --filter "@docmaps/http-server" run build
 
 ################################################################################
 # Download dependencies as a separate step to take advantage of Docker's caching.
@@ -41,12 +45,16 @@ RUN npm i -g [email protected]
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
       pnpm install \
+      --filter "@docmaps/http-server" \
       --prod \
       --frozen-lockfile
 
 ################################################################################
 # Create a new stage to run the application with minimal runtime dependencies
 # where the necessary files are copied from the build stage.
+
+# FIXME: build from Prod instead, once we have resolved the prepare script issue:
+#   https://github.com/Docmaps-Project/docmaps/issues/118
 FROM prod AS runtime
 
 # Copy the production dependencies from the deps stage and also

packages/http-server/README.md

@@ -18,13 +18,12 @@ domain logic into queries for your specific backend (such as a relational databa
 
 ## Development and Testing
 
-See the readme in repository root for general info about this monorepo.
-
 Dependencies for local development include:
 
 ```
 pnpm
-docker # with docker-compose
+node # @^18
+docker # with docker-compose available
 curl
 ```
 
@@ -39,9 +38,65 @@ pnpm test:integration
 **WARN:** If you have never done this before, the tests may timeout due to invoking a `docker pull` for the `oxigraph` image, which
 is used for a local triplestore to use during integration tests.
 
-## Running locally with Docker and Oxigraph
+## Running the http-server locally with Docker and Oxigraph
+
+The http-server can be started locally from a local rebuild of source code using:
+
+```bash
+pnpm compose:up
+```
+
+This will start the server listening on `localhost:8080` with a service container
+running a fresh Oxigraph graph store backend listening on `locahost:33378`. Note
+that the store will initialize empty, but store its data persistently at `tmp/oxigraph_data`
+so that you can upload data and continue to reuse it between launches of the app.
+
+**Seeding data to the graph store:** You can use the script `scripts/upload_to_local_deployment.ts`,
+which ingests any JSON-LD document from standard in and emits triples to the docker-compose
+Oxigraph backend (requires the composed cluster to be running). You can set the environment variable
+`DM_DEV_OXIGRAPH_URL` to something other than `http://localhost:33378` (the default) if you are trying
+to write to some other Oxigraph location.
+
+**WARNING** because of the [algebraic properties of blank nodes](https://docmaps.knowledgefutures.org/pub/eqb8u4v0/release/2),
+SPARQL Update protocol does not support idempotent writes. If you upload the same document multiple times,
+all blank nodes and associated triples will be presumed unique in relation to previous uploads
+of "similar seeming" blank nodes, and will result in duplication in the dataset. Named nodes will
+not be duplicated. For example, Steps and Actions are usually blank nodes, but Docmaps have ids.
+This means that if you upload a typical json-ld Docmap twice, all the steps will be doubled. If an
+action has an ID, then every copy of a step will point to it, whereas if not, there will be a copy
+of the action.
+
+To upload a docmap, pipe/redirect to the script's stdin. For example, [this docmap from github](https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld):
+
+```bash
+curl -s https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld \
+  | npx tsx scripts/upload_to_local_deployment.ts
+```
+
+There is example with a local file:
+
+```bash
+<my_docmap.jsonld npx tsx scripts/upload_to_local_deployment.ts
+```
+
+To confirm that there are docmaps in the store, you can visit `http://localhost:33378` (or whatever you customized
+the Oxigraph backend to serve at). You should see a SPARQL Query prompt and can explore. For example,
+you can submit this query to see how many nodes are of type `pwo:Workflow` (equivalent to `"type": "docmap"` in jsonld):
 
-See the instructions in repository root. You can invoke workspace root scripts like `pnpm run -w compose:up` wihout changing directory.
+```sparql
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX pwo: <http://purl.org/spar/pwo/>
+SELECT (COUNT(*) as ?docmaps) WHERE {
+  ?docmaps rdf:type pwo:Workflow .
+}
+```
+
+To remove all uploads if your state gets contaminated or you want to start over:
+
+```bash
+pnpm compose:repave
+# must restart oxigraph or entire compose cluster to ensure changes take effect
+```
 
 ### Misc
 

docker-compose.local.yml → packages/http-server/docker-compose.local.yml

@@ -8,7 +8,8 @@ services:
       - "./tmp/oxigraph_data:/data"
   server:
     build:
-      context: .
+      dockerfile: ./packages/http-server/Dockerfile
+      context: ./../../
     command: start
     environment:
       NODE_ENV: production

packages/http-server/package.json

@@ -10,6 +10,8 @@
     "test:integration": "ava test/integration/",
     "test:cleanup": "docker compose -f test/integration/assets/docker-compose.yml down",
     "test:unit": "ava test/unit/",
+    "compose:repave": "rm -rf tmp/oxigraph_data/*",
+    "compose:up": "docker compose -f $INIT_CWD/docker-compose.local.yml up --build",
     "lint": "npx eslint .",
     "lint:fix": "npx eslint --fix .",
     "start": "tsx dist/httpserver/main.js",
@@ -36,7 +38,7 @@
     "@zazuko/rdf-vocabularies": "^2023.1.19",
     "commander": "^11.0.0",
     "cors": "^2.8.5",
-    "docmaps-sdk": "^0.14.0",
+    "docmaps-sdk": "workspace:^0.0.0",
     "express": "^4.18.2",
     "fetch-sparql-endpoint": "^4.0.0",
     "fp-ts": "^2.14.0",

packages/ts-etl/package.json

@@ -12,8 +12,7 @@
     "start": "tsx dist/cli.js",
     "lint": "npx eslint .",
     "lint:fix": "npx eslint --fix .",
-    "prepare": "tsc --declaration",
-    "build": "tsc"
+    "build": "tsc --declaration"
   },
   "bin": {
     "docmaps-etl": "dist/cli.js"

packages/ts-sdk/package.json

@@ -15,8 +15,7 @@
     "docs:generate": "typedoc src/index.ts",
     "docs:check": "npm run docs:generate && git add -N . && git diff --exit-code -I 'Defined in.*blob'",
     "clean": "rm -rf dist/",
-    "build": "tsc",
-    "prepare": "tsc --declaration"
+    "build": "tsc --declaration"
   },
   "engines": {
     "node": ">=18.14.0"

packages/widget/package.json

@@ -16,6 +16,7 @@
   "types": "./dist/index.d.ts",
   "dependencies": {
     "d3": "^7.8.5",
+    "d3-force": "^3.0.0",
     "lit": "^2.7.6"
   },
   "devDependencies": {