From a32a2f1200fac90ce87614025797318e7334d4ad Mon Sep 17 00:00:00 2001 From: Andrew Edstrom Date: Wed, 1 Nov 2023 15:45:58 -0600 Subject: [PATCH] http-server: Bring over instructions about running with docker (#156) * put instructions in the right place * build everything for tests * reduce unnecessary matrix redundancy --------- Co-authored-by: andrew edstrom --- .dockerignore | 2 + .github/workflows/example-tests.yaml | 2 +- .github/workflows/http-server-tests.yaml | 9 ++- .github/workflows/spa-tests.yaml | 2 +- .github/workflows/ts-etl-tests.yaml | 2 +- .github/workflows/ts-sdk-tests.yaml | 2 +- .github/workflows/widget-tests.yaml | 2 +- .gitignore | 4 +- CONTRIBUTING.md | 65 ------------------- README.md | 8 ++- package.json | 2 - packages/http-client/package.json | 1 - Dockerfile => packages/http-server/Dockerfile | 10 ++- packages/http-server/README.md | 65 +++++++++++++++++-- .../http-server/docker-compose.local.yml | 3 +- packages/http-server/package.json | 4 +- packages/ts-etl/package.json | 3 +- packages/ts-sdk/package.json | 3 +- packages/widget/package.json | 1 + pnpm-lock.yaml | 41 ++---------- 20 files changed, 107 insertions(+), 124 deletions(-) create mode 100644 .dockerignore delete mode 100644 CONTRIBUTING.md rename Dockerfile => packages/http-server/Dockerfile (85%) rename docker-compose.local.yml => packages/http-server/docker-compose.local.yml (85%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..72008029 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/node_modules +**/tmp diff --git a/.github/workflows/example-tests.yaml b/.github/workflows/example-tests.yaml index d0336053..51483784 100644 --- a/.github/workflows/example-tests.yaml +++ b/.github/workflows/example-tests.yaml @@ -54,7 +54,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} ; - pnpm build; + pnpm run -r build; - name: Test run: | diff --git a/.github/workflows/http-server-tests.yaml b/.github/workflows/http-server-tests.yaml index 1c7f1958..9b8753f3 100644 --- a/.github/workflows/http-server-tests.yaml +++ b/.github/workflows/http-server-tests.yaml @@ -54,7 +54,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} ; - pnpm build; + pnpm run -r build; - name: Test run: | @@ -71,7 +71,7 @@ jobs: strategy: matrix: - node-version: [18.14.0, 18.18.0] + node-version: [18.18.0] # Service containers to run with `runner-job` services: @@ -118,6 +118,11 @@ jobs: cd ${{env.PKG_DIR}} ; pnpm install; + - name: Build cross-workspace deps + run: | + cd ${{env.PKG_DIR}} ; + pnpm run -r build; + - name: Test run: | cd ${{env.PKG_DIR}} ; diff --git a/.github/workflows/spa-tests.yaml b/.github/workflows/spa-tests.yaml index aca12580..a3c2879e 100644 --- a/.github/workflows/spa-tests.yaml +++ b/.github/workflows/spa-tests.yaml @@ -54,7 +54,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} ; - pnpm build; + pnpm run -r build; - name: Test run: | diff --git a/.github/workflows/ts-etl-tests.yaml b/.github/workflows/ts-etl-tests.yaml index bbec51c9..bb71bf14 100644 --- a/.github/workflows/ts-etl-tests.yaml +++ b/.github/workflows/ts-etl-tests.yaml @@ -54,7 +54,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} ; - pnpm build; + pnpm run -r build; - name: Test run: | diff --git a/.github/workflows/ts-sdk-tests.yaml b/.github/workflows/ts-sdk-tests.yaml index c7ccf054..adcbaf39 100644 --- a/.github/workflows/ts-sdk-tests.yaml +++ b/.github/workflows/ts-sdk-tests.yaml @@ -54,7 +54,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} ; - pnpm build; + pnpm run -r build; - name: Test run: | diff --git a/.github/workflows/widget-tests.yaml b/.github/workflows/widget-tests.yaml index 34d34ab0..ba6f0e49 100644 --- a/.github/workflows/widget-tests.yaml +++ b/.github/workflows/widget-tests.yaml @@ -56,7 +56,7 @@ jobs: - name: Verify builds run: | cd ${{env.PKG_DIR}} - pnpm build; + pnpm run -r build; - name: Install Playwright Browsers run: | diff --git a/.gitignore b/.gitignore index bffd1c60..08da8c09 100644 --- a/.gitignore +++ b/.gitignore @@ -134,4 +134,6 @@ dist **/.DS_Store # IntelliJ files -.idea/**/* \ No newline at end of file +.idea/**/* + +**/tmp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index f0d7a895..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,65 +0,0 @@ -# Contribution guide - -This is a living document with helpful context for development workflows related -to this project. It is not comprehensive, we invite you to request clarifications -about any topic that is underspecified here! - -### Running the http-server locally - -The server can be started locally from a local rebuild of source code using: - -```bash -pnpm compose:up -``` - -This will start the server listening on `localhost:8080` with a service container -running a fresh Oxigraph graph store backend listening on `locahost:33378`. Note -that the store will initialize empty, but store its data persistently at `tmp/oxigraph_data` -so that you can upload data and continue to reuse it between launches of the app. - -**Seeding data to the graph store:** You can use the script `scripts/upload_to_local_deployment.ts`, -which ingests any JSON-LD document from standard in and emits triples to the docker-compose -Oxigraph backend (requires the composed cluster to be running). You can set the environment variable -`DM_DEV_OXIGRAPH_URL` to something other than `http://localhost:33378` (the default) if you are trying -to write to some other Oxigraph location. - -**WARNING** because of the [algebraic properties of blank nodes](https://docmaps.knowledgefutures.org/pub/eqb8u4v0/release/2), -SPARQL Update protocol does not support idempotent writes. If you upload the same document multiple times, -all blank nodes and associated triples will be presumed unique in relation to previous uploads -of "similar seeming" blank nodes, and will result in duplication in the dataset. Named nodes will -not be duplicated. For example, Steps and Actions are usually blank nodes, but Docmaps have ids. -This means that if you upload a typical json-ld Docmap twice, all the steps will be doubled. If an -action has an ID, then every copy of a step will point to it, whereas if not, there will be a copy -of the action. - -To upload a docmap, pipe/redirect to the script's stdin. For example, [this docmap from github](https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld): - -```bash -curl -s https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld \ - | npx tsx scripts/upload_to_local_deployment.ts -``` - -There is example with a local file: - -```bash - -PREFIX pwo: -SELECT (COUNT(*) as ?docmaps) WHERE { - ?docmaps rdf:type pwo:Workflow . -} -``` - -To remove all uploads if your state gets contaminated or you want to start over: - -```bash -pnpm compose:repave -# must restart oxigraph or entire compose cluster to ensure changes take effect -``` diff --git a/README.md b/README.md index 8a969480..74850117 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ cross-package behavior, such as release automation. **Releases and tags:** Github Actions uses [multi-semantic-release](https://github.com/dhoulb/multi-semantic-release) to automatically generate semvers based on commit history for each package in the repository. Multiple tags are generated for a single commit if it updates multiple packages. +**Cross-dependencies and typescript:** Several packages depend on `ts-sdk`, and will refer to +the `dist` directory within that package for their source code when making local changes. +For that reason, you may need to run `pnpm --filter docmaps-sdk run build` or `pnpm run -r build` +if you are making local changes to uptream dependencies within this monorepo so that your +downstream code changes will pull those in. + + **Dependencies:** The workspace root builds a Docker image for the http-server. In addition to the npm package dependencies, to do local development you should have the following tools installed: @@ -64,7 +71,6 @@ This Single-page App (SPA) is a simple demonstration of the above tools in actio accessible [live on Github Pages](https://docmaps-project.github.io/docmaps/demo/) where you can plug in a DOI and get a best-effort view of a Docmap as inferred from Crossref's API. - ## Governance As stated in [CODE_OF_CONDUCT.md](/CODE_OF_CONDUCT.md): diff --git a/package.json b/package.json index 51d98047..b85eeece 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,6 @@ "description": "Top-level Docmaps specification module -- not intended for direct npm consumption.", "scripts": { "clean": "manypkg exec $npm_execpath run clean && rm -rf node_modules", - "compose:repave": "rm -rf tmp/oxigraph_data/*", - "compose:up": "docker compose -f $INIT_CWD/docker-compose.local.yml up --build", "test": "node scripts/package-tests.js", "test:packages": "manypkg exec $npm_execpath test", "test:all": "$npm_execpath run test && $npm_execpath run test:packages" diff --git a/packages/http-client/package.json b/packages/http-client/package.json index 16eba84b..885fce2c 100644 --- a/packages/http-client/package.json +++ b/packages/http-client/package.json @@ -14,7 +14,6 @@ "lint": "npx eslint .", "lint:fix": "npx eslint --fix .", "start": "tsx dist/httpserver/main.js", - "prepare": "tsc --declaration", "build": "tsc --declaration" }, "keywords": [], diff --git a/Dockerfile b/packages/http-server/Dockerfile similarity index 85% rename from Dockerfile rename to packages/http-server/Dockerfile index c8a1c337..ee00e6ac 100644 --- a/Dockerfile +++ b/packages/http-server/Dockerfile @@ -4,6 +4,9 @@ # If you need more help, visit the Dockerfile reference guide at # https://docs.docker.com/engine/reference/builder/ +# NOTE: due to use of pnpm workspace, this dockerfile needs to be built +# from the repository root context. + ARG NODE_VERSION=18.14.0 ARG PNPM_VERSION=8.7.6 @@ -26,9 +29,10 @@ FROM base AS build # "devDependencies" to be installed to build. If you don't need this, remove this step. RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ pnpm install \ + --filter "@docmaps/http-server" \ --frozen-lockfile -RUN pnpm run -r build +RUN pnpm --filter "@docmaps/http-server" run build ################################################################################ # Download dependencies as a separate step to take advantage of Docker's caching. @@ -41,12 +45,16 @@ RUN npm i -g typescript@4.9.5 RUN --mount=type=cache,id=pnpm,target=/pnpm/store \ pnpm install \ + --filter "@docmaps/http-server" \ --prod \ --frozen-lockfile ################################################################################ # Create a new stage to run the application with minimal runtime dependencies # where the necessary files are copied from the build stage. + +# FIXME: build from Prod instead, once we have resolved the prepare script issue: +# https://github.com/Docmaps-Project/docmaps/issues/118 FROM prod AS runtime # Copy the production dependencies from the deps stage and also diff --git a/packages/http-server/README.md b/packages/http-server/README.md index f49a34df..6169b249 100644 --- a/packages/http-server/README.md +++ b/packages/http-server/README.md @@ -18,13 +18,12 @@ domain logic into queries for your specific backend (such as a relational databa ## Development and Testing -See the readme in repository root for general info about this monorepo. - Dependencies for local development include: ``` pnpm -docker # with docker-compose +node # @^18 +docker # with docker-compose available curl ``` @@ -39,9 +38,65 @@ pnpm test:integration **WARN:** If you have never done this before, the tests may timeout due to invoking a `docker pull` for the `oxigraph` image, which is used for a local triplestore to use during integration tests. -## Running locally with Docker and Oxigraph +## Running the http-server locally with Docker and Oxigraph + +The http-server can be started locally from a local rebuild of source code using: + +```bash +pnpm compose:up +``` + +This will start the server listening on `localhost:8080` with a service container +running a fresh Oxigraph graph store backend listening on `locahost:33378`. Note +that the store will initialize empty, but store its data persistently at `tmp/oxigraph_data` +so that you can upload data and continue to reuse it between launches of the app. + +**Seeding data to the graph store:** You can use the script `scripts/upload_to_local_deployment.ts`, +which ingests any JSON-LD document from standard in and emits triples to the docker-compose +Oxigraph backend (requires the composed cluster to be running). You can set the environment variable +`DM_DEV_OXIGRAPH_URL` to something other than `http://localhost:33378` (the default) if you are trying +to write to some other Oxigraph location. + +**WARNING** because of the [algebraic properties of blank nodes](https://docmaps.knowledgefutures.org/pub/eqb8u4v0/release/2), +SPARQL Update protocol does not support idempotent writes. If you upload the same document multiple times, +all blank nodes and associated triples will be presumed unique in relation to previous uploads +of "similar seeming" blank nodes, and will result in duplication in the dataset. Named nodes will +not be duplicated. For example, Steps and Actions are usually blank nodes, but Docmaps have ids. +This means that if you upload a typical json-ld Docmap twice, all the steps will be doubled. If an +action has an ID, then every copy of a step will point to it, whereas if not, there will be a copy +of the action. + +To upload a docmap, pipe/redirect to the script's stdin. For example, [this docmap from github](https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld): + +```bash +curl -s https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld \ + | npx tsx scripts/upload_to_local_deployment.ts +``` + +There is example with a local file: + +```bash + +PREFIX pwo: +SELECT (COUNT(*) as ?docmaps) WHERE { + ?docmaps rdf:type pwo:Workflow . +} +``` + +To remove all uploads if your state gets contaminated or you want to start over: + +```bash +pnpm compose:repave +# must restart oxigraph or entire compose cluster to ensure changes take effect +``` ### Misc diff --git a/docker-compose.local.yml b/packages/http-server/docker-compose.local.yml similarity index 85% rename from docker-compose.local.yml rename to packages/http-server/docker-compose.local.yml index ab09d18b..8a592b61 100644 --- a/docker-compose.local.yml +++ b/packages/http-server/docker-compose.local.yml @@ -8,7 +8,8 @@ services: - "./tmp/oxigraph_data:/data" server: build: - context: . + dockerfile: ./packages/http-server/Dockerfile + context: ./../../ command: start environment: NODE_ENV: production diff --git a/packages/http-server/package.json b/packages/http-server/package.json index 7a314079..819490e7 100644 --- a/packages/http-server/package.json +++ b/packages/http-server/package.json @@ -10,6 +10,8 @@ "test:integration": "ava test/integration/", "test:cleanup": "docker compose -f test/integration/assets/docker-compose.yml down", "test:unit": "ava test/unit/", + "compose:repave": "rm -rf tmp/oxigraph_data/*", + "compose:up": "docker compose -f $INIT_CWD/docker-compose.local.yml up --build", "lint": "npx eslint .", "lint:fix": "npx eslint --fix .", "start": "tsx dist/httpserver/main.js", @@ -36,7 +38,7 @@ "@zazuko/rdf-vocabularies": "^2023.1.19", "commander": "^11.0.0", "cors": "^2.8.5", - "docmaps-sdk": "^0.14.0", + "docmaps-sdk": "workspace:^0.0.0", "express": "^4.18.2", "fetch-sparql-endpoint": "^4.0.0", "fp-ts": "^2.14.0", diff --git a/packages/ts-etl/package.json b/packages/ts-etl/package.json index b41831dc..45ae8e0f 100644 --- a/packages/ts-etl/package.json +++ b/packages/ts-etl/package.json @@ -12,8 +12,7 @@ "start": "tsx dist/cli.js", "lint": "npx eslint .", "lint:fix": "npx eslint --fix .", - "prepare": "tsc --declaration", - "build": "tsc" + "build": "tsc --declaration" }, "bin": { "docmaps-etl": "dist/cli.js" diff --git a/packages/ts-sdk/package.json b/packages/ts-sdk/package.json index 83d623a6..a1fa5bae 100644 --- a/packages/ts-sdk/package.json +++ b/packages/ts-sdk/package.json @@ -15,8 +15,7 @@ "docs:generate": "typedoc src/index.ts", "docs:check": "npm run docs:generate && git add -N . && git diff --exit-code -I 'Defined in.*blob'", "clean": "rm -rf dist/", - "build": "tsc", - "prepare": "tsc --declaration" + "build": "tsc --declaration" }, "engines": { "node": ">=18.14.0" diff --git a/packages/widget/package.json b/packages/widget/package.json index 9c284379..f531f6b5 100644 --- a/packages/widget/package.json +++ b/packages/widget/package.json @@ -16,6 +16,7 @@ "types": "./dist/index.d.ts", "dependencies": { "d3": "^7.8.5", + "d3-force": "^3.0.0", "lit": "^2.7.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd4c37c4..4ed1e4f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -153,8 +153,8 @@ importers: specifier: ^2.8.5 version: 2.8.5 docmaps-sdk: - specifier: ^0.14.0 - version: 0.14.0 + specifier: workspace:^0.0.0 + version: link:../ts-sdk express: specifier: ^4.18.2 version: 4.18.2 @@ -456,6 +456,9 @@ importers: d3: specifier: ^7.8.5 version: 7.8.5 + d3-force: + specifier: ^3.0.0 + version: 3.0.0 lit: specifier: ^2.7.6 version: 2.8.0 @@ -4291,24 +4294,6 @@ packages: dependencies: path-type: 4.0.0 - /docmaps-sdk@0.14.0: - resolution: {integrity: sha512-3F23bp+SEuxmO6PdwyO/Wqk+ABTvGnZ9Bqc2pOxx91z9n/hRmxQVb3PZkO3fvpZF6End1Ky+JL8sAKvkWE9T/w==} - engines: {node: '>=18.14.0'} - dependencies: - '@rdfjs/data-model': 2.0.1 - '@rdfjs/parser-n3': 2.0.1 - '@rdfjs/serializer-jsonld-ext': 4.0.0 - fp-ts: 2.16.1 - io-ts: 2.2.20(fp-ts@2.16.1) - io-ts-types: 0.5.19(fp-ts@2.16.1)(io-ts@2.2.20)(monocle-ts@2.3.13)(newtype-ts@0.3.5) - monocle-ts: 2.3.13(fp-ts@2.16.1) - newtype-ts: 0.3.5(fp-ts@2.16.1)(monocle-ts@2.3.13) - rdf-ext: 2.2.0 - readable-stream: 4.3.0 - transitivePeerDependencies: - - web-streams-polyfill - dev: false - /doctrine@3.0.0: resolution: {integrity: sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==} engines: {node: '>=6.0.0'} @@ -5951,26 +5936,12 @@ packages: lit-html: 2.8.0 dev: false - /lit-html@2.7.4: - resolution: {integrity: sha512-/Jw+FBpeEN+z8X6PJva5n7+0MzCVAH2yypN99qHYYkq8bI+j7I39GH+68Z/MZD6rGKDK9RpzBw7CocfmHfq6+g==} - dependencies: - '@types/trusted-types': 2.0.3 - dev: false - /lit-html@2.8.0: resolution: {integrity: sha512-o9t+MQM3P4y7M7yNzqAyjp7z+mQGa4NS4CxiyLqFPyFWyc4O+nodLrkrxSaCTrla6M5YOLaT3RpbbqjszB5g3Q==} dependencies: '@types/trusted-types': 2.0.3 dev: false - /lit@2.7.5: - resolution: {integrity: sha512-i/cH7Ye6nBDUASMnfwcictBnsTN91+aBjXoTHF2xARghXScKxpD4F4WYI+VLXg9lqbMinDfvoI7VnZXjyHgdfQ==} - dependencies: - '@lit/reactive-element': 1.6.2 - lit-element: 3.3.2 - lit-html: 2.7.4 - dev: false - /lit@2.8.0: resolution: {integrity: sha512-4Sc3OFX9QHOJaHbmTMk28SYgVxLN3ePDjg7hofEft2zWlehFL3LiAuapWc4U/kYwMYJSh2hTCPZ6/LIC7ii0MA==} dependencies: @@ -8908,7 +8879,7 @@ packages: dependencies: '@spider-ui/tooltip': 0.2.7(@spider-ui/global-event-registry@0.2.7)(upgraded-element@0.6.5) dompurify: 2.4.5 - lit: 2.7.5 + lit: 2.8.0 transitivePeerDependencies: - '@spider-ui/global-event-registry' - upgraded-element