feat(http-server): Migrate ts-api-server/runtime to docmaps/http-serv…

…er (#140)

* adds http-server package (from runtime)
* rename runtime to http-server

.github/workflows/http-server-tests.yaml

@@ -0,0 +1,124 @@
+name: Test http-server
+
+on:
+  push:
+    # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+  workflow_call:
+
+env:
+  PKG_DIR: "packages/http-server"
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.18.0]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('${{env.PKG_DIR}}/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm install;
+
+      - name: Verify builds
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm build;
+
+      - name: Test
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm test:unit;
+
+      - name: Lint Check
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm lint;
+
+  integration_tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [18.14.0, 18.18.0]
+
+    # Service containers to run with `runner-job`
+    services:
+      # Label used to access the service container
+      sparql_backend:
+        # Docker Hub image
+        image: ghcr.io/oxigraph/oxigraph
+        #
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 33078:7878
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('${{env.PKG_DIR}}/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm install;
+
+      - name: Test
+        run: |
+          cd ${{env.PKG_DIR}} ;
+          pnpm test:integration --timeout=30s ;

.github/workflows/release.yaml

@@ -19,6 +19,8 @@ env:
 jobs:
   test-ts-sdk:
     uses: ./.github/workflows/ts-sdk-tests.yaml
+  test-ts-http-server:
+    uses: ./.github/workflows/http-server-tests.yaml
   test-ts-etl:
     uses: ./.github/workflows/ts-etl-tests.yaml
   test-example:
@@ -76,7 +78,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
         # must ignore spa because we need spa in the workspace file for development, but spa is not released.
-        run: npx multi-semantic-release --ignore-packages=packages/spa,packages/example,packages/widget
+        run: npx multi-semantic-release --ignore-packages=packages/spa,packages/example,packages/widget,packages/http-server
 
   release-github-pages-docs:
     uses: ./.github/workflows/gh-pages.yaml

CONTRIBUTING.md

@@ -0,0 +1,65 @@
+# Contribution guide
+
+This is a living document with helpful context for development workflows related
+to this project. It is not comprehensive, we invite you to request clarifications
+about any topic that is underspecified here!
+
+### Running the http-server locally
+
+The server can be started locally from a local rebuild of source code using:
+
+```bash
+pnpm compose:up
+```
+
+This will start the server listening on `localhost:8080` with a service container
+running a fresh Oxigraph graph store backend listening on `locahost:33378`. Note
+that the store will initialize empty, but store its data persistently at `tmp/oxigraph_data`
+so that you can upload data and continue to reuse it between launches of the app.
+
+**Seeding data to the graph store:** You can use the script `scripts/upload_to_local_deployment.ts`,
+which ingests any JSON-LD document from standard in and emits triples to the docker-compose
+Oxigraph backend (requires the composed cluster to be running). You can set the environment variable
+`DM_DEV_OXIGRAPH_URL` to something other than `http://localhost:33378` (the default) if you are trying
+to write to some other Oxigraph location.
+
+**WARNING** because of the [algebraic properties of blank nodes](https://docmaps.knowledgefutures.org/pub/eqb8u4v0/release/2),
+SPARQL Update protocol does not support idempotent writes. If you upload the same document multiple times,
+all blank nodes and associated triples will be presumed unique in relation to previous uploads
+of "similar seeming" blank nodes, and will result in duplication in the dataset. Named nodes will
+not be duplicated. For example, Steps and Actions are usually blank nodes, but Docmaps have ids.
+This means that if you upload a typical json-ld Docmap twice, all the steps will be doubled. If an
+action has an ID, then every copy of a step will point to it, whereas if not, there will be a copy
+of the action.
+
+To upload a docmap, pipe/redirect to the script's stdin. For example, [this docmap from github](https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld):
+
+```bash
+curl -s https://raw.githubusercontent.com/Docmaps-Project/docmaps/main/examples/docmaps-example-elife-02.jsonld \
+  | npx tsx scripts/upload_to_local_deployment.ts
+```
+
+There is example with a local file:
+
+```bash
+<my_docmap.jsonld npx tsx scripts/upload_to_local_deployment.ts
+```
+
+To confirm that there are docmaps in the store, you can visit `http://localhost:33378` (or whatever you customized
+the Oxigraph backend to serve at). You should see a SPARQL Query prompt and can explore. For example,
+you can submit this query to see how many nodes are of type `pwo:Workflow` (equivalent to `"type": "docmap"` in jsonld):
+
+```sparql
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX pwo: <http://purl.org/spar/pwo/>
+SELECT (COUNT(*) as ?docmaps) WHERE {
+  ?docmaps rdf:type pwo:Workflow .
+}
+```
+
+To remove all uploads if your state gets contaminated or you want to start over:
+
+```bash
+pnpm compose:repave
+# must restart oxigraph or entire compose cluster to ensure changes take effect
+```

Dockerfile

@@ -0,0 +1,67 @@
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/engine/reference/builder/
+
+ARG NODE_VERSION=18.14.0
+ARG PNPM_VERSION=8.7.6
+
+################################################################################
+# Use node image for base image for all stages.
+FROM node:${NODE_VERSION}-alpine as base
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+# Set working directory for all build stages.
+COPY . /app
+WORKDIR /app
+
+################################################################################
+# Create a stage for building the application.
+FROM base AS build
+# Download additional development dependencies before building, as some projects require
+# "devDependencies" to be installed to build. If you don't need this, remove this step.
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
+      pnpm install \
+      --frozen-lockfile
+
+RUN pnpm run -r build
+
+################################################################################
+# Download dependencies as a separate step to take advantage of Docker's caching.
+# Leverage a cache mount to /root/.local/share/pnpm/store to speed up subsequent builds.
+# Leverage bind mounts to package.json and pnpm-lock.yaml to avoid having to copy them
+# into this layer.
+FROM base AS prod
+
+RUN npm i -g [email protected]
+
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
+      pnpm install \
+      --prod \
+      --frozen-lockfile
+
+################################################################################
+# Create a new stage to run the application with minimal runtime dependencies
+# where the necessary files are copied from the build stage.
+FROM prod AS runtime
+
+# Copy the production dependencies from the deps stage and also
+# the built application from the build stage into the image.
+COPY --from=build /app/packages/http-server/dist /app/packages/http-server/dist
+WORKDIR /app/packages/http-server
+
+# Use production node environment by default.
+ENV NODE_ENV production
+
+# Run the application as a non-root user.
+USER node
+
+# Expose the port that the application listens on.
+EXPOSE 8000
+
+# Run the application.
+ENTRYPOINT [ "pnpm", "start" ]

README.md

@@ -18,6 +18,17 @@ cross-package behavior, such as release automation.
 **Releases and tags:** Github Actions uses [multi-semantic-release](https://github.com/dhoulb/multi-semantic-release) to automatically generate semvers based on commit history for each package in the repository. Multiple tags
 are generated for a single commit if it updates multiple packages.
 
+**Dependencies:** The workspace root builds a Docker image for the http-server. In addition to the
+npm package dependencies, to do local development you should have the following tools installed:
+
+```bash
+pnpm # @^8.7
+node # @^18
+docker # with docker-compose available
+```
+
+see [CONTRIBUTING.md](/CONTRIBUTING.md) for more info about local development.
+
 ## Persistent URLs
 
 Documentation: https://w3id.org/docmaps
@@ -35,6 +46,11 @@ that library natively integrates with `fp-ts` and enables easy encoding & decodi
 from raw data types at runtime by creating Prototypical classes in runtime namespace
 along with the types/interfaces in type namespace.
 
+### [http-server](/packages/http-server)
+
+This package is a Node server that serves docmaps. It is the Typescript 
+reference implementation of Docmaps Project RFC#001 API Server Interoperability Protocol.
+
 ### [ts-etl](/packages/ts-etl)
 
 This package contains a CLI tool based on `commander.js` for generating docmaps. Currently,
@@ -48,6 +64,7 @@ This Single-page App (SPA) is a simple demonstration of the above tools in actio
 accessible [live on Github Pages](https://docmaps-project.github.io/docmaps/demo/) where you can
 plug in a DOI and get a best-effort view of a Docmap as inferred from Crossref's API.
 
+
 ## Governance
 
 As stated in [CODE_OF_CONDUCT.md](/CODE_OF_CONDUCT.md):

docker-compose.local.yml

@@ -0,0 +1,20 @@
+name: docmaps-api-server-local
+services:
+  oxigraph:
+    image: ghcr.io/oxigraph/oxigraph
+    ports:
+      - "33378:7878"
+    volumes:
+      - "./tmp/oxigraph_data:/data"
+  server:
+    build:
+      context: .
+    command: start
+    environment:
+      NODE_ENV: production
+      DM_SERVER_API_URL: http://localhost:8080
+      DM_SERVER_PORT: 8080
+      DM_BACKEND_TYPE: sparql-endpoint
+      DM_BACKEND_SPARQL_ENDPOINT_URL: http://oxigraph:7878/query
+    ports:
+      - 8080:8080

package.json

@@ -5,6 +5,8 @@
   "description": "Top-level Docmaps specification module -- not intended for direct npm consumption.",
   "scripts": {
     "clean": "manypkg exec $npm_execpath run clean && rm -rf node_modules",
+    "compose:repave": "rm -rf tmp/oxigraph_data/*",
+    "compose:up": "docker compose -f $INIT_CWD/docker-compose.local.yml up --build",
     "test": "node scripts/package-tests.js",
     "test:packages": "manypkg exec $npm_execpath test",
     "test:all": "$npm_execpath run test && $npm_execpath run test:packages"
@@ -19,10 +21,14 @@
     "@rdfjs/serializer-ntriples": "^2.0.0",
     "@rdfjs/serializer-turtle": "^1.1.1",
     "multi-semantic-release": "^3.0.2",
-    "rdf-ext": "^2.2.0",
-    "rdf-validate-shacl": "^0.5.0"
+    "rdf-ext": "^2.2.0"
   },
   "devDependencies": {
+    "axios": "^1.6.0",
+    "jsonld-streaming-parser": "^3.2.1",
+    "multi-semantic-release": "^3.0.2",
+    "rdf-validate-shacl": "^0.4.5",
+    "tsx": "^3.12.7",
     "typescript": "^4.9.5"
   }
 }

packages/http-server/.eslintrc.cjs

@@ -0,0 +1,23 @@
+module.exports = {
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:@typescript-eslint/eslint-recommended',
+    'plugin:prettier/recommended',
+  ],
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint', 'prettier'],
+  root: true,
+  ignorePatterns: ['dist/'],
+  rules: {
+    '@typescript-eslint/no-inferrable-types': 0,
+    '@typescript-eslint/no-unused-vars': [
+      'error',
+      {
+        varsIgnorePattern: '^_',
+        argsIgnorePattern: '^_',
+        caughtErrorsIgnorePattern: '^_',
+      },
+    ],
+  },
+}

packages/http-server/.prettierrc.cjs

@@ -0,0 +1,8 @@
+module.exports = {
+  semi: false,
+  trailingComma: 'all',
+  singleQuote: true,
+  quoteProps: 'as-needed',
+  printWidth: 100,
+  tabWidth: 2,
+}