Dev

.github/workflows/nist-pages.yml

@@ -0,0 +1,21 @@
+name: NIST Pages
+on:
+  push:
+    branches:
+      - master
+jobs:
+  deploy-pages:
+    container: ghcr.io/usnistgov/continuous-scientific-python:latest
+    env:
+      # POETRY_VIRTUALENVS_PATH: "${CI_PROJECT_DIR}/.cache/venv"
+      MKDOCS_SITE_DIR: "public"
+      MKDOCS_SITE_URL: "https://pages.nist.gov/nestor/"
+      MKDOCS_REPO_URL: "https://github.com/usnistgov/nestor"
+
+
+    steps:
+      - name: "install dev dependencies"
+        run: poetry install
+      - name: "subtree script for nist-pages branch"
+        run: |
+          ./nist-pages-deploy.sh

.gitignore

@@ -1,29 +1,165 @@
-#   special folder cache or others
-**/__pycache__/
-.idea/
-**/.ipynb_checkpoints
-**.egg-info
-
-
-build
-dist
-public
-_build
-_static
-_templates
-_pdf
-.doctrees
-
-#   special file type
-**.pyc
-**.csv
-**/reveal.js
-**.h5
-
-# exception
-!nestor/settings.yaml
-!nestor/datasets/**
-
-/poetry.lock
-**/.mypy_cache/
-**/.auctec-auto
+# use jupytext
+*.ipynb
+
+
+### GITIGNORE.io TEMPLATE ###
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks
+
+### JupyterNotebooks ###
+# gitignore template for Jupyter Notebooks
+# website: http://jupyter.org/
+
+.ipynb_checkpoints
+*/.ipynb_checkpoints/*
+
+# IPython
+profile_default/
+ipython_config.py
+
+# Remove previous ipynb_checkpoints
+#   git rm -r .ipynb_checkpoints/
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+
+# IPython
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks
+/public/
+/.idea/
+/docs/.auctex-auto/
+*.iob

.gitlab-ci.yml

@@ -1,31 +1,56 @@
-image: python:3.7
+default:
+  image: $CI_REGISTRY/kea/templates/docker-conda:master
+  before_script:
+    - poetry install
+  cache:
+    key: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - .cache/venv
 
 stages:
-  # - test
-  # - lint
+  - test
   - deploy
 
-before_script:
-  - conda env update --file gldeployenv.yaml
-  - conda activate nestor-dev
-  - poetry install
-
-# Unit Tests:
-#   stage: test
-#   script:
-#     - poetry run pytest
-
-# Python Code Lint:
-#   stage: lint
-#   script:
-#     - poetry run task format 
+variables:
+  POETRY_VIRTUALENVS_PATH: "${CI_PROJECT_DIR}/.cache/venv"
+  MKDOCS_SITE_DIR: "public"
+
+pytest:
+  stage: test
+  script:
+    - poetry run pytest --junitxml=report.xml
+  artifacts:
+    when: always
+    reports:
+      junit: report.xml
 
 pages:
   stage: deploy
+  variables:
+    MKDOCS_SITE_URL: "https://kea.ipages.nist.gov/nestor-suite/nestor/"
+    MKDOCS_REPO_URL: "https://gitlab.nist.gov/gitlab/kea/nestor-suite/nestor/"
+
   script:
-    - poetry run task deploy-docs
+    # - conda env update --file docs/examples/nb-env.yml
+    # - jupytext --to notebook --execute docs/examples/**/*.py
+    - poetry run task deploy
   artifacts:
     paths:
       - public
   only:
-    - dev
+    - docs
+    - master
+
+# nist-pages:
+#   stage: deploy
+#   variables:
+#     MKDOCS_SITE_URL: "https://pages.nist.gov/nestor/"
+#     MKDOCS_REPO_URL: "https://github.com/usnistgov/nestor"
+
+#   script:
+#     - conda env update --file docs/examples/nb-env.yml
+#     - jupytext --to notebook --execute docs/examples/**/*.py
+#     - poetry run task deploy-docs
+#     - git push nist-origin `git subtree split --prefix public master`:nist-pages --force
+#   only:
+#     - master

CODEMETA.yaml

@@ -0,0 +1,17 @@
+# NIST Opensource Portal repositories categories and themes.
+# Source: https://www.nist.gov/topics
+# The following NIST topics and categories are based on the NIST
+# Taxonomy. It can be found here:
+# https://data.nist.gov/od/id/691DDF3315711C14E0532457068146BE1907
+
+categories:
+  - scientific-software:
+      - python
+      - annotation
+  - ai-ml:
+      - NLP
+themes:
+  - Artificial intelligence
+  - Information Technology
+  - Manufacturing
+  - Mathematics & Statistics

docs/README.md

@@ -9,15 +9,26 @@
 
 ## Purpose
 
+**Nestor** is a toolkit for using Natural Language Processing (NLP) with efficient user-interaction to perform structured data extraction with minimal annotation time-cost.
+
+### The Problem
 NLP in technical domains requires context sensitivity.
 Whether for medical notes, engineering work-orders, or social/behavioral coding, experts often use specialized vocabulary with over-loaded meanings and jargon.
 This is incredibly difficult for off-the-shelf NLP systems to parse through. 
 
-The common solution is to contextualize NLP models.
-For instance, medical NLP has been greatly advanced with the advent of labeled, bio-specific datasets, which have domain-relevant named-entity tags and vocabulary sets. 
+The common solution is to contextualize and adapt NLP models to technical text -- Technical Language Processing (TLP)[@brundage2020technical].
+For instance, medical research has been greatly advanced with the advent of labeled, bio-specific datasets, which have domain-relevant named-entity tags and vocabulary sets. 
 Unfortunately for analysts of these types of data, creating resources like this is incredibly time consuming. 
 This is where `nestor` comes in. 
 
+### Why Maintenance and Manufacturing?
+
+A reader may notice a heavy focus on maintenance and manufacturing in the Nestor documentation and design. 
+While this is a common problem in technical domains, generally, Nestor got its start in manufacturing data analysis. 
+A large amount of maintenance data is *already* available for use in advanced manufacturing systems, but in a currently-unusable form: service tickets and maintenance work orders (MWOs).
+
+For further reading, see [@sexton2017hybrid] [@sharp2017toward] [@brundage2020technical]. 
+
 ## Quick Links
 
 - [Get started](getting-started.md)
@@ -34,13 +45,7 @@ This application was originally designed to help manufacturers "tag" their maint
 The goal is to help  build context-rich labels in data sets that previously were too unstructured or filled with jargon to analyze.
 The current build is in very early alpha, so please be patient in using this application. If you have any questions, please do not hesitate to contact us (see [Who are we?](#who-are-we). ) 
 
-### Why?
 
-There is often a large amount of maintenance data *already* available for use in Smart Manufacturing systems, but in a currently-unusable form: service tickets and maintenance work orders (MWOs).
-**Nestor** is a toolkit for using Natural Language Processing (NLP) with efficient user-interaction to perform structured data extraction with minimal annotation time-cost. 
-For further reading, see [@sexton2017hybrid] [@sharp2017toward]. 
-
-### Features
 
 -   Rank keywords found in your data by importance, saving you time
 -   Suggest term unification by similarity (e.g. spelling), for quick review
@@ -51,26 +56,26 @@ For further reading, see [@sexton2017hybrid] [@sharp2017toward].
 
 Planned: 
 
-- Customizable entity types and rules
-- export to NER training formats
-- command-line app and REST API
+- Customizable entity types and rules,
+- Export to NER training formats,
+- Command-line app and REST API.
 
 
 ## Who are we?
 
-This toolkit is a part of the Knowledge Extraction and Application for Smart Manufacturing (KEA) project, within the Systems Integration Division at NIST. 
+This toolkit is a part of the [Knowledge Extraction and Application for Smart Manufacturing (KEA)](https://www.nist.gov/programs-projects/knowledge-extraction-and-application-manufacturing-operations) project, within the [Systems Integration Division](https://www.nist.gov/el/systems-integration-division-73400) at NIST. 
 
 
 ### Projects that use Nestor
 
-- Various [Nestor GUIs](gui-links.md)
-- [nestor exploratory data analysis](https://github.com/usnistgov/nestor-eda) (dashboard, viz, etc.)
+- Various [Nestor GUIs](gui-links.md): ways to use the full human-centered Nestor workflow in a user-interface. 
+- [`nestor-eda`](https://github.com/usnistgov/nestor-eda): (exploratory data analysis): things to do with Nestor-annotated data (dashboard, viz, etc.)
 
 
 ### Points of Contact
-- Email the dev team at <[email protected]>
--   [Thurston Sexton](https://www.nist.gov/people/thurston-sexton) [@tbsexton](https://github.com/tbsexton) Nestor Technical Lead 
--   [Michael Brundage](https://www.nist.gov/people/michael-p-brundage) Principal Investigator 
+- Email the development team at <[email protected]>
+-   [Thurston Sexton](https://www.nist.gov/people/thurston-sexton) [@tbsexton](https://github.com/tbsexton) Nestor Technical Lead, Associate Project Leader 
+-   [Michael Brundage](https://www.nist.gov/people/michael-p-brundage) Project Leader
 
 
 ### Why "KEA"?
@@ -85,4 +90,7 @@ Plugins are installed as development dependencies through poetry (e.g. `taskipy`
 
 Notebooks should be kept nicely git-friendly with [Jupytext](https://github.com/mwouts/jupytext)
 
+## Other Tools/Resources
+Know of other tools? Or want to find similar resources as Nestor? A community driven [TLP Community of Interest (COI)](https://www.nist.gov/el/technical-language-processing-community-interest) has been created to provide publicly available resources to the community. Check out our [awesomelist](https://github.com/TLP-COI/awesome-tlp).
+
 \bibliography