Resolve merge conflicts

.coveragerc

@@ -2,7 +2,7 @@
 # https://coverage.readthedocs.io/en/latest/config.html
 
 [run]
-source = src/example
+source = src/pe_reports
 omit =
 branch = true
 

.flake8

@@ -22,4 +22,9 @@ select = C,D,E,F,W,B,B950
 # operators.  It no longer agrees with PEP8.  See, for example, here:
 # https://github.com/ambv/black/issues/21. Guido agrees here:
 # https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b.
-ignore = E501,W503
+#
+# Also ignore flake8's error about whitespaces before a ':'. (E203)
+# No longer complies with PEP8 See example, here:
+# https://github.com/PyCQA/pycodestyle/issues/373
+# and here: https://github.com/psf/black/issues/315
+ignore = E203,E501,E712,W503

.github/CODEOWNERS

@@ -4,7 +4,6 @@
 # repo. Unless a later match takes precedence, these owners will be
 # requested for review when someone opens a pull request.
 * @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj @rapidray12 @schmelz1 @cdunn17 @aloftus23 @Matthew-Grayson @nickviola
-
 # These folks own any files in the .github directory at the root of
 # the repository and any of its subdirectories.
 /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj

.github/dependabot.yml

@@ -5,6 +5,7 @@
 # these updates when the pull request(s) in the appropriate skeleton are merged
 # and Lineage processes these changes.
 
+version: 2
 updates:
   - directory: /
     ignore:
@@ -27,8 +28,18 @@ updates:
       # - dependency-name: github/codeql-action/init
     package-ecosystem: github-actions
     schedule:
-      interval: weekly
-
+      interval: "weekly"
+    ignore:
+      # Managed by cisagov/skeleton-generic
+      - dependency-name: actions/cache
+      - dependency-name: actions/checkout
+      - dependency-name: actions/setup-go
+      - dependency-name: actions/setup-python
+      - dependency-name: hashicorp/setup-terraform
+      - dependency-name: mxschmitt/action-tmate
+      # Managed by cisagov/skeleton-python-library
+      - dependency-name: actions/download-artifact
+      - dependency-name: actions/upload-artifact
   - directory: /
     package-ecosystem: pip
     schedule:

.github/labels.yml

@@ -59,9 +59,16 @@
 - color: "d73a4a"
   description: This issue or pull request addresses a security issue
   name: security
+- color: "8f4413"
+  description: Issue or PR from Previous Sprints
+  name: Tech Debt
 - color: "00008b"
   description: This issue or pull request adds or otherwise modifies test code
   name: test
+- color: "a4d70a"
+  description: Script is in testing and validation phase
+  name: Testing
+>>>>>>> pe-reports/CD-pe-reports-api-extended
 - color: "1d76db"
   description: This issue or pull request pulls in upstream updates
   name: upstream update

.github/workflows/build.yml

@@ -43,20 +43,31 @@ jobs:
           egress-policy: audit
       - id: setup-env
         uses: cisagov/setup-env-github-action@develop
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          # Temporarily hold back support for Python 3.11 due to the
+          # version pin for the PyMuPDF package. Pre-built wheels for
+          # Python 3.11 were not made available until the 1.21.0
+          # release and building the PyMuPDF package from source
+          # proves problematic due to dependencies. Please see the
+          # following issue for more information:
+          # https://github.com/cisagov/pe-reports/issues/441
+          #
+          # It's also the case that we cannot build the wheel for
+          # reportlab under Python 3.11 because its C code accesses
+          # the C struct members of PyFrameObject directly, and these
+          # struct members are no longer public as of Python 3.11.
+          # See here for more details:
+          # https://docs.python.org/3/whatsnew/3.11.html#pyframeobject-3-11-hiding
+          python-version: "3.10"
       # We need the Go version and Go cache location for the actions/cache step,
       # so the Go installation must happen before that.
       - id: setup-go
         uses: actions/setup-go@v4
         with:
-          # There is no expectation for actual Go code so we disable caching as
-          # it relies on the existence of a go.sum file.
-          cache: false
-          go-version: "1.20"
+          go-version: "1.19"
       - name: Lookup Go cache directory
         id: go-cache
         run: |
@@ -133,6 +144,11 @@ jobs:
           PACKAGE_URL: github.com/terraform-docs/terraform-docs
           PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }}
         run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
@@ -146,8 +162,6 @@ jobs:
         if: env.RUN_TMATE
   test:
     name: test source - py${{ matrix.python-version }}
-    needs:
-      - diagnostics
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -159,18 +173,17 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
-          - "3.11"
-          - "3.12"
+          # Temporarily hold back support for Python 3.11 due to the version pin
+          # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made
+          # available until the 1.21.0 release and building the PyMuPDF package from
+          # source proves problematic due to dependencies. Please see the following
+          # issue for more information: https://github.com/cisagov/pe-reports/issues/441
+          # - "3.11"
         include:
           - os: ubuntu-20.04
             python-version: "3.6"
     steps:
-      - id: harden-runner
-        name: Harden the runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - id: setup-python
         uses: actions/setup-python@v4
         with:
@@ -190,6 +203,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -212,15 +230,9 @@ jobs:
   coveralls-finish:
     runs-on: ubuntu-latest
     needs:
-      - diagnostics
       - test
     steps:
-      - id: harden-runner
-        name: Harden the runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - id: setup-python
         uses: actions/setup-python@v4
         with:
@@ -240,6 +252,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -254,7 +271,6 @@ jobs:
   build:
     name: build wheel - py${{ matrix.python-version }}
     needs:
-      - diagnostics
       - lint
       - test
     runs-on: ${{ matrix.os }}
@@ -268,18 +284,17 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
-          - "3.11"
-          - "3.12"
+          # Temporarily hold back support for Python 3.11 due to the version pin
+          # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made
+          # available until the 1.21.0 release and building the PyMuPDF package from
+          # source proves problematic due to dependencies. Please see the following
+          # issue for more information: https://github.com/cisagov/pe-reports/issues/441
+          # - "3.11"
         include:
           - os: ubuntu-20.04
             python-version: "3.6"
     steps:
-      - id: harden-runner
-        name: Harden the runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - id: setup-python
         uses: actions/setup-python@v4
         with:
@@ -298,6 +313,11 @@ jobs:
             ${{ hashFiles('setup.py') }}"
           restore-keys: |
             ${{ env.BASE_CACHE_KEY }}
+      # Install some necessary packages for building wheels
+      - uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: libmupdf-dev
+          version: 1.3.0
       - name: Install build dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel
@@ -315,7 +335,6 @@ jobs:
   test-build:
     name: test built wheel - py${{ matrix.python-version }}
     needs:
-      - diagnostics
       - build
     runs-on: ${{ matrix.os }}
     strategy:
@@ -328,18 +347,17 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
-          - "3.11"
-          - "3.12"
+          # Temporarily hold back support for Python 3.11 due to the version pin
+          # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made
+          # available until the 1.21.0 release and building the PyMuPDF package from
+          # source proves problematic due to dependencies. Please see the following
+          # issue for more information: https://github.com/cisagov/pe-reports/issues/441
+          # - "3.11"
         include:
           - os: ubuntu-20.04
             python-version: "3.6"
     steps:
-      - id: harden-runner
-        name: Harden the runner
-        uses: step-security/harden-runner@v2
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - id: setup-python
         uses: actions/setup-python@v4
         with:

.github/workflows/codeql-analysis.yml

@@ -63,7 +63,7 @@ jobs:
           egress-policy: audit
 
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/sync-labels.yml

@@ -19,10 +19,10 @@ jobs:
       issues: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
       - name: Sync repository labels
         if: success()
-        uses: crazy-max/ghaction-github-labeler@v5
+        uses: crazy-max/ghaction-github-labeler@v4
         with:
           # This is a hideous ternary equivalent so we only do a dry run unless
           # this workflow is triggered by the develop branch.

.gitignore

@@ -2,11 +2,51 @@
 # Files already tracked by Git are not affected.
 # See: https://git-scm.com/docs/gitignore
 
+## macOS ##
+.DS_Store
+
+## Project Specific ##
+*.html
+*.log
+build/
+output/
+src/pe_reports/*.html
+src/pe_reports/assets/
+
 ## Python ##
 __pycache__
 .coverage
 .mypy_cache
 .pytest_cache
 .python-version
 *.egg-info
-dist
+src/pe_reports/data/database.ini
+src/adhoc/data/database.ini
+build/*
+dist/*
+pe-reports/src/pe_reports/data/rabbitmq/*
+pe-reports/celerybeat-schedule.dat
+/src/pe_reports/data/rabbitmq/data
+src/adhoc/Bulletin/bulletin_template_filled.html
+src/adhoc/Bulletin/creds_bulletin_template_filled.html
+flaskLog.log
+src/adhoc/Bulletin/background.png
+src/adhoc/Bulletin/bulletin_template.html
+src/adhoc/Bulletin/creds_bulletin_template.html
+src/adhoc/Bulletin/footer_img.png
+src/adhoc/Bulletin/Franklin_Gothic_Book_Regular.ttf
+src/adhoc/Bulletin/Header2.png
+src/adhoc/Bulletin/Header_img.png
+src/adhoc/data/database.ini
+*.log
+*.log.*
+src/pe_reports/assets/*.png
+src/pe_reports/pe_reports_django_project/.env
+pe-reports/src/pe_reports/scorecard_assets/*.png
+src/pe_asm/port_scans/src/pkg
+src/pe_asm/port_scans/config.json
+=*
+src/pe_reports/pe_reports_django_project/*.bak
+src/pe_reports/pe_reports_django_project/*.dir
+src/pe_reports/pe_reports_django_project/*.dat
+dnstwist_output.txt

.pre-commit-config.yaml

@@ -141,7 +141,22 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies:
+          - boto3-stubs
+          - celery-types
+          - pandas-stubs
+          - types-chevron
+          - types-colorama
+          - types-docopt
+          - types-Flask-Migrate
+          - types-psycopg2
+          - types-Pygments
+          - types-pyOpenSSL
+          - types-PyYAML
+          - types-redis
+          - types-requests
+          - types-retry
           - types-setuptools
+          - types-python-dateutil==2.8.19
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.10.1
     hooks: