Merge pull request #440 from GeoDerp/custom_cec

Revert skforecast

.devcontainer/devcontainer.json

@@ -30,7 +30,13 @@
 		// Configure properties specific to VS Code.
 		"vscode": {
 		  // Add the IDs of extensions you want installed when the container is created.
-		  "extensions": ["ms-python.debugpy", "ms-python.python","charliermarsh.ruff"],
+		  // can add "ms-python.mypy-type-checker"
+		  "extensions": [
+			"ms-python.debugpy",
+			"ms-python.python",
+			"charliermarsh.ruff",
+			"SonarSource.sonarlint-vscode"
+		],
 		  "settings": {
 			"[python]": {
 			  "editor.formatOnSave": true,

.github/workflows/code-quality.yml

@@ -0,0 +1,63 @@
+name: "Code Quality Scan"
+# Use tools such as mypy ruf and sonarcloud to check for code quality & code security
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    env:
+      OS: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --reinstall --extra test
+
+      - name: Install ruff
+        run: uv tool install ruff
+
+      - name: Run Ruff
+        run: uvx ruff check --output-format=github .
+
+      # - name: Type Check with Mypy
+      #   shell: bash
+      #   run: |
+      #     uv run mypy ./src/
+
+      - name: SonarQube Scan
+        uses: SonarSource/sonarqube-scan-action@v4
+        env: 
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+  scan-pr:
+    needs:
+      - build
+    uses: "geoderp/osv-scanner-action/.github/workflows/[email protected]"
+    with:
+      fail-on-vuln: false
+      scan-args: |-
+        --recursive
+        ./
+    permissions:
+      security-events: write
+      contents: read
+      actions: read

.github/workflows/codecov.yaml

@@ -13,11 +13,10 @@ jobs:
         os: [ubuntu-latest]
     env:
       OS: ${{ matrix.os }}
-      PYTHON: '3.11'
     steps:
     - uses: actions/checkout@master
     - name: Install uv
-        uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v5
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
@@ -26,9 +25,9 @@ jobs:
       run: |
         uv venv
         uv pip install .[test]
-        coverage run -m --source=emhass unittest
-        coverage report
-        coverage xml
+        uv run coverage run -m --source=emhass unittest
+        uv run coverage report
+        uv run coverage xml
     - name: Upload coverage reports to Codecov
       run: |
         # Replace `linux` below with the appropriate OS

.github/workflows/docker-build-test.yaml

@@ -22,8 +22,6 @@ jobs:
         platform:
           [
             { target_arch: amd64, os_version: debian },
-            { target_arch: armv7, os_version: debian },
-            { target_arch: armhf, os_version: raspbian },
             { target_arch: aarch64, os_version: debian },
           ]
     steps:
@@ -68,8 +66,6 @@ jobs:
         platform:
           [
             { target_arch: amd64 },
-            { target_arch: armv7 },
-            { target_arch: armhf },
             { target_arch: aarch64 }
           ]
     # Check Docker image debian and python packages list for known vulnerabilities

.github/workflows/publish_docker-test.yaml

@@ -4,7 +4,7 @@ name: "Publish Docker test image"
 
 on:
   push:
-    branches: [testing]
+    branches: [master]
   workflow_dispatch:
 
 env:
@@ -25,8 +25,6 @@ jobs:
         platform:
           [
             { target_arch: amd64, os_version: debian },
-            { target_arch: armv7, os_version: debian },
-            { target_arch: armhf, os_version: raspbian },
             { target_arch: aarch64, os_version: debian },
           ]
     steps:
@@ -112,8 +110,6 @@ jobs:
         platform:
           [
             { target_arch: amd64 },
-            { target_arch: armv7 },
-            { target_arch: armhf },
             { target_arch: aarch64 },
           ]
     uses: "geoderp/osv-scanner-action/.github/workflows/[email protected]"

.github/workflows/publish_docker.yaml

@@ -24,8 +24,6 @@ jobs:
         platform:
           [
             { target_arch: amd64, os_version: debian },
-            { target_arch: armv7, os_version: debian },
-            { target_arch: armhf, os_version: raspbian },
             { target_arch: aarch64, os_version: debian },
           ]
     steps:
@@ -111,8 +109,6 @@ jobs:
         platform:
           [
             { target_arch: amd64 },
-            { target_arch: armv7 },
-            { target_arch: armhf },
             { target_arch: aarch64 },
           ]
     uses: "geoderp/osv-scanner-action/.github/workflows/[email protected]"

.github/workflows/python-test.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.12"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     env:
       OS: ${{ matrix.os }}
@@ -36,8 +36,8 @@ jobs:
       - name: Special dependencies for macos
         run: |
           brew install hdf5
-          uv pip install numpy==1.26.0
-          uv pip install tables==3.9.1
+          uv pip install numpy==2.0.0
+          uv pip install tables==3.10.2
         if: ${{ matrix.os == 'macos-latest' }}
       - name: Install EMHASS with test dependencies
         run: |

.github/workflows/updatePVLibDB.yaml

@@ -15,7 +15,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version-file: ".python-version"
     - name: install requirements
       run: python3 -m pip install --break-system-packages .[test]
     - name: pull latest SAM CEC files

.gitignore

@@ -11,6 +11,7 @@ secrets_emhass.yaml
 .vscode/launch.json
 .vscode/settings.json
 .vscode/tasks.json
+.devcontainer/devcontainer.json
 *.pkl
 **/app
 

.python-version

@@ -1 +1 @@
-3.11
+3.12

.vscode/launch.json

@@ -26,6 +26,25 @@
         "LOGGING_LEVEL": "DEBUG" 
       },
     },
+    {
+      "name": "EMHASS Production",
+      "type": "debugpy",
+      "request": "launch",
+      "console": "integratedTerminal",
+      "module": "gunicorn",
+      "purpose": [
+        "debug-in-terminal"
+      ],
+      "args": ["emhass.web_server:create_app()"],
+      "justMyCode": true,
+      "env": {
+        //"LEGACY_CONFIG_PATH": "/workspaces/emhass/tests/config_emhass.yaml",
+        "CONFIG_PATH": "/workspaces/emhass/config.json",
+        "SECRETS_PATH": "/workspaces/emhass/secrets_emhass.yaml",
+        "DATA_PATH": "/workspaces/emhass/data/",
+        "LOGGING_LEVEL": "DEBUG" 
+      },
+    },
     {
       "name": "EMHASS with options.json",
       "type": "debugpy",

.vscode/settings.json

@@ -13,5 +13,14 @@
     "python.defaultInterpreterPath": ".venv/bin/python",
     "cSpell.words": [
         "automations"
-    ]
+    ],
+    "sonarlint.connectedMode.project": {
+        "connectionId": "geoderp-emhass",
+        "projectKey": "GeoDerp_emhass"
+    },
+    "sonarlint.disableTelemetry": true,
+    "sonarlint.focusOnNewCode": true,
+    "sonarlint.analyzerProperties": {},
+    "mypy-type-checker.reportingScope": "workspace",
+    "editor.defaultFormatter": "charliermarsh.ruff"
 }

Dockerfile

@@ -54,6 +54,7 @@ RUN apt-get update \
 # add build packadges (just in case wheel does not exist)
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
+    python3-setuptools \
     gcc \
     patchelf \
     cmake \
@@ -68,15 +69,6 @@ RUN uv python install
 # specify hdf5
 RUN ln -s /usr/include/hdf5/serial /usr/include/hdf5/include && export HDF5_DIR=/usr/include/hdf5
 
-# note, its a good idea to remove the "llvm-dev" package and "LLVM_CONFIG=/usr/bin/llvm-config pip3 install 'llvmlite>=0.43'" once the llvmlite package has been fixed in piwheels
-RUN [[ "${TARGETARCH}" == "armhf" || "${TARGETARCH}" == "armv7" ]] && apt-get update && apt-get install -y --no-install-recommends llvm-dev && LLVM_CONFIG=/usr/bin/llvm-config uv pip install --break-system-packages --no-cache-dir --system 'llvmlite>=0.43' ||  echo "skipping llvm-dev install"
-
-# try, symlink apt cbc, to pulp cbc, in python directory (for 32bit)
-RUN [[ "${TARGETARCH}" == "armhf" || "${TARGETARCH}" == "armv7"  ]] &&  ln -sf /usr/bin/cbc /usr/local/lib/python3.11/dist-packages/pulp/solverdir/cbc/linux/32/cbc || echo "cbc symlink didnt work/not required"
-
-# if armv7, try install libatomic1 to fix scipy issue
-RUN [[ "${TARGETARCH}" == "armv7" ]] && apt-get update && apt-get install libatomic1 || echo "libatomic1 cant be installed"
-
 # make sure data directory exists
 RUN mkdir -p /app/data/
 
@@ -113,17 +105,23 @@ LABEL \
     io.hass.description="EMHASS: Energy Management for Home Assistant" \
     io.hass.version=${BUILD_VERSION} \
     io.hass.type="addon" \
-    io.hass.arch="aarch64|amd64|armhf|armv7" \
+    io.hass.arch="aarch64|amd64" \
     org.opencontainers.image.source="https://github.com/davidusb-geek/emhass" \
     org.opencontainers.image.description="EMHASS python package and requirements, in Home Assistant Debian container."
 
-# build EMHASS
+# Set up venv
 RUN uv venv && . .venv/bin/activate
-RUN [[ "${TARGETARCH}" == "armhf" || "${TARGETARCH}" == "armv7" ]] && uv pip install --verbose --extra-index-url https://www.piwheels.org/simple . || uv pip install --verbose .
+
+RUN [[ "${TARGETARCH}" == "aarch64" ]] && uv pip install --verbose ndindex || echo "libatomic1 cant be installed"
+
+
+# install packadges and build EMHASS
+RUN uv pip install --verbose .
 RUN uv lock
 
 # remove build only packages
 RUN apt-get remove --purge -y --auto-remove \
+    python3-setuptools \
     gcc \
     patchelf \
     cmake \
@@ -132,11 +130,13 @@ RUN apt-get remove --purge -y --auto-remove \
     && rm -rf /var/lib/apt/lists/*
 
 ENTRYPOINT [ "uv", "run", "gunicorn", "emhass.web_server:create_app()" ]
-# old
-# ENTRYPOINT [ "uv", "run", "--link-mode=copy", "--allow-insecure-host=localhost:5000", "--frozen", "-m", "emhass.web_server"]
 
 # for running Unittest
 #COPY tests/ /app/tests
 #RUN apt-get update &&  apt-get install python3-requests-mock -y
 #COPY data/ /app/data/
 #ENTRYPOINT ["uv","run","unittest","discover","-s","./tests","-p","test_*.py"]
+
+# Example of 32 bit specific 
+# try, symlink apt cbc, to pulp cbc, in python directory (for 32bit)
+#RUN [[ "${TARGETARCH}" == "armhf" || "${TARGETARCH}" == "armv7"  ]] &&  ln -sf /usr/bin/cbc /usr/local/lib/python3.11/dist-packages/pulp/solverdir/cbc/linux/32/cbc || echo "cbc symlink didnt work/not required"

docs/conf.py

@@ -12,51 +12,52 @@
 
 import os
 import sys
-for x in os.walk('../src/'):
-  sys.path.insert(0, x[0])
+
+for x in os.walk("../src/"):
+    sys.path.insert(0, x[0])
 
 # -- Project information -----------------------------------------------------
 
-project = 'emhass'
-copyright = '2021-2025, David HERNANDEZ'
-author = 'David HERNANDEZ'
+project = "emhass"
+copyright = "2021-2025, David HERNANDEZ"
+author = "David HERNANDEZ"
 
 # The full version, including alpha/beta/rc tags
-release = '0.12.4'
+release = "0.12.4"
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc', 'myst_parser']
+extensions = ["sphinx.ext.autodoc", "myst_parser"]
 
 myst_enable_extensions = [
     "amsmath",
     "dollarmath",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-## EMHASS custom theme  
+html_static_path = ["_static"]
+## EMHASS custom theme
 html_css_files = [
-    'css/custom.css',
-]
+    "css/custom.css",
+]

gunicorn.conf.py

@@ -1,11 +1,10 @@
 import os
-from distutils.util import strtobool
 
 bind = f"{os.getenv('IP', '0.0.0.0')}:{os.getenv('PORT', '5000')}"
 
 workers = int(os.getenv("WEB_CONCURRENCY", 1))
 threads = int(os.getenv("PYTHON_MAX_THREADS", 8))
 
-reload = bool(strtobool(os.getenv("WEB_RELOAD", "true")))
+reload = (os.getenv("WEB_RELOAD", "true").lower()) == "true"
 
 timeout = int(os.getenv("WEB_TIMEOUT", 240))