Feature/dummy labview

.github/workflows/python-package.yml

@@ -14,7 +14,7 @@ jobs:
     # runs-on: [self-hosted, linux, X64]
     strategy:
       matrix:
-        python-version: [3.9.12]
+        python-version: ["3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -30,22 +30,19 @@ jobs:
         env
     - name: Install dependencies
       run: |
-        pip install -r requirements/local.txt
-        pip install -e .
-
+        pip install -e .[local]
     - name: Docker-compose start mqtt
       run: | 
         cd docker/test_tracker_dcs_web
         docker-compose up -d 
         cd -
     - name: Run black --check .
-      run: black --check . --exclude _actions
+      run: black --check . --exclude _actions --exclude "tracker_dcs_web/_version.py"
     - name: Test with pytest
       run: |
         cd unittests
         pip install pytest-cov
         pytest --cov=../ --cov-report=xml
-        ls
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:
@@ -55,4 +52,4 @@ jobs:
         flags: unittests
         name: codecov-umbrella
         # path_to_write_report: ./coverage/codecov_report.txt
-        verbose: true
+        verbose: true

.gitignore

@@ -128,3 +128,5 @@ dmypy.json
 # Pyre type checker
 .pyre/
 .idea
+
+_version.py

Dockerfile

@@ -1,22 +1,51 @@
-FROM python:3.9
+FROM python:3.10-slim
 
-RUN apt-get update && apt-get -y upgrade
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install git && \
+    rm -rf /var/lib/apt/lists/*
 
-WORKDIR /code
 
-# install dependencies
 
-COPY requirements ./requirements
+ARG FROM_PATH=.
+ARG USER=service-user
+ARG HOME=/home/$USER
+ARG TO_PATH=$HOME/service
+ARG VIRTUAL_ENV=$TO_PATH/venv
+
+# Create service directory
+RUN mkdir -p $TO_PATH
+WORKDIR $TO_PATH
+
+# Create virtual environment
+RUN python -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements/docker.txt
 
-# install this package
+# Copy necessary files
+COPY $FROM_PATH/pyproject.toml pyproject.toml
+COPY $FROM_PATH/.git .git
+
+RUN mkdir ./tracker_dcs_web
+RUN --mount=type=cache,target=$HOME/.cache/pip pip install .
+
+# Copy service source code
+COPY $FROM_PATH/tracker_dcs_web $TO_PATH/tracker_dcs_web
+
+# service user
+RUN /usr/sbin/useradd -m -u 5000 $USER
+
+# prepare storage dir
+RUN mkdir -p $TO_PATH/files
+ENV STORAGE_DIR=/$TO_PATH/files
+RUN chown -R $USER $STORAGE_DIR
+
+RUN rm -rf /$TO_PATH/.git
+
+# Create service user
+USER $USER
+
+ENV PYTHONPATH="$PWD:$PYTHONPATH"
 
-COPY tracker_dcs_web/ ./tracker_dcs_web
-COPY setup.py ./
-RUN pip install -e .
 
-# create user
 
-RUN useradd --create-home appuser
-USER appuser

README.md

@@ -6,32 +6,35 @@ Input from outside, e.g. LabView
 
 ## Developer instructions
 
-Local installation: 
+Environment creation: 
+
+```
+python -m venv venv
+source venv/bin/activate
+```
+
+Installation: 
 
 ```
-conda create -n web_server python=3.9
-conda activate web_server
 pip install --upgrade pip
-pip install -r requirements/local.txt
-pip install -e .  
+pip install -e .[local]  
 ```
 
 ## Environment variables 
 
-These secrets will be printed out by the server. 
-They are not used for authentication, so set them to whatever you want. 
-
-* `APP_USER`: app user
-* `APP_PASSWORD`: app password
-* `MQTT_HOST`: mosquitto host, e.g. localhost (don't include the port)
+| Variable        | Description     | Mandatory | Default   | 
+|-----------------|-----------------|-----------|-----------| 
+| MQTT_HOST       | mosquitto host  | no        | localhost | 
+| MQTT_PORT       | mosquitto port  | no        | 1883      | 
+| MQTT_TOPIC_DATA | mqtt root topic | no        | /labview  |
 
 ## Docker stack 
 
-To run the unit tests you need an MQTT broker. Start it: 
+To run the unit tests on the web server locally you need an MQTT broker. Start it: 
 
 ```commandline
 cd docker/test_tracker_dcs_web
-docker-compose up -d 
+docker compose up -d 
 ```
 
 ## Unit tests and linting 
@@ -68,10 +71,26 @@ See the docker-compose stack, the web_server is already integrated.
 
 To make a request to the root path: 
 
-```commandline
-curl curl localhost:8001/
+To be written
+
+## Injecting data 
+
+* Start the DCS stack
+* Go to the root of this package and then do:
+
+```
+export DATADIR=${PWD}/unittests/data/labview
+dummy_labview http://localhost:8001 ${DATADIR}/mapping.txt ${DATADIR}/header.txt ${DATADIR}/measures.txt -n 10 
 ```
 
+* Make sure you get mqtt messages : 
 
+```
+docker exec stack-mosquitto-1 mosquitto_sub -v -t /labview
+```
 
+```
+/labview {"0 (PS)": 23.794608, "2 (PS)": 24.270239, "3 (PS)": 24.368866, "7 (PS)": 24.584218, "8 (PS)": 25.116412, "9 (PS)": 25.1832, "10 (PS)": 25.10516, "11 (PS)": 25.095721, "14 (PS)": 23.724818, "15 (PS)": 25.148199, "20 (PS)": 24.442453, "21 (PS)": 24.837407, "23 (PS)": 25.257441, "24 (PS)": 25.285556, "28 (PS)": 24.313686, "30 (PS)": 25.762269, "31 (PS)": 24.577083, "1 (2S)": 25.372097, "26 (2S)": 24.423724, "33 (2S)": 24.756658, "35 (2S)": 24.558114, "36 (2S)": 25.237437, "37 (2S)": 24.666911, "38 (2S)": 24.866404, "39 (2S)": 23.705506, "40 (2S)": 24.590622, "41 (2S)": 24.929428, "42 (2S)": 24.592504, "43 (2S)": 24.361401, "44 (2S)": 24.615935, "45 (2S)": 24.23221, "46 (2S)": 24.932867, "47 (2S)": 24.36999, "48 (2S)": 24.945494, "49 (2S)": 25.058509, "50 (2S)": 24.389153, "51 (2S)": 24.32974, "52 (2S)": 24.72595, "53 (2S)": 24.368981, "54 (2S)": 24.427689, "55 (2S)": 24.955819, "56 (2S)": 24.954875, "57 (2S)": 24.435747, "58 (2S)": 24.459637, "59 (2S)": 25.145329, "60 (2S)": 23.671368, "63 (2S)": 24.625984, "64 (2S)": 25.448417, "69": 27.223336, "70": 18.907616, "117": 28.239884, "118": 28.324767, "119": 28.506608, "1_T [oC]": 26.204834, "1_Igro. [%]": 0.905784, "1_Tr [oC]": -35.269409, "1_Tg [oC]": -32.027641, "1_M [Mol/l]": 303.450836, "2_T [oC]": 26.095947, "2_Igro. [%]": 0.919002, "2_Tr [oC]": -35.186695, "2_Tg [oC]": -31.950109, "2_M [Mol/l]": 305.948853, "PS [V]": -6.534825e-05, "2S [V]": 0.000188, "P [mbar]": 58.099059, "Datetime_ns": 1652951192000000000}
+/labview {"0 (PS)": 23.474371, "2 (PS)": 23.844079, "3 (PS)": 24.138962, "7 (PS)": 24.540439, "8 (PS)": 25.033106, "9 (PS)": 25.019948, "10 (PS)": 25.038958, "11 (PS)": 25.047888, "14 (PS)": 23.451034, "15 (PS)": 25.107758, "20 (PS)": 24.051482, "21 (PS)": 24.631699, "23 (PS)": 25.223314, "24 (PS)": 25.217837, "28 (PS)": 24.071647, "30 (PS)": 25.693499, "31 (PS)": 24.337925, "1 (2S)": 25.32945, "26 (2S)": 24.360936, "33 (2S)": 24.698105, "35 (2S)": 24.508007, "36 (2S)": 25.183981, "37 (2S)": 24.609425, "38 (2S)": 24.818768, "39 (2S)": 23.707785, "40 (2S)": 24.538172, "41 (2S)": 24.873097, "42 (2S)": 24.553522, "43 (2S)": 24.329078, "44 (2S)": 24.561973, "45 (2S)": 24.098901, "46 (2S)": 24.896844, "47 (2S)": 24.317725, "48 (2S)": 24.901188, "49 (2S)": 24.994155, "50 (2S)": 24.246465, "51 (2S)": 24.182257, "52 (2S)": 24.664652, "53 (2S)": 24.381589, "54 (2S)": 24.392611, "55 (2S)": 24.893014, "56 (2S)": 24.911699, "57 (2S)": 24.422063, "58 (2S)": 24.422109, "59 (2S)": 25.082687, "60 (2S)": 23.696626, "63 (2S)": 24.579633, "64 (2S)": 25.405447, "69": 27.104352, "70": 19.102592, "117": 28.122215, "118": 28.217106, "119": 28.426495, "1_T [oC]": 26.12793, "1_Igro. [%]": 0.919471, "1_Tr [oC]": -35.162464, "1_Tg [oC]": -31.927399, "1_M [Mol/l]": 306.684235, "2_T [oC]": 26.030273, "2_Igro. [%]": 0.932644, "2_Tr [oC]": -35.077103, "2_Tg [oC]": -31.847408, "2_M [Mol/l]": 309.287231, "PS [V]": 0.000281, "2S [V]": -4.769084e-05, "P [mbar]": 58.309803, "Datetime_ns": 1652951202000000000}
+```
 

pyproject.toml

@@ -0,0 +1,79 @@
+[project]
+name = "tracker_dcs_web"
+description = "Web server accepting labview data for IP2I CMS tracker DCS"
+dynamic = ["version"]
+authors = [
+    { name = "Colin Bernet", email = "[email protected]" },
+]
+maintainers = [
+    { name = "Colin Bernet", email = "[email protected]" },
+]
+
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = ["tracker", "DCS", "labview", "CMS", "IP2I"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+]
+
+dependencies = [
+    "fastapi==0.95.*",
+    "paho-mqtt>=1.5.1, <1.6.0",
+    "pandas>=2.0.3, <2.1.0",
+    "python-multipart>=0.0.6, <0.1",
+    "pyyaml>=6.0.1, <6.1",
+    # "requests",
+    "uvicorn==0.22.*",
+]
+
+[project.optional-dependencies]
+docker = [
+]
+local = [
+    "black[d]==23.3.*",
+    "coverage==7.2.*",
+    "httpx==0.24.*",
+    "pytest==7.3.*",
+    "pytest-asyncio==0.21.*",
+    "pytest-mock==3.10.*",
+    "python-dotenv==1.0.*",
+    "requests>=2.31.0, <2.32"
+]
+
+
+[project.urls]
+repository = "https://github.com/cms-tedd-ip2i/tracker_dcs_web"
+
+[project.scripts]
+dummy_labview = "tracker_dcs_web.scripts.dummy_labview:main"
+
+
+[build-system]
+requires = [
+    "setuptools >= 66.1.1",
+    "wheel",
+    "setuptools-scm[toml] >= 7.1.0",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["tracker_dcs_web"]
+
+[tool.setuptools_scm]
+git_describe_command = "git describe --tags --match 'v[0-9]*'"
+write_to = "tracker_dcs_web/_version.py"
+
+[tool.coverage.run]
+source = ["tracker_dcs_web"]
+
+[tool.black]
+include = '\.pyi?$' # All Python files
+exclude = '''
+/(
+  \.git
+  | build
+  | dist
+  | venv
+  | tracker_dcs_web/_version.py
+)/
+'''

tracker_dcs_web/mqtt.py

@@ -3,15 +3,20 @@
 from tracker_dcs_web.utils.logger import logger
 
 
-mqtt_host = os.environ["MQTT_HOST"]
+mqtt_host = os.environ.get("MQTT_HOST", "localhost")
+mqtt_port = int(os.environ.get("MQTT_PORT", 1883))
 
 
 def on_connect(client, userdata, flags, rc):
-    # Subscribing in on_connect() means that if we lose the connection and
-    # reconnect then subscriptions will be renewed.
-    logger.info(f"connected to mqtt broker: {mqtt_host}")
+    logger.info(f"connected to mosquitto: {mqtt_host}:{mqtt_port}")
+
+
+def on_publish(client, userdata, result):
+    logger.info(f"data published: {result}")
 
 
 client = mqtt.Client()
 client.on_connect = on_connect
-client.connect(mqtt_host, 1883, 60)
+client.on_publish = on_publish
+client.connect(mqtt_host, mqtt_port, 60)
+client.loop_start()

tracker_dcs_web/scripts/dummy_labview.py

@@ -0,0 +1,93 @@
+import argparse
+import itertools
+import logging
+import mimetypes
+import os
+import pathlib
+import requests
+import time
+from typing import TextIO
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def send_data(data: str, url: str):
+    logging.info("send data")
+    response = requests.post(f"{url}/data", params={"measurements": data})
+    response.raise_for_status()
+
+
+def send_mapping(mapping: TextIO, url: str):
+    mapping.seek(0)
+    filename = "mapping.txt"
+    response = requests.post(
+        f"{url}/mapping",
+        files={
+            "upload_file": (
+                filename,
+                mapping,
+                mimetypes.guess_type(filename)[0],
+            )
+        },
+    )
+    response.raise_for_status()
+
+
+def good_file(file_path) -> pathlib.Path:
+    """type for argparse (file that exists)"""
+    if not os.path.isfile(file_path):
+        raise argparse.ArgumentTypeError(f"{file_path} does not exist or is not a file")
+    return file_path
+
+
+def parse_args(args: [str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Send dummy labview data")
+    parser.add_argument("url", type=str, help="web server url")
+    parser.add_argument("mapping", type=good_file, help="mapping file")
+    parser.add_argument("header", type=good_file, help="header file")
+    parser.add_argument("data", type=good_file, help="data file")
+    parser.add_argument("-t", type=int, help="time period", default=2)
+    parser.add_argument("-n", type=int, help="number of messages", default=None)
+
+    args = parser.parse_args(args)
+    return args
+
+
+def main(args=None):
+    """Load items.
+
+    > load_rei tests/data/SCHNEIDER_*.txt
+
+    """
+    import sys
+
+    if not args:
+        args = sys.argv[1:]
+    args = parse_args(args)
+
+    with open(args.mapping) as mapping_file:
+        send_mapping(mapping_file, args.url)
+
+    with open(args.header) as header_file:
+        header = header_file.read()
+    send_data(header, args.url)
+
+    with open(args.data) as data_file:
+        data = data_file.readlines()
+
+    n_sent = 0
+    for dataline in itertools.cycle(data):
+        # cycling on measurements
+        send_data(dataline, args.url)
+        if args.n:
+            n_sent += 1
+            if n_sent >= args.n:
+                break
+        time.sleep(args.t)
+
+    logging.info(f"sent {n_sent} data")
+
+
+if __name__ == "__main__":
+    main()