TVB-2113: add bin path to modify distribution python path; updated th…

…e documentation
the-virtual-brain · Dec 4, 2024 · 5e39d33 · 5e39d33
1 parent ae03542
commit 5e39d33
Show file tree

Hide file tree

Showing 8 changed files with 112 additions and 25 deletions.
diff --git a/tvb_build/Jenkinsfile-Mac b/tvb_build/Jenkinsfile-Mac
@@ -60,6 +60,18 @@ pipeline {
                 archiveArtifacts artifacts: 'TVB_Mac*.zip'
             }
         }
+        stage ('Tests on SqLite') {
+            steps {
+                sh '''#!/bin/bash
+                    source /Applications/anaconda3/etc/profile.d/conda.sh
+                    conda activate mac-distribution
+                    cd tvb_bin
+                    /bin/bash run_tests.sh
+                    exit 0
+                '''
+                junit 'tvb_bin/TEST_OUTPUT/results_*.xml'
+            }
+        }
     }
     post {
         changed {

diff --git a/tvb_build/Jenkinsfile-Step1 b/tvb_build/Jenkinsfile-Step1
@@ -50,6 +50,26 @@ pipeline {
                 archiveArtifacts artifacts: 'dist/*'
             }
         }
+        stage ('Compute coverage') {
+            steps {
+                withDockerContainer(image: '${FULL_DOCKER_IMAGE_NAME}') {
+                    sh '''#!/bin/bash
+                        source activate tvb-run
+                        rm -rf TEST_OUTPUT
+                        mkdir TEST_OUTPUT
+                        cd tvb_build
+                        bash install_full_tvb.sh
+                        cd ../tvb_library
+                        py.test --cov-config .coveragerc --cov=tvb tvb/tests --cov-branch --cov-report xml:../TEST_OUTPUT/coverage_lib.xml --junitxml=../TEST_OUTPUT/results_lib.xml
+                        cd ../tvb_framework
+                        py.test --cov-config .coveragerc --cov=tvb tvb/tests --cov-branch --cov-report xml:../TEST_OUTPUT/coverage_frw.xml --junitxml=../TEST_OUTPUT/results_frw.xml
+                        exit 0
+                    '''
+                }
+                 junit 'TEST_OUTPUT/results_*.xml'
+                 step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'TEST_OUTPUT/coverage_*.xml', failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false])
+            }
+        }
     }
     post {
         always {

diff --git a/tvb_build/Jenkinsfile-Windows b/tvb_build/Jenkinsfile-Windows
@@ -16,6 +16,14 @@ pipeline {
                 }
             }
         }
+        stage ('Tests on SqLite') {
+            steps {
+                withDockerContainer(image: "${FULL_DOCKER_IMAGE_NAME}") {
+                    powershell 'cd tvb_build; cmd /k "activate tvb-run & install_full_tvb.bat & cd ../tvb_bin & run_tests.bat"; exit 0'
+                }
+                junit 'tvb_bin/TEST_OUTPUT/results_*.xml'
+            }
+        }
         stage('Copy Step 1') {
              steps {
                  bat '''

diff --git a/tvb_build/build_from_conda.py b/tvb_build/build_from_conda.py
@@ -35,7 +35,7 @@
 
 
 class Config:
-    def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, commands_map, command_factory):
+    def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, commands_map, command_factory, bin_path=None):
         # System paths:
         self.anaconda_env_path = anaconda_env_path
 
@@ -69,6 +69,10 @@ def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, comma
         _artifact_glob = "TVB_" + platform_name + "_*.zip"
         self.artifact_glob = join(self.build_folder, _artifact_glob)  # this is used to match old artifacts
         self.artifact_pth = join(self.build_folder, self.artifact_name)
+        if bin_path:
+            self.bin_path = join(self.target_library_root, bin_path)
+        else:
+            self.bin_path = None
 
     @staticmethod
     def win64():
@@ -86,7 +90,7 @@ def win64():
             'bin\\tvb_stop.bat': 'distribution stop',
             'bin\\jupyter_notebook.bat': set_path + 'cd ..\\bin\n..\\tvb_data\\Scripts\\jupyter lab ..\\demo_scripts',
             'demo_scripts\\jupyter_notebook.bat': set_path + 'cd ..\\demo_scripts\n..\\tvb_data\\Scripts\\jupyter lab',
-            'bin\\activate_tvb_env.bat':'REM Conda must be installed before running this script \n conda activate ../tvb_data \n cmd /K'
+            'bin\\activate_tvb_env.bat':'REM Conda must be installed before running this script\nconda activate ../tvb_data\ncmd /K'
         }
 
         return Config("Windows", "C:\\miniconda\\envs\\tvb-run",
@@ -116,11 +120,11 @@ def linux64():
             'bin/tvb_stop.sh': 'bash ./distribution.sh stop',
             'bin/jupyter_notebook.sh': set_path + 'cd ../bin\n../tvb_data/bin/python -m jupyterlab ../demo_scripts',
             'demo_scripts/jupyter_notebook.sh': set_path + 'cd ../demo_scripts\n../tvb_data/bin/python -m jupyterlab',
-            'bin/activate_tvb_env.sh': '# Conda must be installed before running this script \n conda activate ../tvb_data \n $SHELL'
+            'bin/activate_tvb_env.sh': '# Conda must be installed before running this script;\n# Run this script with source.\nconda activate ../tvb_data \n'
         }
 
         return Config("Linux", "/opt/conda/envs/tvb-run", join("lib", Environment.PYTHON_FOLDER, "site-packages"),
-                      commands_map, _create_unix_command)
+                      commands_map, _create_unix_command, "bin")
 
 
 def _log(indent, msg):
@@ -203,7 +207,24 @@ def _create_windows_script(target_file, command):
     os.chmod(target_file, 0o755)
 
 
+def _replace_first_line_if_pattern(pathname: str, pattern: str, replacement: str):
+    """
+    Replaces the first line of a file with a given string if the pathname contains a specific pattern.
+    """
+    for filename in os.listdir(pathname):
+        file_path = os.path.join(pathname, filename)
+        if pattern in file_path:
+            with open(file_path, 'r') as file:
+                lines = file.readlines()
+                if lines:
+                    lines[0] = replacement + '\n'
+            with open(file_path, 'w') as file:
+                file.writelines(lines)
+            _log(1, f"First line of {file_path} replaced with: {replacement}")
+
+
 def _modify_pth(pth_name):
+    # Log if one of the files pip, pip3 or pip3.11 are missing, but do not stop the execution if it is missing
     """
     Replace tvb links with paths
     """
@@ -280,6 +301,8 @@ def prepare_anaconda_dist(config):
 
     _log(1, "Modifying PTH " + config.easy_install_pth)
     _modify_pth(config.easy_install_pth)
+    if config.bin_path:
+        _replace_first_line_if_pattern(config.bin_path, 'bin/pip', '#!../tvb_data/bin/python')
     _fix_jupyter_kernel(config.target_library_root, config.platform_name == "Windows")
 
     _log(1, "Creating command files:")

diff --git a/tvb_build/docker/Dockerfile-build b/tvb_build/docker/Dockerfile-build
@@ -18,7 +18,7 @@ RUN service postgresql start && createdb -O postgres tvb-test && psql --command
 
 USER root
 RUN conda update -n base -c defaults conda; conda init bash
-RUN conda create -y --name tvb-run python=3.11 nomkl numba scipy numpy cython psycopg2
+RUN conda create -y --name tvb-run python=3.11 pip nomkl numba scipy numpy cython psycopg2
 RUN conda install -y --name tvb-run -c conda-forge jupyterlab tvb-gdist
 RUN /opt/conda/envs/tvb-run/bin/pip install --upgrade pip
 RUN /opt/conda/envs/tvb-run/bin/pip install lockfile scikit-build

diff --git a/tvb_build/setup_mac.py b/tvb_build/setup_mac.py
@@ -257,7 +257,7 @@ def prepare_mac_dist():
     _create_command_file(os.path.join(DIST_FOLDER, "bin", 'tvb_stop'),
                          'source ./distribution.command stop', 'Stopping TVB related processes.', True)
     _create_command_file(os.path.join(DIST_FOLDER, "bin", 'activate_tvb_env'),
-                         'conda activate ../tvb_data \n $SHELL', 'Conda must be installed before running this script', True)
+                         'conda activate ../tvb_data \n', 'Conda must be installed before running this script', True)
     jupyter_command = '/Applications/{}/Contents/Resources/bin/jupyter lab '.format(APP)
     _create_command_file(os.path.join(DIST_FOLDER, "bin", 'jupyter_notebook'),
                          jupyter_command + '../demo_scripts', 'Launching IPython Notebook from TVB Distribution')

diff --git a/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml b/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml
@@ -237,7 +237,7 @@
     <dependency env="mac" name="asttokens">
         <full-name value="asttokens"/>
         <project-home value="https://pypi.org/project/asttokens/"/>
-        <version value="[2.2.1,2.4.1]"/>
+        <version value="[2.2.1,2.4.1,3.0.0]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_asttokens.txt"/>
         <license-type value="Apache_ 2"/>
@@ -1197,7 +1197,7 @@
     <dependency env="[win,mac,linux]" name="cryptography">
         <full-name value="cryptography"/>
         <project-home value="https://github.com/pyca/cryptography"/>
-        <version value="[39.0.1,41.0.2,43.0.3]"/>
+        <version value="[39.0.1,41.0.2,43.0.3,44.0.0]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_crypto.txt"/>
         <license-type value="BSD_"/>
@@ -1637,7 +1637,7 @@
     <dependency env="[win,mac,linux]" name="fastjsonschema">
         <full-name value="Fastjsonschema"/>
         <project-home value="https://github.com/horejsek/python-fastjsonschema"/>
-        <version value="2.20.0"/>
+        <version value="[2.20.0,2.21.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_fastjsonschema.txt"/>
         <license-type value="BSD_ 3 clause"/>
@@ -1699,7 +1699,7 @@
     <dependency env="[win,mac,linux]" name="fonttools">
         <full-name value="Fonttools"/>
         <project-home value="https://github.com/fonttools/fonttools"/>
-        <version value="[4.54.1,4.55.0]"/>
+        <version value="[4.54.1,4.55.0,4.55.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_fonttools.txt"/>
         <license-type value="MIT_"/>
@@ -2053,7 +2053,7 @@
     <dependency env="[win,linux]" name="imageio">
         <full-name value="imageio"/>
         <project-home value="http://imageio.github.io/"/>
-        <version value="[2.18.1,2.31.0,2.31.1,2.36.0]"/>
+        <version value="[2.18.1,2.31.0,2.31.1,2.36.0,2.36.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_imageio.txt"/>
         <license-type value="BSD_ 2nd clause"/>
@@ -2214,7 +2214,7 @@
     <dependency env="[win,mac,linux]" name="ipython">
         <full-name value="iPython"/>
         <project-home value="http://ipython.org/"/>
-        <version value="[8.13.2,8.14.0,8.28.0,8.29.0]"/>
+        <version value="[8.13.2,8.14.0,8.28.0,8.29.0,8.30.0]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_ipython.txt"/>
         <license-type value="BSD_ 3 clause"/>
@@ -2602,7 +2602,7 @@
     <dependency env="[win,mac,linux]" name="jupyterlab">
         <full-name value="Jupyterlab"/>
         <project-home value="https://jupyter.org/"/>
-        <version value="[4.0.0,4.0.1,4.0.3,4.2.5,4.3.1]"/>
+        <version value="[4.0.0,4.0.1,4.0.3,4.2.5,4.3.1,4.3.2]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_jupyterlab.txt"/>
         <license-type value="BSD_ 3nd clause"/>
@@ -2674,7 +2674,7 @@
     <dependency env="[win,mac,linux]" name="jwt">
         <full-name value="JWT"/>
         <project-home value="https://github.com/davedoesdev/python-jwt"/>
-        <version value="[2.7.0,2.8.0,2.9.0,2.10.0]"/>
+        <version value="[2.7.0,2.8.0,2.9.0,2.10.0,2.10.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_jwt.txt"/>
         <license-type value="MIT_"/>
@@ -3064,7 +3064,7 @@
     <dependency env="[win,mac,linux]" name="pyjwt">
         <full-name value="Pyjwt"/>
         <project-home value="https://github.com/davedoesdev/python-jwt"/>
-        <version value="[2.7.0,2.9.0,2.10.0]"/>
+        <version value="[2.7.0,2.9.0,2.10.0,2.10.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_pyjwt.txt"/>
         <license-type value="MIT_"/>
@@ -3074,7 +3074,7 @@
     <dependency env="[win,mac,linux]" name="keycloak">
         <full-name value="Keycloak Identity and Access Management"/>
         <project-home value="https://www.keycloak.org/"/>
-        <version value="[2.16.3,3.0.0,3.3.0,4.6.2,4.7.0,4.7.2]"/>
+        <version value="[2.16.3,3.0.0,3.3.0,4.6.2,4.7.0,4.7.2,4.7.3]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_keycloak.txt"/>
         <license-type value="Apache_ 2"/>
@@ -3084,7 +3084,7 @@
     <dependency env="[win,mac,linux]" name="python-keycloak">
         <full-name value="Python Keycloak"/>
         <project-home value="https://github.com/marcospereirampj/python-keycloak/"/>
-        <version value="[2.16.3,3.0.0,3.3.0,4.6.2,4.7.0,4.7.2]"/>
+        <version value="[2.16.3,3.0.0,3.3.0,4.6.2,4.7.0,4.7.2,4.7.3]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_python_keycloak.txt"/>
         <license-type value="MIT_"/>
@@ -3434,7 +3434,7 @@
     <dependency env="[win,mac,linux]" name="mako">
         <full-name value="Mako"/>
         <project-home value="https://pypi.org/project/Mako/"/>
-        <version value="[1.2.4,1.3.6]"/>
+        <version value="[1.2.4,1.3.6,1.3.7]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_mako.txt"/>
         <license-type value="MIT_"/>
@@ -3505,7 +3505,7 @@
     <dependency env="[win,mac,linux]" name="matplotlib">
         <full-name value="MatPlotLib"/>
         <project-home value="https://matplotlib.org/stable/users/license.html"/>
-        <version value="[3.5.3,3.9.2]"/>
+        <version value="[3.5.3,3.9.2,3.9.3]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_matplotlib.txt"/>
         <license-type value="BSD_ (PSF_ based)"/>
@@ -3708,7 +3708,7 @@
     <dependency env="[win,mac,linux]" name="nbclient">
         <full-name value="A client library for executing notebooks."/>
         <project-home value="https://jupyter.org/"/>
-        <version value="[0.8.0,0.10.0]"/>
+        <version value="[0.8.0,0.10.0,0.10.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_jupyter.txt"/>
         <license-type value="BSD_ modified"/>
@@ -4433,7 +4433,7 @@
     <dependency env="[win,mac,linux]" name="pytest">
         <full-name value="PyTest"/>
         <project-home value="http://docs.pytest.org/"/>
-        <version value="[3.3.1,4.4.2,8.3.3]"/>
+        <version value="[3.3.1,4.4.2,8.3.3,8.3.4]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_pytest.txt"/>
         <license-type value="MIT_"/>
@@ -4739,7 +4739,7 @@
     <dependency env="[win,linux,mac]" name="rpds-py">
         <full-name value="Rpds-py"/>
         <project-home value="https://github.com/crate-py/rpds"/>
-        <version value="[0.20.0,0.21.0]"/>
+        <version value="[0.20.0,0.21.0,0.22.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_rpds_py.txt"/>
         <license-type value="MIT_"/>
@@ -5476,7 +5476,7 @@
     <dependency env="[win,mac,linux]" name="tornado">
         <full-name value="Tornado"/>
         <project-home value="http://www.tornadoweb.org/en/stable/"/>
-        <version value="[6.3.2,6.4.1]"/>
+        <version value="[6.3.2,6.4.1,6.4.2]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_tornado.txt"/>
         <license-type value="Apache_ 2"/>
@@ -5517,7 +5517,7 @@
     <dependency env="[win,mac,linux]" name="tomli">
         <full-name value="tomli"/>
         <project-home value="https://github.com/hukkin/tomli"/>
-        <version value="[2.0.1,2.0.2,2.1.0]"/>
+        <version value="[2.0.1,2.0.2,2.1.0,2.2.1]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_tomli.txt"/>
         <license-type value="MIT_"/>
@@ -6035,7 +6035,7 @@
     <dependency env="[win,mac,win]" name="zope.interface">
         <full-name value="Zope.interface"/>
         <project-home value="https://pypi.python.org/pypi/zope.interface"/>
-        <version value="7.1.1"/>
+        <version value="[7.1.1,7.2]"/>
         <usage value="dynamic linking"/>
         <license value="LICENSE_zope.txt"/>
         <license-type value="ZPL 2.1"/>

diff --git a/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst b/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst
@@ -45,6 +45,7 @@ Launching the application
 
 In the TVB_Distribution folder you should find a sub-folder `bin` with a number of scripts:
 
+- activate_tvb_env
 - tvb_start
 - tvb_clean
 - tvb_stop
@@ -137,6 +138,29 @@ A Python shell will appear.
    $ ./distribution.sh start COMMAND_PROFILE
 
 
+Activate the conda environment
+..............................
+
+On Windows, from the terminal, you can run the conda environment activation script or you can double click on the script
+
+.. code-block:: bash
+
+   $ activate_tvb_env.bat
+
+On Linux, from the terminal, you can run the conda environment with source:
+
+.. code-block:: bash
+
+   $ source activate_tvb_env.sh
+
+On Mac, from the terminal, you can run the conda environment with source:
+
+.. code-block:: bash
+
+   $ source activate_tvb_env.command
+
+After activating the Conda environment, you can modify the currently installed Python packages.
+
 Configuring TVB
 ---------------