From 5e39d33a6a08c3b1892a7c74491fe3b5a2f36a8d Mon Sep 17 00:00:00 2001 From: adrianciu Date: Fri, 29 Nov 2024 10:55:36 +0100 Subject: [PATCH] TVB-2113: add bin path to modify distribution python path; updated the documentation --- tvb_build/Jenkinsfile-Mac | 12 ++++++ tvb_build/Jenkinsfile-Step1 | 20 ++++++++++ tvb_build/Jenkinsfile-Windows | 8 ++++ tvb_build/build_from_conda.py | 31 +++++++++++++-- tvb_build/docker/Dockerfile-build | 2 +- tvb_build/setup_mac.py | 2 +- .../packages_accepted.xml | 38 +++++++++---------- .../UserGuide/UserGuide-Installation.rst | 24 ++++++++++++ 8 files changed, 112 insertions(+), 25 deletions(-) diff --git a/tvb_build/Jenkinsfile-Mac b/tvb_build/Jenkinsfile-Mac index 79c98ba5a8..da5618c46d 100644 --- a/tvb_build/Jenkinsfile-Mac +++ b/tvb_build/Jenkinsfile-Mac @@ -60,6 +60,18 @@ pipeline { archiveArtifacts artifacts: 'TVB_Mac*.zip' } } + stage ('Tests on SqLite') { + steps { + sh '''#!/bin/bash + source /Applications/anaconda3/etc/profile.d/conda.sh + conda activate mac-distribution + cd tvb_bin + /bin/bash run_tests.sh + exit 0 + ''' + junit 'tvb_bin/TEST_OUTPUT/results_*.xml' + } + } } post { changed { diff --git a/tvb_build/Jenkinsfile-Step1 b/tvb_build/Jenkinsfile-Step1 index 3f6921fbe1..c8b7ed7126 100644 --- a/tvb_build/Jenkinsfile-Step1 +++ b/tvb_build/Jenkinsfile-Step1 @@ -50,6 +50,26 @@ pipeline { archiveArtifacts artifacts: 'dist/*' } } + stage ('Compute coverage') { + steps { + withDockerContainer(image: '${FULL_DOCKER_IMAGE_NAME}') { + sh '''#!/bin/bash + source activate tvb-run + rm -rf TEST_OUTPUT + mkdir TEST_OUTPUT + cd tvb_build + bash install_full_tvb.sh + cd ../tvb_library + py.test --cov-config .coveragerc --cov=tvb tvb/tests --cov-branch --cov-report xml:../TEST_OUTPUT/coverage_lib.xml --junitxml=../TEST_OUTPUT/results_lib.xml + cd ../tvb_framework + py.test --cov-config .coveragerc --cov=tvb tvb/tests --cov-branch --cov-report xml:../TEST_OUTPUT/coverage_frw.xml --junitxml=../TEST_OUTPUT/results_frw.xml + exit 0 + ''' + } + junit 'TEST_OUTPUT/results_*.xml' + step([$class: 'CoberturaPublisher', autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: 'TEST_OUTPUT/coverage_*.xml', failUnhealthy: false, failUnstable: false, maxNumberOfBuilds: 0, onlyStable: false, sourceEncoding: 'ASCII', zoomCoverageChart: false]) + } + } } post { always { diff --git a/tvb_build/Jenkinsfile-Windows b/tvb_build/Jenkinsfile-Windows index e67f0f0563..0bc8fb5cee 100644 --- a/tvb_build/Jenkinsfile-Windows +++ b/tvb_build/Jenkinsfile-Windows @@ -16,6 +16,14 @@ pipeline { } } } + stage ('Tests on SqLite') { + steps { + withDockerContainer(image: "${FULL_DOCKER_IMAGE_NAME}") { + powershell 'cd tvb_build; cmd /k "activate tvb-run & install_full_tvb.bat & cd ../tvb_bin & run_tests.bat"; exit 0' + } + junit 'tvb_bin/TEST_OUTPUT/results_*.xml' + } + } stage('Copy Step 1') { steps { bat ''' diff --git a/tvb_build/build_from_conda.py b/tvb_build/build_from_conda.py index 573862f5b6..f247e5d474 100644 --- a/tvb_build/build_from_conda.py +++ b/tvb_build/build_from_conda.py @@ -35,7 +35,7 @@ class Config: - def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, commands_map, command_factory): + def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, commands_map, command_factory, bin_path=None): # System paths: self.anaconda_env_path = anaconda_env_path @@ -69,6 +69,10 @@ def __init__(self, platform_name, anaconda_env_path, site_packages_suffix, comma _artifact_glob = "TVB_" + platform_name + "_*.zip" self.artifact_glob = join(self.build_folder, _artifact_glob) # this is used to match old artifacts self.artifact_pth = join(self.build_folder, self.artifact_name) + if bin_path: + self.bin_path = join(self.target_library_root, bin_path) + else: + self.bin_path = None @staticmethod def win64(): @@ -86,7 +90,7 @@ def win64(): 'bin\\tvb_stop.bat': 'distribution stop', 'bin\\jupyter_notebook.bat': set_path + 'cd ..\\bin\n..\\tvb_data\\Scripts\\jupyter lab ..\\demo_scripts', 'demo_scripts\\jupyter_notebook.bat': set_path + 'cd ..\\demo_scripts\n..\\tvb_data\\Scripts\\jupyter lab', - 'bin\\activate_tvb_env.bat':'REM Conda must be installed before running this script \n conda activate ../tvb_data \n cmd /K' + 'bin\\activate_tvb_env.bat':'REM Conda must be installed before running this script\nconda activate ../tvb_data\ncmd /K' } return Config("Windows", "C:\\miniconda\\envs\\tvb-run", @@ -116,11 +120,11 @@ def linux64(): 'bin/tvb_stop.sh': 'bash ./distribution.sh stop', 'bin/jupyter_notebook.sh': set_path + 'cd ../bin\n../tvb_data/bin/python -m jupyterlab ../demo_scripts', 'demo_scripts/jupyter_notebook.sh': set_path + 'cd ../demo_scripts\n../tvb_data/bin/python -m jupyterlab', - 'bin/activate_tvb_env.sh': '# Conda must be installed before running this script \n conda activate ../tvb_data \n $SHELL' + 'bin/activate_tvb_env.sh': '# Conda must be installed before running this script;\n# Run this script with source.\nconda activate ../tvb_data \n' } return Config("Linux", "/opt/conda/envs/tvb-run", join("lib", Environment.PYTHON_FOLDER, "site-packages"), - commands_map, _create_unix_command) + commands_map, _create_unix_command, "bin") def _log(indent, msg): @@ -203,7 +207,24 @@ def _create_windows_script(target_file, command): os.chmod(target_file, 0o755) +def _replace_first_line_if_pattern(pathname: str, pattern: str, replacement: str): + """ + Replaces the first line of a file with a given string if the pathname contains a specific pattern. + """ + for filename in os.listdir(pathname): + file_path = os.path.join(pathname, filename) + if pattern in file_path: + with open(file_path, 'r') as file: + lines = file.readlines() + if lines: + lines[0] = replacement + '\n' + with open(file_path, 'w') as file: + file.writelines(lines) + _log(1, f"First line of {file_path} replaced with: {replacement}") + + def _modify_pth(pth_name): + # Log if one of the files pip, pip3 or pip3.11 are missing, but do not stop the execution if it is missing """ Replace tvb links with paths """ @@ -280,6 +301,8 @@ def prepare_anaconda_dist(config): _log(1, "Modifying PTH " + config.easy_install_pth) _modify_pth(config.easy_install_pth) + if config.bin_path: + _replace_first_line_if_pattern(config.bin_path, 'bin/pip', '#!../tvb_data/bin/python') _fix_jupyter_kernel(config.target_library_root, config.platform_name == "Windows") _log(1, "Creating command files:") diff --git a/tvb_build/docker/Dockerfile-build b/tvb_build/docker/Dockerfile-build index ed98fe37e7..b4acca0a5d 100644 --- a/tvb_build/docker/Dockerfile-build +++ b/tvb_build/docker/Dockerfile-build @@ -18,7 +18,7 @@ RUN service postgresql start && createdb -O postgres tvb-test && psql --command USER root RUN conda update -n base -c defaults conda; conda init bash -RUN conda create -y --name tvb-run python=3.11 nomkl numba scipy numpy cython psycopg2 +RUN conda create -y --name tvb-run python=3.11 pip nomkl numba scipy numpy cython psycopg2 RUN conda install -y --name tvb-run -c conda-forge jupyterlab tvb-gdist RUN /opt/conda/envs/tvb-run/bin/pip install --upgrade pip RUN /opt/conda/envs/tvb-run/bin/pip install lockfile scikit-build diff --git a/tvb_build/setup_mac.py b/tvb_build/setup_mac.py index 025df24351..c78fc9427c 100644 --- a/tvb_build/setup_mac.py +++ b/tvb_build/setup_mac.py @@ -257,7 +257,7 @@ def prepare_mac_dist(): _create_command_file(os.path.join(DIST_FOLDER, "bin", 'tvb_stop'), 'source ./distribution.command stop', 'Stopping TVB related processes.', True) _create_command_file(os.path.join(DIST_FOLDER, "bin", 'activate_tvb_env'), - 'conda activate ../tvb_data \n $SHELL', 'Conda must be installed before running this script', True) + 'conda activate ../tvb_data \n', 'Conda must be installed before running this script', True) jupyter_command = '/Applications/{}/Contents/Resources/bin/jupyter lab '.format(APP) _create_command_file(os.path.join(DIST_FOLDER, "bin", 'jupyter_notebook'), jupyter_command + '../demo_scripts', 'Launching IPython Notebook from TVB Distribution') diff --git a/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml b/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml index a87813c5e1..62f5f7ac99 100644 --- a/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml +++ b/tvb_build/tvb_build/third_party_licenses/packages_accepted.xml @@ -237,7 +237,7 @@ - + @@ -1197,7 +1197,7 @@ - + @@ -1637,7 +1637,7 @@ - + @@ -1699,7 +1699,7 @@ - + @@ -2053,7 +2053,7 @@ - + @@ -2214,7 +2214,7 @@ - + @@ -2602,7 +2602,7 @@ - + @@ -2674,7 +2674,7 @@ - + @@ -3064,7 +3064,7 @@ - + @@ -3074,7 +3074,7 @@ - + @@ -3084,7 +3084,7 @@ - + @@ -3434,7 +3434,7 @@ - + @@ -3505,7 +3505,7 @@ - + @@ -3708,7 +3708,7 @@ - + @@ -4433,7 +4433,7 @@ - + @@ -4739,7 +4739,7 @@ - + @@ -5476,7 +5476,7 @@ - + @@ -5517,7 +5517,7 @@ - + @@ -6035,7 +6035,7 @@ - + diff --git a/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst b/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst index 098aa35981..d948831560 100644 --- a/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst +++ b/tvb_documentation/manuals/UserGuide/UserGuide-Installation.rst @@ -45,6 +45,7 @@ Launching the application In the TVB_Distribution folder you should find a sub-folder `bin` with a number of scripts: +- activate_tvb_env - tvb_start - tvb_clean - tvb_stop @@ -137,6 +138,29 @@ A Python shell will appear. $ ./distribution.sh start COMMAND_PROFILE +Activate the conda environment +.............................. + +On Windows, from the terminal, you can run the conda environment activation script or you can double click on the script + +.. code-block:: bash + + $ activate_tvb_env.bat + +On Linux, from the terminal, you can run the conda environment with source: + +.. code-block:: bash + + $ source activate_tvb_env.sh + +On Mac, from the terminal, you can run the conda environment with source: + +.. code-block:: bash + + $ source activate_tvb_env.command + +After activating the Conda environment, you can modify the currently installed Python packages. + Configuring TVB ---------------