-
Notifications
You must be signed in to change notification settings - Fork 48
287 lines (255 loc) · 15.5 KB
/
generate-conda-packages.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
name: Generate conda packages
# This action automatically generate conda packages for the packages in the robotology-superbuild
on:
workflow_dispatch:
inputs:
upload_conda_binaries:
description: 'If true, upload conda binaries, for any other value only generate them without uploading.'
required: true
default: 'true'
test_metapackages_generation:
description: 'If true, also test metapackages generation. Not compatible with upload_conda_binaries set to true.'
required: true
default: 'false'
schedule:
# Run the job once a week
- cron: '0 0 * * 2'
release:
types: [published]
jobs:
# Regardless of the branch on which this action is trigged,
# the CONDA_BUILD_NUMBER CMake option needs to be read from the
# master branch, to avoid that a conda package generation from the
# master branch and one from a releases/YYYY.MM branch use the same
# CONDA_BUILD_NUMBER
get-conda-build-number:
name: "Read Conda Build number from master branch"
runs-on: ubuntu-latest
# Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job)
outputs:
conda_build_number: ${{ steps.step1.outputs.conda_build_number }}
steps:
- uses: actions/checkout@v4
with:
ref: 'master'
- id: step1
name: Get CONDA_BUILD_NUMBER and set it as output
shell: bash
run: |
# Get CONDA_BUILD_NUMBER via grep and set it to an environment variable
export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'`
echo "conda_build_number=${CONDA_BUILD_NUMBER}" >> $GITHUB_OUTPUT
generate-conda-packages:
name: "Generate conda packages @${{ matrix.os }}"
runs-on: ${{ matrix.os }}
needs: get-conda-build-number
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-20.04
conda_platform: linux-64
- os: macos-11.0
conda_platform: osx-64
- os: windows-2019
conda_platform: win-64
steps:
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
with:
miniforge-variant: Miniforge3
miniforge-version: latest
- name: Install files to enable compilation of mex files [Conda/Linux]
if: contains(matrix.os, 'ubuntu')
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020b_mexa64.zip
unzip msdk_R2020b_mexa64.zip
rm msdk_R2020b_mexa64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020b_mexa64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexa64" >> $GITHUB_ENV
- name: Install files to enable compilation of mex files [Conda/macOS]
if: contains(matrix.os, 'macos')
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexmaci64.zip
unzip msdk_R2020a_mexmaci64.zip
rm msdk_R2020a_mexmaci64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexmaci64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexmaci64" >> $GITHUB_ENV
- name: Install files to enable compilation of mex files [Conda/Windows]
if: contains(matrix.os, 'windows')
shell: bash
run: |
curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexw64.zip
unzip msdk_R2020a_mexw64.zip
rm msdk_R2020a_mexw64.zip
echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexw64" >> $GITHUB_ENV
echo "GHA_Matlab_MEX_EXTENSION=mexw64" >> $GITHUB_ENV
- name: Dependencies for conda recipes generation and upload
shell: bash -l {0}
run: |
conda install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller
- name: Print used environment
shell: bash -l {0}
run: |
conda list
env
- name: Generate recipes [Linux&macOS]
if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu')
shell: bash -l {0}
run: |
mkdir build
cd build
cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
- name: Generate recipes [Windows]
if: contains(matrix.os, 'windows')
shell: bash -l {0}
run: |
mkdir build
cd build
cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} ..
- name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages
if: github.event_name == 'release'
shell: bash -l {0}
run: |
cd build
cmake -DROBOTOLOGY_PROJECT_TAGS=Custom -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
- name: Specify additional option if we want generate robotology-distro metapackages
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true'
shell: bash -l {0}
run: |
cd build
cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON .
- name: Fail if both upload_conda_binaries and test_metapackages_generation are passed
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true'
uses: actions/github-script@v7
with:
script: |
core.setFailed('Build failed. It is not possible to set to true both upload_conda_binaries and test_metapackages_generation.')
- name: If test_metapackages_generation is enabled set an arbitrary version number
if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true'
shell: bash -l {0}
run: |
sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION 1234.0)/g' ./packaging/windows/CMakeLists.txt
sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION 1234.0)/g' ./conda/cmake/CondaGenerationOptions.cmake
- name: Set CONDA_BLD_PATH on Windows to avoid long path issues [Windows]
if: contains(matrix.os, 'windows')
shell: bash -l {0}
run: |
echo "CONDA_BLD_PATH=C:\\bld" >> $GITHUB_ENV
- name: Set CONDA_BLD_PATH on non-Windows for consistency [non Windows]
if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu')
shell: bash -l {0}
run: |
echo "CONDA_BLD_PATH=${CONDA_PREFIX}/conda-bld" >> $GITHUB_ENV
- name: Build conda packages
shell: bash -l {0}
run: |
conda config --remove channels defaults
conda config --env --set channel_priority strict
cd build/conda/generated_recipes
# Debug generated recipes
cat */meta.yaml
cat */bld.bat
cat */build.sh
# We use the conda_build_config from conda-forge-pinning, and our local one
# See https://docs.conda.io/projects/conda-build/en/latest/resources/variants.html#creating-conda-build-variant-config-files
# We manually specify the build order as an old workaround for mambabuild, in theory this can be avoided, but it needs to be tested
# see https://github.com/mamba-org/boa/issues/117
# We also manually delete the recipe directory to avoid that it is built again,
# specifically because we can't use --skip-existing as these packages are part of the robotology channel
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-contrib-common
rm -rf icub-contrib-common
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-firmware-shared
rm -rf icub-firmware-shared
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robots-configuration
rm -rf robots-configuration
# icub-main needs robotology channel as it also depends on esdcan, and local to avoid that robotology packages are used instead of local built one
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology icub-main
rm -rf icub-main
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wearables
rm -rf wearables
# Channels specified explicitly due to https://github.com/robotology/robotology-superbuild/issues/1140
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge human-dynamics-estimation
rm -rf human-dynamics-estimation
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer
rm -rf robot-log-visualizer
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml idyntree-matlab-bindings
rm -rf idyntree-matlab-bindings
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blockfactory
rm -rf blockfactory
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp-matlab-bindings
rm -rf yarp-matlab-bindings
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wb-toolbox
rm -rf wb-toolbox
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blocktest
rm -rf blocktest
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml .
- name: Build robotology-distro conda metapackage (if necessary)
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true')
shell: bash -l {0}
run: |
cd build/conda/generated_recipes_metapackages
# Debug generated recipes
cat */meta.yaml
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro
- name: Upload conda packages
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/
ls *.tar.bz2
anaconda upload --skip-existing *.tar.bz2
# Only on linux-64 we upload noarch packages (noarch packages need to be uploaded only once, as they can be reused across platforms)
- name: Upload conda packages (noarch)
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: matrix.conda_platform == 'linux-64' && (github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true'))
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/noarch/
ls *.tar.bz2
anaconda upload --skip-existing *.tar.bz2
- name: Build robotology-distro-all conda metapackage (if necessary)
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true')
shell: bash -l {0}
run: |
cd build/conda/generated_recipes_metapackages
# Debug generated recipes
cat */meta.yaml
conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all
- name: Upload robotology-distro-all metapackage
shell: bash -l {0}
# Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true'
if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true')
env:
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
run: |
cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/
ls robotology-distro-all-*.tar.bz2
anaconda upload --skip-existing robotology-distro-all-*.tar.bz2
# If the generate-conda-packages completed correctly and binaries are uploaded,
# bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake
# of master branch for future builds
# the master branch is always used in case the action is triggered by a release on
# a release/vYYYY.MM branch
bump-conda-build-number:
name: "Bump Conda Build number for future builds"
runs-on: ubuntu-latest
needs: generate-conda-packages
if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')
steps:
- uses: actions/checkout@v4
with:
ref: 'master'
- name: Bump Conda Build number for future builds
shell: bash
run: |
sh ./scripts/robotologyBumpCondaBuildNumber.sh
- uses: EndBug/[email protected]
with:
default_author: github_actions
message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload'