Add a fetcher for AWS S3 Argo GDAC data #1202
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI tests | |
| # CI tests ran every time new code is pushed to the repo | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches-ignore: | |
| - 'dependabot*' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| detect-ci-skip: | |
| name: detect ci skip | |
| runs-on: ubuntu-latest | |
| if: | | |
| github.repository == 'euroargodev/argopy' | |
| && (github.event_name == 'push' || github.event_name == 'pull_request') | |
| outputs: | |
| skipped: ${{ steps.detect-skip.outputs.trigger-found }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - uses: xarray-contrib/[email protected] | |
| id: detect-skip | |
| with: | |
| keyword: "[skip-ci]" | |
| core-pinned: | |
| # CI tests for environments with core requirements in pinned versions | |
| # All commits must pass this | |
| name: Core - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| needs: detect-ci-skip | |
| runs-on: ${{ matrix.os }} | |
| if: needs.detect-ci-skip.outputs.skipped == 'false' | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| continue-on-error: ${{ matrix.experimental }} | |
| # timeout-minutes: 45 | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| os: ["ubuntu-latest", "windows-latest", "macos-latest"] | |
| experimental: [false] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-core-pinned.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-Core-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup core-pinned' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest core-pinned' | |
| continue-on-error: true | |
| - name: Upload coverage to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./cov.xml | |
| flags: unittests, core, pinned, py${{matrix.python-version}} | |
| name: codecov-github | |
| fail_ci_if_error: false | |
| env_vars: RUNNER_OS,PYTHON_VERSION | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Save tests log as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-Core-${{matrix.python-version}}-${{matrix.os}} | |
| path: ${{env.LOG_FILE}} | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true | |
| all-pinned: | |
| # CI tests for environments with all requirements in pinned versions | |
| # All commits must pass this | |
| name: All - Pinned - Py${{matrix.python-version}} - ${{ matrix.os }} | |
| needs: detect-ci-skip | |
| runs-on: ${{ matrix.os }} | |
| if: needs.detect-ci-skip.outputs.skipped == 'false' | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| continue-on-error: ${{ matrix.experimental }} | |
| # timeout-minutes: 45 | |
| strategy: | |
| max-parallel: 12 | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| experimental: [false] | |
| steps: | |
| - name: Energy Estimation - Initialize | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| with: | |
| task: start-measurement | |
| company-uuid: ${{ secrets.CARBONDB_COMPANY_UUID }} | |
| project-uuid: ${{ secrets.CARBONDB_PROJECT_UUID }} | |
| machine-uuid: ${{ secrets.CARBONDB_MACHINE_UUID_PYTESTS }} | |
| continue-on-error: true | |
| - uses: actions/checkout@v4 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: us-west-1 | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/ga-ci-tests-argopy-01 | |
| - name: Set environment variables | |
| run: | | |
| echo "CONDA_ENV_FILE=ci/requirements/py${{matrix.python-version}}-all-pinned.yml" >> $GITHUB_ENV | |
| echo "PYTHON_VERSION=${{matrix.python-version}}" >> $GITHUB_ENV | |
| echo "LOG_FILE=argopy-tests-All-Pinned-Py${{matrix.python-version}}-${{matrix.os}}.log" >> $GITHUB_ENV | |
| - name: Setup Micromamba ${{ matrix.python-version }} | |
| uses: mamba-org/setup-micromamba@v2 | |
| with: | |
| micromamba-version: '1.5.10-0' | |
| environment-name: argopy-tests | |
| environment-file: ${{ env.CONDA_ENV_FILE }} | |
| init-shell: bash | |
| cache-environment: true | |
| cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}" | |
| create-args: >- | |
| python=${{matrix.python-version}} | |
| - name: Install argopy | |
| run: | | |
| python -m pip install --no-deps -e . | |
| - name: Version info | |
| run: | | |
| micromamba info | |
| micromamba list | |
| - name: Energy Estimation - Measure Tests Setup | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'setup all-pinned' | |
| continue-on-error: true | |
| - name: Lint with flake8 | |
| run: | | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Test with pytest | |
| run: | | |
| pytest -ra -v -s -c argopy/tests/pytest.ini --durations=10 --cov=./ --cov-config=.coveragerc --cov-report xml:cov.xml --cov-report term-missing --log-file=${{env.LOG_FILE}} --junitxml=junit.xml -o junit_family=legacy | |
| - name: Energy Estimation - Measure Tests Exec | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: get-measurement | |
| label: 'pytest all-pinned' | |
| continue-on-error: true | |
| - name: Upload coverage to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| file: ./cov.xml | |
| flags: unittests, all, pinned, py${{matrix.python-version}} | |
| name: codecov-github | |
| fail_ci_if_error: false | |
| env_vars: RUNNER_OS,PYTHON_VERSION | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload tests log to GA | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Argopy-Tests-All-${{matrix.python-version}}-${{matrix.os}} | |
| path: ${{env.LOG_FILE}} | |
| retention-days: 90 | |
| - name: Energy Estimation - Show Results | |
| if: ${{matrix.os == 'ubuntu-latest'}} | |
| uses: green-coding-solutions/[email protected] | |
| env: | |
| ELECTRICITY_MAPS_TOKEN: ${{ secrets.ELECTRICITY_MAPS_TOKEN }} | |
| with: | |
| task: display-results | |
| send-data: true | |
| continue-on-error: true |