Skip to content

Commit

Permalink
Merge pull request #18 from CMCC-Foundation/add_method_to_image
Browse files Browse the repository at this point in the history
merging branche add_method_to_image to branch main
  • Loading branch information
gtramonte authored Sep 26, 2024
2 parents 49979a8 + 0b8174d commit 21fbbd6
Show file tree
Hide file tree
Showing 26 changed files with 352 additions and 1,237 deletions.
41 changes: 41 additions & 0 deletions .github/workflows/build-push-docker-base.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Build geokube-base Docker image and push to geokube-base container registry

on:
pull_request:
workflow_dispatch:

jobs:
build-base:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set Docker image tag name
run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
- name: TAG ECHO
run: echo ${{ env.TAG }}
- uses: actions/setup-python@v1
with:
python-version: 3.9
- name: Install WHEEL tool
run: pip install wheel
- name: Build WHEEL file
run: python setup.py bdist_wheel
- name: Login to Scaleway Container Registry
uses: docker/login-action@v3
with:
username: nologin
password: ${{ secrets.SCALEWAY_PASSWORD }}
registry: ${{ vars.GEOKUBE_REGISTRY }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.base
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: |
${{ vars.GEOKUBE_REGISTRY }}/geokube-base:${{ env.TAG }}
${{ vars.GEOKUBE_REGISTRY }}/geokube-base:latest
16 changes: 11 additions & 5 deletions .github/workflows/build-push-docker_pr.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
name: Build geokube Docker image and push to geokube container registry

on:
pull_request:
types: [opened, synchronize]
workflow_run:
workflows: [ 'build-base' ]
types: [ completed ]
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set Docker image tag name
run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV
- name: TAG ECHO
run: echo ${{ env.TAG }}
- uses: actions/setup-python@v1
with:
python-version: 3.9
Expand All @@ -19,19 +23,21 @@ jobs:
- name: Build WHEEL file
run: python setup.py bdist_wheel
- name: Login to Scaleway Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: nologin
password: ${{ secrets.SCALEWAY_PASSWORD }}
registry: ${{ vars.GEOKUBE_REGISTRY }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v2
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: |
${{ vars.GEOKUBE_REGISTRY }}/geokube:${{ env.TAG }}
${{ vars.GEOKUBE_REGISTRY }}/geokube:latest
6 changes: 4 additions & 2 deletions .github/workflows/build-push-docker_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@ jobs:
username: nologin
password: ${{ secrets.SCALEWAY_PASSWORD }}
registry: ${{ vars.GEOKUBE_REGISTRY }}
- name: Set up Docker Buildx
uses: docker/build-push-action@v4
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
cache-from: type=gha
cache-to: type=gha,mode=max
tags: |
${{ vars.GEOKUBE_REGISTRY }}/geokube:${{ env.RELEASE_TAG }}
65 changes: 12 additions & 53 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -1,63 +1,22 @@
name: Unit Tests

on:
pull_request:
types: [opened, reopened, edited, assigned, synchronize]
workflow_run:
workflows: [ 'build' ]
types: [ completed ]
workflow_dispatch:

jobs:
build:
pytest:
runs-on: ubuntu-latest
container:
image: ${{ vars.GEOKUBE_REGISTRY }}/geokube:latest
credentials:
username: nologin
password: ${{ secrets.SCALEWAY_PASSWORD }}
steps:
- uses: actions/checkout@v2

- uses: actions/setup-python@v1
with:
python-version: 3.9

- uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
auto-activate-base: true
python-version: 3.9

- name: Conda create environemnt
shell: bash -l {0}
run: conda create --name gk

- name: Conda activate
shell: bash -l {0}
run: conda activate gk

- run: |
conda install -c conda-forge --yes --freeze-installed xesmf cartopy psycopg2 'bokeh>=2.4.2,<3' pandas=1.4.3 netCDF4 scipy xarray=2022.6.0
- run: |
conda run pip install geopandas
- run: conda run pip install pytest-github-actions-annotate-failures

- run: |
conda run python setup.py install
- run: |
conda run pip install black==22.3.0
- name: Check formatting in `tests` catalog
run: conda run black -l 79 --preview tests

- name: Check formatting in `geokube` catalog
run: conda run black -l 79 --preview geokube

- name: Check for modified files
id: git-check
run: echo ::set-output name=modified::$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi)

- name: Verify code formatting
if: steps.git-check.outputs.modified == 'true'
uses: actions/github-script@v3
with:
script: |
core.setFailed('`geokube` or `tests` directory is not well formatted! Use command `black -l 79 --preview geokube` and `black -l 79 --preview tests`')
- name: Run test
run: pytest tests

- name: Run pytest
run: conda run pytest tests
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,9 @@ _old/

# Netcdf files
*.nc
development.py
Dockerfile.dev
nuts_shape
*.png
*.zarr
.DS_Store
24 changes: 9 additions & 15 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,15 +1,9 @@
FROM continuumio/miniconda3:22.11.1
COPY environment.yaml /tmp/enviroment.yaml
RUN conda config --set allow_conda_downgrades true
RUN conda env update --name base --file /tmp/enviroment.yaml --prune
RUN conda install -c conda-forge gxx_linux-64==11.1.0
RUN conda update -n base -c defaults conda
RUN conda clean -afy \
&& find /opt/conda/ -follow -type f -name '*.a' -delete \
&& find /opt/conda/ -follow -type f -name '*.pyc' -delete \
&& find /opt/conda/ -follow -type f -name '*.js.map' -delete \
&& find /opt/conda/lib/python*/site-packages/bokeh/server/static -follow -type f -name '*.js' ! -name '*.min.js' -delete
COPY dist/geokube-0.2.6b2-py3-none-any.whl /
RUN pip install /geokube-0.2.6b2-py3-none-any.whl
RUN rm /geokube-0.2.6b2-py3-none-any.whl
ENV LD_LIBRARY_PATH=/opt/conda/x86_64-conda-linux-gnu/lib:/usr/lib/x86_64-linux-gnu
#FROM rg.fr-par.scw.cloud/geokube/geokube-base:2024.05.03.08.14
#new version with Zarr support
FROM rg.fr-par.scw.cloud/geokube/geokube-base:2024.09.26.09.01

COPY dist/geokube-0.2.7-py3-none-any.whl /
RUN pip install /geokube-0.2.7-py3-none-any.whl --break-system-packages
RUN rm /geokube-0.2.7-py3-none-any.whl

#ENV LD_LIBRARY_PATH=/opt/conda/x86_64-conda-linux-gnu/lib:/usr/lib/x86_64-linux-gnu
27 changes: 27 additions & 0 deletions Dockerfile.base
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
FROM ubuntu:24.10

RUN apt update && apt install -y build-essential git gfortran python3-pip python3 libudunits2-dev

ENV UDUNITS2_XML_PATH=/usr/share/xml/udunits/udunits2.xml

RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1

WORKDIR /opt
RUN git clone https://github.com/esmf-org/esmf.git

ENV ESMF_DIR=/opt/esmf
ENV ESMF_F90COMPILEOPTS=-fallow-argument-mismatch

WORKDIR /opt/esmf
RUN git checkout v8.6.1
RUN make -j4

ENV ESMFMKFILE=/opt/esmf/lib/libO/Linux.gfortran.64.mpiuni.default/esmf.mk

RUN pip install numpy setuptools==74.1.2 --break-system-packages

WORKDIR /opt/esmf/src/addon/esmpy
RUN pip install . --break-system-packages

RUN pip install git+https://github.com/pangeo-data/xesmf.git --break-system-packages #@v0.6.3
2 changes: 1 addition & 1 deletion environment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ dependencies:
- pandas=1.4.3
- netCDF4
- scipy
- xarray=2022.6.0
- xarray=2022.10.0
- dask=2023.1.1
- esmpy=8.2.0
- xesmf=0.6.3
Expand Down
Loading

0 comments on commit 21fbbd6

Please sign in to comment.