CI.yml

name: CI

on: [pull_request, merge_group, workflow_dispatch]

env:
  ARKOUDA_QUICK_COMPILE: true

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Check for tabs
      run: |
        ! git --no-pager grep -n $'\t' -- '*.chpl'
        
  mypy:
    runs-on: ubuntu-latest
    container:
      image: chapel/chapel:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      run: |
        apt-get update && apt-get install -y python3-pip libhdf5-dev hdf5-tools libzmq3-dev libcurl4-openssl-dev
        python3 -m pip install types-pkg_resources versioneer
        python3 -m pip install -e .[dev]
    - name: Arkouda mypy
      run: |
        python3 -m mypy arkouda

  docs:
    runs-on: ubuntu-latest
    container:
      image: chapel/chapel:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      run: |
        apt-get update && apt-get install -y python3-pip libhdf5-dev hdf5-tools libzmq3-dev libcurl4-openssl-dev
        python3 -m pip install versioneer
        python3 -m pip install -e .[dev]
    - name: Arkouda make doc
      run: |
        make doc

  flake8:
    runs-on: ubuntu-latest
    container:
      image: chapel/chapel:2.1.0
    steps:
      - uses: actions/checkout@v3
      - name: Install Dependencies
        uses: nick-fields/retry@v2
        with:
          timeout_seconds: 1200  # or use timeout_minutes
          max_attempts: 2
          retry_wait_seconds: 60
          retry_on: error
          command: |
            apt-get update && apt-get install -y python3-pip libhdf5-dev hdf5-tools libzmq3-dev libcurl4-openssl-dev
            python3 -m pip install types-pkg_resources versioneer
            python3 -m pip install -e .[dev]
      - name: Arkouda flake8
        run: |
          flake8 arkouda          

  arkouda_python_portability:
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        python-version: ['3.9', '3.10', '3.11']
    container:
      image: chapel/chapel:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.python-version}}
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |   
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget
          make install-arrow-quick
          apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
          make install-iconv
          echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
    - name: Check python version
      run: |
        python3 --version
    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Build/Install Arkouda
      run: |
        make
        python3 -m pip install -e .[dev]
    - name: Arkouda unit tests
      run: |
        make test-python size=100

  arkouda_makefile_almalinux:
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        python-version: ['3.9', '3.10', '3.11']
    container:
      image: ajpotts/almalinux-with-arkouda-deps:1.0.0
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.python-version}}
    - name: Check python version
      run: |
        python3 --version
    - name: Make install-arrow
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-arrow        
    - name: Make install-hdf5
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |  
          make install-hdf5
    - name: Make install-zmq
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: | 
          make install-zmq        
    - name: Make install-iconv
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: | 
          make install-iconv
    - name: Make install-idn2
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-idn2        
    - name: Make install-blosc
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-blosc     


  arkouda_makefile:
    runs-on: ubuntu-20.04
    strategy:
      matrix:
        python-version: ['3.9', '3.10', '3.11']
    container:
      image: chapel/chapel:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{matrix.python-version}}
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget build-essential cmake
          apt-get update && apt-get install -y -V python3-pip libcurl4-openssl-dev
          apt-get update -y && apt-get -y -V upgrade 
    - name: Check python version
      run: |
        python3 --version

    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Make install-arrow-quick
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-arrow-quick
    - name: Make install-arrow
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
            apt-get remove -y apache-arrow-apt-source
            make arrow-clean
            make install-arrow    
    - name: Make install-hdf5
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-hdf5
    - name: Make install-zmq
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-zmq        
    - name: Make install-iconv
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: | 
          make install-iconv
    - name: Make install-idn2
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-idn2        
    - name: Make install-blosc
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          make install-blosc      
  

  arkouda_chpl_portability:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        chpl-version: ['2.0.0','2.1.0','2.2.0']
    container:
      image: chapel/chapel:${{matrix.chpl-version}}
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget
          make install-arrow-quick
          apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
          make install-iconv
        
          echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
    - name: Check chpl version
      run: |
        chpl --version
    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Build/Install Arkouda
      run: |
        make
        python3 -m pip install -e .[dev]
    - name: Arkouda unit tests
      run: |
        make test-python size=100

  arkouda_multi-dim:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        chpl-version: ['2.0.0']
    container:
      image: chapel/chapel:${{matrix.chpl-version}}
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget
          make install-arrow-quick
          apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
          make install-iconv
          echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
    - name: Check chpl version
      run: |
        chpl --version
    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Use MultiDim Configs
      run: |
        cp .configs/registration-config-multi-dim.json registration-config.json
    - name: Build/Install Arkouda
      run: |
        make
        python3 -m pip install -e .[dev]
    - name: Arkouda unit tests
      run: |
        make test-python size=100

  arkouda_tests_linux:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - image: chapel
            threads: 2
#          - image: chapel-gasnet-smp
#            threads: 1
    env:
      CHPL_RT_NUM_THREADS_PER_LOCALE: ${{matrix.threads}}
    container:
      image: chapel/${{matrix.image}}:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget
          make install-arrow-quick
          apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev
          make install-iconv
          echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Build/Install Arkouda
      run: |
        make
        python3 -m pip install -e .[dev]
    - name: Arkouda make check
      run: |
        make check
    - name: Arkouda unit tests
      if: matrix.image != 'chapel-gasnet-smp'
      env:
        ARKOUDA_PYTEST_OPTIONS: "--durations=0 --durations-min=5.0"
      run: |
        make test-python size=100
    - name: Arkouda benchmark --correctness-only
      if: matrix.image != 'chapel-gasnet-smp'
      run: |
        ./benchmarks/run_benchmarks.py --correctness-only
        ./benchmarks/run_benchmarks.py --size=100 --gen-graphs
        

  arkouda_benchmark_linux:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          - image: chapel
            threads: 2
#          - image: chapel-gasnet-smp
#            threads: 1
    env:
      CHPL_RT_NUM_THREADS_PER_LOCALE: ${{matrix.threads}}
    container:
      image: chapel/${{matrix.image}}:2.1.0
    steps:
    - uses: actions/checkout@v3
    - name: Install dependencies
      uses: nick-fields/retry@v2
      with:
        timeout_seconds: 1200  # or use timeout_minutes
        max_attempts: 2
        retry_wait_seconds: 60
        retry_on: error
        command: |
          apt-get update && apt-get install -y -V ca-certificates lsb-release wget
          make install-arrow-quick
          apt-get update && apt-get install -y libhdf5-dev hdf5-tools libzmq3-dev python3-pip libarrow-dev libparquet-dev libcurl4-openssl-dev libidn2-dev 
          pip install pytest-benchmark==4.0.0
          make install-iconv
          echo "\$(eval \$(call add-path,/usr/lib/x86_64-linux-gnu/hdf5/serial/))" >> Makefile.paths
          mkdir -p benchmark_v2/data
    - name: Install Chapel frontend bindings
      run: |
        (cd $CHPL_HOME/tools/chapel-py && python3 -m pip install .)
    - name: Build/Install Arkouda
      run: |
        make
        python3 -m pip install -e .[dev]
    - name: Arkouda make check
      run: |
        make check
    - name: Arkouda benchmark
      run: |
        make benchmark size_bm=10