added new module data_tools to handle v2 ingestion/extraction #641
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| push: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| jobs: | |
| release: | |
| if: github.ref == 'refs/heads/master' | |
| runs-on: self-hosted | |
| timeout-minutes: 6 | |
| environment: | |
| name: release | |
| env: | |
| GIT_COMMITTER_NAME: "sig-autobot" | |
| GIT_COMMITTER_EMAIL: "[email protected]" | |
| steps: | |
| - id: generate_token | |
| uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 | |
| with: | |
| app_id: ${{ secrets.SIGTECH_PYTHON_APP_ID }} | |
| private_key: ${{ secrets.SIGTECH_PYTHON_PRIVATE_KEY }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ steps.generate_token.outputs.token }} | |
| - name: "Check commit messages match angular style" | |
| run: | | |
| ! git log origin/master..HEAD --pretty=format:%s | \ | |
| grep -v -E '^(build|chore|ci|docs|feat|fix|perf|style|refactor|test):' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| cache: 'pip' | |
| cache-dependency-path: '.lockfiles/tools.txt' | |
| - run: pip install --no-dependencies -r .lockfiles/tools.txt | |
| - run: make release | |
| - run: echo "RELEASE_SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT | |
| id: release_sha | |
| outputs: | |
| release_sha: ${{ steps.release_sha.outputs.RELEASE_SHA }} | |
| build: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 6 | |
| if: ${{ always() && !cancelled() && !contains(needs.*.result, 'failure') }} | |
| needs: [release] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| ref: '${{ needs.release.outputs.release_sha || github.event.pull_request.head.sha }}' | |
| - name: "Check commit messages match angular style" | |
| run: | | |
| ! git log origin/master..HEAD --pretty=format:%s | \ | |
| grep -v -E '^(build|chore|ci|docs|feat|fix|perf|style|refactor|test):' | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| cache: 'pip' | |
| cache-dependency-path: '.lockfiles/tools.txt' | |
| - run: pip install --no-dependencies -r .lockfiles/tools.txt | |
| - run: make lint | |
| - run: make check | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: tests | |
| path: | | |
| tests/ | |
| examples/ | |
| README.md | |
| - run: rm -rf examples scripts tests | |
| - run: make build | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: | | |
| dist/sigtech-*.whl | |
| dist/sigtech-*.tar.gz | |
| - run: echo "SOURCE_FILENAME=$(basename sigtech-*.tar.gz)" >> $GITHUB_OUTPUT | |
| id: source_filename | |
| working-directory: dist | |
| - run: echo "WHEEL_FILENAME=$(basename sigtech-*.whl)" >> $GITHUB_OUTPUT | |
| id: wheel_filename | |
| working-directory: dist | |
| outputs: | |
| source_filename: ${{ steps.source_filename.outputs.SOURCE_FILENAME }} | |
| wheel_filename: ${{ steps.wheel_filename.outputs.WHEEL_FILENAME }} | |
| test: | |
| needs: [build] | |
| if: ${{ always() && !cancelled() && !contains(needs.*.result, 'failure') }} | |
| env: | |
| SIGTECH_API_KEY: ${{ secrets.SIGTECH_API_KEY }} | |
| SIGTECH_PLATFORM_TOKEN: ${{ secrets.SIGTECH_PLATFORM_TOKEN }} | |
| FRED_API_KEY: ${{ secrets.FRED_API_KEY }} | |
| strategy: | |
| matrix: | |
| # https://devguide.python.org/versions/ | |
| include: | |
| - os: 'ubuntu-20.04' | |
| python: '3.8' | |
| pip: 'pandas==1.4.1 numpy==1.24.4' | |
| timeout: 6 | |
| - os: 'ubuntu-22.04' | |
| python: '3.9' | |
| pip: >- | |
| pandas==1.4.1 | |
| numpy==1.26.4 | |
| matplotlib==3.7.5 | |
| timeout: 6 | |
| - os: 'ubuntu-22.04' | |
| python: '3.9' | |
| pip: 'pandas==2.0.3 numpy==1.25.0' | |
| timeout: 6 | |
| - os: 'ubuntu-22.04' | |
| python: '3.10' | |
| pip: 'pandas==2.0.3 numpy==1.25.0' | |
| timeout: 6 | |
| - os: 'ubuntu-22.04' | |
| python: '3.10' | |
| # Google Colab | |
| pip: >- | |
| charset-normalizer==3.3.2 | |
| matplotlib==3.7.1 | |
| matplotlib-inline==0.1.7 | |
| urllib3==2.0.7 | |
| six==1.16.0 | |
| python-dateutil==2.8.2 | |
| requests==2.31.0 | |
| numpy==1.25.2 | |
| pandas==2.0.3 | |
| timeout: 6 | |
| - os: 'ubuntu-22.04' | |
| python: '3.11' | |
| pip: 'pandas==2.0.3 numpy==1.26.4' | |
| timeout: 12 | |
| - os: 'ubuntu-24.04' | |
| python: '3.12' | |
| pip: 'pandas==2.2.0 numpy==1.26.4' | |
| timeout: 12 | |
| # latest os/pandas | |
| - os: 'ubuntu-latest' | |
| python: '3.12' | |
| pip: 'pandas' | |
| timeout: 12 | |
| - os: 'windows-latest' | |
| python: '3.12' | |
| pip: 'pandas' | |
| timeout: 12 | |
| - os: 'macos-latest' | |
| python: '3.12' | |
| pip: 'pandas' | |
| timeout: 12 | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: ${{ matrix.timeout }} | |
| steps: | |
| - if: ${{ matrix.os == 'macos-latest' }} | |
| run: "brew install libomp" | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '${{ matrix.python }}' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| - run: 'pip install ${{ matrix.pip }}' | |
| - working-directory: dist | |
| run: "pip install ${{needs.build.outputs.wheel_filename}}[test]" | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: tests | |
| path: tests | |
| - run: pytest --full-trace . | |
| working-directory: tests | |
| pypi-publish: | |
| if: ${{ always() && !cancelled() && !contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }} | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| needs: [ test ] | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/project/sigtech/ | |
| permissions: | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/[email protected] | |
| with: | |
| packages-dir: dist/ | |
| verbose: true | |
| print-hash: true | |
| pypi-test: | |
| if: ${{ always() && !cancelled() && !contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: [ pypi-publish ] | |
| env: | |
| SIGTECH_API_KEY: ${{ secrets.SIGTECH_API_KEY }} | |
| SIGTECH_PLATFORM_TOKEN: ${{ secrets.SIGTECH_PLATFORM_TOKEN }} | |
| steps: | |
| - run: pip install sigtech | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: tests | |
| path: tests | |
| - run: | | |
| for file in *.py; do | |
| python "$file" | |
| done | |
| working-directory: tests/examples/scripts |