daily_crawl.yaml

name: Daily Crawl of Enabled Spiders

on:
  schedule:
    # Execute once a day at 6PM (BRT)
    - cron: "0 21 * * *"
  workflow_dispatch:

jobs:
  daily-crawl:
    runs-on: ubuntu-latest
    env:
      SHUB_APIKEY: ${{ secrets.SHUB_APIKEY }}
      SCRAPY_CLOUD_PROJECT_ID: ${{ secrets.SCRAPY_CLOUD_PROJECT_ID }}
      FILES_STORE: ${{ secrets.FILES_STORE }}
      QUERIDODIARIO_DATABASE_URL: ${{ secrets.QUERIDODIARIO_DATABASE_URL }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
      AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
      SPIDERMON_DISCORD_FAKE: ${{ secrets.SPIDERMON_DISCORD_FAKE }}
      SPIDERMON_DISCORD_WEBHOOK_URL: ${{ secrets.SPIDERMON_DISCORD_WEBHOOK_URL }}
      ZYTE_SMARTPROXY_APIKEY: ${{ secrets.ZYTE_SMARTPROXY_APIKEY }}
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v2
      with:
        python-version: "3.10"
    - name: Prepare environment
      run: |
        python -m pip install --upgrade pip
        pip install click python-decouple scrapinghub SQLAlchemy psycopg2
    - name: Schedule jobs
      run: |
        cd data_collection/
        python scheduler.py schedule-enabled-spiders