Skip to content

Commit

Permalink
BACKEND: Double quotes and linting for all python files (#3893)
Browse files Browse the repository at this point in the history
  • Loading branch information
@benhammondmusic
benhammondmusic authored Dec 23, 2024
1 parent e2f8772 commit 46eaa5e
Show file tree
Hide file tree
Showing 125 changed files with 4,080 additions and 4,159 deletions.
1 change: 0 additions & 1 deletion .github/linters/black.ini
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# configuration settings for black python linter when run by SuperLinter on GitHub actions CI
[tool.black]
skip-string-normalization = true
line-length = 120
82 changes: 41 additions & 41 deletions airflow/dags/acs_condition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
_ACS_DATASET_NAME = "acs_condition"

default_args = {
'start_date': days_ago(0),
'execution_timeout': timedelta(minutes=15),
"start_date": days_ago(0),
"execution_timeout": timedelta(minutes=15),
}

data_ingestion_dag = DAG(
Expand All @@ -22,122 +22,122 @@

# CACHE ACS SOURCE INTO TMP JSON IN BUCKETS

acs_condition_gcs_payload_2012 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2012')
acs_condition_gcs_payload_2012 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2012")
acs_condition_gcs_operator_2012 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2012', acs_condition_gcs_payload_2012, data_ingestion_dag
"acs_condition_to_gcs_2012", acs_condition_gcs_payload_2012, data_ingestion_dag
)

acs_condition_gcs_payload_2013 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2013')
acs_condition_gcs_payload_2013 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2013")
acs_condition_gcs_operator_2013 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2013', acs_condition_gcs_payload_2013, data_ingestion_dag
"acs_condition_to_gcs_2013", acs_condition_gcs_payload_2013, data_ingestion_dag
)

acs_condition_gcs_payload_2014 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2014')
acs_condition_gcs_payload_2014 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2014")
acs_condition_gcs_operator_2014 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2014', acs_condition_gcs_payload_2014, data_ingestion_dag
"acs_condition_to_gcs_2014", acs_condition_gcs_payload_2014, data_ingestion_dag
)

acs_condition_gcs_payload_2015 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2015')
acs_condition_gcs_payload_2015 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2015")
acs_condition_gcs_operator_2015 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2015', acs_condition_gcs_payload_2015, data_ingestion_dag
"acs_condition_to_gcs_2015", acs_condition_gcs_payload_2015, data_ingestion_dag
)

acs_condition_gcs_payload_2016 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2016')
acs_condition_gcs_payload_2016 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2016")
acs_condition_gcs_operator_2016 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2016', acs_condition_gcs_payload_2016, data_ingestion_dag
"acs_condition_to_gcs_2016", acs_condition_gcs_payload_2016, data_ingestion_dag
)

acs_condition_gcs_payload_2017 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2017')
acs_condition_gcs_payload_2017 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2017")
acs_condition_gcs_operator_2017 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2017', acs_condition_gcs_payload_2017, data_ingestion_dag
"acs_condition_to_gcs_2017", acs_condition_gcs_payload_2017, data_ingestion_dag
)

acs_condition_gcs_payload_2018 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2018')
acs_condition_gcs_payload_2018 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2018")
acs_condition_gcs_operator_2018 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2018', acs_condition_gcs_payload_2018, data_ingestion_dag
"acs_condition_to_gcs_2018", acs_condition_gcs_payload_2018, data_ingestion_dag
)

acs_condition_gcs_payload_2019 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2019')
acs_condition_gcs_payload_2019 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2019")
acs_condition_gcs_operator_2019 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2019', acs_condition_gcs_payload_2019, data_ingestion_dag
"acs_condition_to_gcs_2019", acs_condition_gcs_payload_2019, data_ingestion_dag
)

acs_condition_gcs_payload_2020 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2020')
acs_condition_gcs_payload_2020 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2020")
acs_condition_gcs_operator_2020 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2020', acs_condition_gcs_payload_2020, data_ingestion_dag
"acs_condition_to_gcs_2020", acs_condition_gcs_payload_2020, data_ingestion_dag
)

acs_condition_gcs_payload_2021 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2021')
acs_condition_gcs_payload_2021 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2021")
acs_condition_gcs_operator_2021 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2021', acs_condition_gcs_payload_2021, data_ingestion_dag
"acs_condition_to_gcs_2021", acs_condition_gcs_payload_2021, data_ingestion_dag
)

acs_condition_gcs_payload_2022 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year='2022')
acs_condition_gcs_payload_2022 = util.generate_gcs_payload(_ACS_WORKFLOW_ID, year="2022")
acs_condition_gcs_operator_2022 = util.create_gcs_ingest_operator(
'acs_condition_to_gcs_2022', acs_condition_gcs_payload_2022, data_ingestion_dag
"acs_condition_to_gcs_2022", acs_condition_gcs_payload_2022, data_ingestion_dag
)

# PROCESS AND WRITE TO BQ

acs_condition_bq_payload_2012 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2012')
acs_condition_bq_payload_2012 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2012")
acs_condition_bq_operator_2012 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2012", acs_condition_bq_payload_2012, data_ingestion_dag
)

acs_condition_bq_payload_2013 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2013')
acs_condition_bq_payload_2013 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2013")
acs_condition_bq_operator_2013 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2013", acs_condition_bq_payload_2013, data_ingestion_dag
)

acs_condition_bq_payload_2014 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2014')
acs_condition_bq_payload_2014 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2014")
acs_condition_bq_operator_2014 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2014", acs_condition_bq_payload_2014, data_ingestion_dag
)

acs_condition_bq_payload_2015 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2015')
acs_condition_bq_payload_2015 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2015")
acs_condition_bq_operator_2015 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2015", acs_condition_bq_payload_2015, data_ingestion_dag
)

acs_condition_bq_payload_2016 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2016')
acs_condition_bq_payload_2016 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2016")
acs_condition_bq_operator_2016 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2016", acs_condition_bq_payload_2016, data_ingestion_dag
)

acs_condition_bq_payload_2017 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2017')
acs_condition_bq_payload_2017 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2017")
acs_condition_bq_operator_2017 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2017", acs_condition_bq_payload_2017, data_ingestion_dag
)

acs_condition_bq_payload_2018 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2018')
acs_condition_bq_payload_2018 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2018")
acs_condition_bq_operator_2018 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2018", acs_condition_bq_payload_2018, data_ingestion_dag
)

acs_condition_bq_payload_2019 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2019')
acs_condition_bq_payload_2019 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2019")
acs_condition_bq_operator_2019 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2019", acs_condition_bq_payload_2019, data_ingestion_dag
)

acs_condition_bq_payload_2020 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2020')
acs_condition_bq_payload_2020 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2020")
acs_condition_bq_operator_2020 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2020", acs_condition_bq_payload_2020, data_ingestion_dag
)

acs_condition_bq_payload_2021 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2021')
acs_condition_bq_payload_2021 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2021")
acs_condition_bq_operator_2021 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2021", acs_condition_bq_payload_2021, data_ingestion_dag
)

acs_condition_bq_payload_2022 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2022')
acs_condition_bq_payload_2022 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2022")
acs_condition_bq_operator_2022 = util.create_bq_ingest_operator(
"acs_condition_to_bq_2022", acs_condition_bq_payload_2022, data_ingestion_dag
)

# EXPORT FROM BQ TO BUCKETS
acs_condition_exporter_payload_race = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_race",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_race",
}
acs_condition_exporter_operator_race = util.create_exporter_operator(
"acs_condition_exporter_race",
Expand All @@ -146,16 +146,16 @@
)

acs_condition_exporter_payload_age = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_age",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_age",
}
acs_condition_exporter_operator_age = util.create_exporter_operator(
"acs_condition_exporter_age", acs_condition_exporter_payload_age, data_ingestion_dag
)

acs_condition_exporter_payload_sex = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_sex",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_sex",
}
acs_condition_exporter_operator_sex = util.create_exporter_operator(
"acs_condition_exporter_sex", acs_condition_exporter_payload_sex, data_ingestion_dag
Expand Down
94 changes: 47 additions & 47 deletions airflow/dags/acs_population.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,124 +6,124 @@
from datetime import timedelta
import util

_ACS_WORKFLOW_ID = 'ACS_POPULATION'
_ACS_DATASET_NAME = 'acs_population'
_ACS_WORKFLOW_ID = "ACS_POPULATION"
_ACS_DATASET_NAME = "acs_population"

default_args = {
'start_date': days_ago(0),
'execution_timeout': timedelta(minutes=15),
"start_date": days_ago(0),
"execution_timeout": timedelta(minutes=15),
}

data_ingestion_dag = DAG(
'acs_population_ingestion_dag',
"acs_population_ingestion_dag",
default_args=default_args,
schedule_interval=None,
description='Ingestion configuration for ACS Population',
description="Ingestion configuration for ACS Population",
)


acs_pop_gcs_payload = util.generate_gcs_payload(_ACS_WORKFLOW_ID)
acs_pop_gcs_operator = util.create_gcs_ingest_operator('acs_population_to_gcs', acs_pop_gcs_payload, data_ingestion_dag)
acs_pop_gcs_operator = util.create_gcs_ingest_operator("acs_population_to_gcs", acs_pop_gcs_payload, data_ingestion_dag)


acs_pop_bq_payload_2009 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2009')
acs_pop_bq_payload_2009 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2009")
acs_pop_bq_operator_2009 = util.create_bq_ingest_operator(
'acs_population_to_bq_2009', acs_pop_bq_payload_2009, data_ingestion_dag
"acs_population_to_bq_2009", acs_pop_bq_payload_2009, data_ingestion_dag
)

acs_pop_bq_payload_2010 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2010')
acs_pop_bq_payload_2010 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2010")
acs_pop_bq_operator_2010 = util.create_bq_ingest_operator(
'acs_population_to_bq_2010', acs_pop_bq_payload_2010, data_ingestion_dag
"acs_population_to_bq_2010", acs_pop_bq_payload_2010, data_ingestion_dag
)

acs_pop_bq_payload_2011 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2011')
acs_pop_bq_payload_2011 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2011")
acs_pop_bq_operator_2011 = util.create_bq_ingest_operator(
'acs_population_to_bq_2011', acs_pop_bq_payload_2011, data_ingestion_dag
"acs_population_to_bq_2011", acs_pop_bq_payload_2011, data_ingestion_dag
)

acs_pop_bq_payload_2012 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2012')
acs_pop_bq_payload_2012 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2012")
acs_pop_bq_operator_2012 = util.create_bq_ingest_operator(
'acs_population_to_bq_2012', acs_pop_bq_payload_2012, data_ingestion_dag
"acs_population_to_bq_2012", acs_pop_bq_payload_2012, data_ingestion_dag
)

acs_pop_bq_payload_2013 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2013')
acs_pop_bq_payload_2013 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2013")
acs_pop_bq_operator_2013 = util.create_bq_ingest_operator(
'acs_population_to_bq_2013', acs_pop_bq_payload_2013, data_ingestion_dag
"acs_population_to_bq_2013", acs_pop_bq_payload_2013, data_ingestion_dag
)

acs_pop_bq_payload_2014 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2014')
acs_pop_bq_payload_2014 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2014")
acs_pop_bq_operator_2014 = util.create_bq_ingest_operator(
'acs_population_to_bq_2014', acs_pop_bq_payload_2014, data_ingestion_dag
"acs_population_to_bq_2014", acs_pop_bq_payload_2014, data_ingestion_dag
)

acs_pop_bq_payload_2015 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2015')
acs_pop_bq_payload_2015 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2015")
acs_pop_bq_operator_2015 = util.create_bq_ingest_operator(
'acs_population_to_bq_2015', acs_pop_bq_payload_2015, data_ingestion_dag
"acs_population_to_bq_2015", acs_pop_bq_payload_2015, data_ingestion_dag
)

acs_pop_bq_payload_2016 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2016')
acs_pop_bq_payload_2016 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2016")
acs_pop_bq_operator_2016 = util.create_bq_ingest_operator(
'acs_population_to_bq_2016', acs_pop_bq_payload_2016, data_ingestion_dag
"acs_population_to_bq_2016", acs_pop_bq_payload_2016, data_ingestion_dag
)

acs_pop_bq_payload_2017 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2017')
acs_pop_bq_payload_2017 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2017")
acs_pop_bq_operator_2017 = util.create_bq_ingest_operator(
'acs_population_to_bq_2017', acs_pop_bq_payload_2017, data_ingestion_dag
"acs_population_to_bq_2017", acs_pop_bq_payload_2017, data_ingestion_dag
)

acs_pop_bq_payload_2018 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2018')
acs_pop_bq_payload_2018 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2018")
acs_pop_bq_operator_2018 = util.create_bq_ingest_operator(
'acs_population_to_bq_2018', acs_pop_bq_payload_2018, data_ingestion_dag
"acs_population_to_bq_2018", acs_pop_bq_payload_2018, data_ingestion_dag
)

acs_pop_bq_payload_2019 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2019')
acs_pop_bq_payload_2019 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2019")
acs_pop_bq_operator_2019 = util.create_bq_ingest_operator(
'acs_population_to_bq_2019', acs_pop_bq_payload_2019, data_ingestion_dag
"acs_population_to_bq_2019", acs_pop_bq_payload_2019, data_ingestion_dag
)

acs_pop_bq_payload_2020 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2020')
acs_pop_bq_payload_2020 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2020")
acs_pop_bq_operator_2020 = util.create_bq_ingest_operator(
'acs_population_to_bq_2020', acs_pop_bq_payload_2020, data_ingestion_dag
"acs_population_to_bq_2020", acs_pop_bq_payload_2020, data_ingestion_dag
)

acs_pop_bq_payload_2021 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2021')
acs_pop_bq_payload_2021 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2021")
acs_pop_bq_operator_2021 = util.create_bq_ingest_operator(
'acs_population_to_bq_2021', acs_pop_bq_payload_2021, data_ingestion_dag
"acs_population_to_bq_2021", acs_pop_bq_payload_2021, data_ingestion_dag
)

acs_pop_bq_payload_2022 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year='2022')
acs_pop_bq_payload_2022 = util.generate_bq_payload(_ACS_WORKFLOW_ID, _ACS_DATASET_NAME, year="2022")
acs_pop_bq_operator_2022 = util.create_bq_ingest_operator(
'acs_population_to_bq_2022', acs_pop_bq_payload_2022, data_ingestion_dag
"acs_population_to_bq_2022", acs_pop_bq_payload_2022, data_ingestion_dag
)


acs_pop_exporter_payload_race = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_race",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_race",
}
acs_pop_exporter_operator_race = util.create_exporter_operator(
'acs_population_exporter_race', acs_pop_exporter_payload_race, data_ingestion_dag
"acs_population_exporter_race", acs_pop_exporter_payload_race, data_ingestion_dag
)

acs_pop_exporter_payload_age = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_age",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_age",
}
acs_pop_exporter_operator_age = util.create_exporter_operator(
'acs_population_exporter_age', acs_pop_exporter_payload_age, data_ingestion_dag
"acs_population_exporter_age", acs_pop_exporter_payload_age, data_ingestion_dag
)

acs_pop_exporter_payload_sex = {
'dataset_name': _ACS_DATASET_NAME,
'demographic': "by_sex",
"dataset_name": _ACS_DATASET_NAME,
"demographic": "by_sex",
}
acs_pop_exporter_operator_sex = util.create_exporter_operator(
'acs_population_exporter_sex', acs_pop_exporter_payload_sex, data_ingestion_dag
"acs_population_exporter_sex", acs_pop_exporter_payload_sex, data_ingestion_dag
)

connector1 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id='connector1')
connector2 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id='connector2')
connector3 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id='connector3')
connector1 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id="connector1")
connector2 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id="connector2")
connector3 = DummyOperator(default_args=default_args, dag=data_ingestion_dag, task_id="connector3")

# ensure CACHING step runs, then 2009 to make new BQ tables
# then run the rest of the years in parallel chunks
Expand Down
Loading

0 comments on commit 46eaa5e

Please sign in to comment.