diff --git a/.test-compose.yml b/.test-compose.yml index f58c925a..4517408f 100644 --- a/.test-compose.yml +++ b/.test-compose.yml @@ -16,6 +16,11 @@ services: user: "${UID}:${GID}" extra_hosts: - "gateway.docker.internal:host-gateway" + depends_on: + pgrouting: + condition: service_healthy + links: + - pgrouting pgrouting: image: pgrouting/pgrouting @@ -26,9 +31,15 @@ services: - "5433:5433" volumes: - db-data:/var/lib/postgis:/postgresql/13/main + healthcheck: + test: ["CMD-SHELL", "pg_isready", "-q", "-d", "postgres", "-U", "postgres" ] + interval: 5s + timeout: 5s + retries: 5 environment: POSTGRES_PASSWORD: ghscic POSTGRES_HOST: ghscic_postgis + PGPORT: 5433 volumes: db-data: diff --git a/process/subprocesses/_02_create_osm_resources.py b/process/subprocesses/_02_create_osm_resources.py index 10b38f1c..9fc6b088 100644 --- a/process/subprocesses/_02_create_osm_resources.py +++ b/process/subprocesses/_02_create_osm_resources.py @@ -21,7 +21,7 @@ def create_poly_boundary_file(config): db_pwd = config['db_pwd'] feature = f"""PG:"dbname={db} host={db_host} port={db_port} user={db_user} password={db_pwd}" {config['buffered_urban_study_region']}""" print('Create poly file, using command: '), - command = f'python ogr2poly.py {feature} -f "db"' + command = f'python /home/ghsci/process/subprocesses/ogr2poly.py {feature} -f "db"' print(command) sp.call(command, shell=True) command = f'mv {os.path.basename(config["codename_poly"])} {config["codename_poly"]}' diff --git a/process/subprocesses/_11_neighbourhood_analysis.py b/process/subprocesses/_11_neighbourhood_analysis.py index 3af6c0ec..df0fe723 100644 --- a/process/subprocesses/_11_neighbourhood_analysis.py +++ b/process/subprocesses/_11_neighbourhood_analysis.py @@ -291,7 +291,6 @@ def calculate_sample_point_indicators( / sample_points[columns].std() ).sum(axis=1) # grid_id and edge_ogc_fid are integers - print(sample_points.query('grid_id.isna()')) sample_points[sample_points.columns[0:2]] = sample_points[ sample_points.columns[0:2] ].astype(int) diff --git a/process/subprocesses/ghsci.py b/process/subprocesses/ghsci.py index 61300246..423a645c 100644 --- a/process/subprocesses/ghsci.py +++ b/process/subprocesses/ghsci.py @@ -137,6 +137,94 @@ def drop(self): drop_resources(self) + def _create_database(self): + """Create database for this study region.""" + from _00_create_database import create_database + + create_database(self.codename) + return f"Database {self.config['db']} created." + + def _create_study_region(self): + """Create study region boundaries for this study region.""" + from _01_create_study_region import create_study_region + + create_study_region(self.codename) + return 'Study region boundaries created.' + + def _create_osm_resources(self): + """Create OSM resources for this study region.""" + from _02_create_osm_resources import create_osm_resources + + create_osm_resources(self.codename) + return 'OSM resources created.' + + def _create_network_resources(self): + """Create network resources for this study region.""" + from _03_create_network_resources import create_network_resources + + create_network_resources(self.codename) + return 'Network resources created.' + + def _create_population_grid(self): + """Create population grid for this study region.""" + from _04_create_population_grid import create_population_grid + + create_population_grid(self.codename) + return 'Population grid created.' + + def _create_destinations(self): + """Compile destinations for this study region.""" + from _05_compile_destinations import compile_destinations + + compile_destinations(self.codename) + return 'Destinations compiled.' + + def _create_open_space_areas(self): + """Create open space areas for this study region.""" + from _06_open_space_areas_setup import open_space_areas_setup + + open_space_areas_setup(self.codename) + return 'Open space areas created.' + + def _create_neighbourhoods(self): + """Create neighbourhood relations between nodes for this study region.""" + from _07_locate_origins_destinations import nearest_node_locations + + nearest_node_locations(self.codename) + return 'Neighbourhoods created.' + + def _create_destination_summary_tables(self): + """Create destination summary tables for this study region.""" + from _08_destination_summary import destination_summary + + destination_summary(self.codename) + return 'Destination summary tables created.' + + def _link_urban_covariates(self): + """Link urban covariates to nodes for this study region.""" + from _09_urban_covariates import link_urban_covariates + + link_urban_covariates(self.codename) + return 'Urban covariates linked.' + + def _gtfs_analysis(self): + from _10_gtfs_analysis import gtfs_analysis + + gtfs_analysis(self.codename) + return 'GTFS analysis completed.' + + def _neighbourhood_analysis(self): + from _11_neighbourhood_analysis import neighbourhood_analysis + + neighbourhood_analysis(self.codename) + return 'Neighbourhood analysis completed.' + + def _area_analysis(self): + from _12_aggregation import aggregate_study_region_indicators + + aggregate_study_region_indicators(self.codename) + return 'Area analysis completed.' + def get_engine(self): """Given configuration details, create a database engine.""" engine = create_engine( diff --git a/process/tests/tests.py b/process/tests/tests.py index 53f110a1..e04bcedd 100644 --- a/process/tests/tests.py +++ b/process/tests/tests.py @@ -21,6 +21,7 @@ Successful running of all tests may require running of tests within the global-indicators Docker container, hence the use of a custom .test-compose.yml for this purpose. """ +import os import sys import unittest @@ -35,50 +36,75 @@ class tests(unittest.TestCase): """A collection of tests to help ensure functionality.""" - def test_global_indicators_shell(self): + def test_1_global_indicators_shell(self): """Unix shell script should only have unix-style line endings.""" counts = calculate_line_endings('../global-indicators.sh') lf = counts.pop(b'\n') self.assertTrue(sum(counts.values()) == 0 and lf > 0) - def test_project_setup(self): + def test_2_project_setup(self): """Check if _project_setup.py imported successfully.""" self.assertTrue(project_setup) - def test_load_example_region(self): + def test_3_load_example_region(self): """Load example region.""" codename = 'example_ES_Las_Palmas_2023' r = ghsci.Region(codename) - # def test_example_analysis(self): - # """Analyse example region.""" - # codename = 'example_ES_Las_Palmas_2023' - # r = ghsci.Region(codename) - # r.analysis() - - # def test_example_generate(self): - # """Generate resources for example region.""" - # codename = 'example_ES_Las_Palmas_2023' - # r = ghsci.Region(codename) - # r.generate() - - # def test_sensitivity(self): - # """Test sensitivity analysis of urban intersection parameter.""" - # reference = 'example_ES_Las_Palmas_2023' - # comparison = 'ES_Las_Palmas_2023_test_not_urbanx' - # # create modified version of reference configuration - # with open(f'./configuration/regions/{reference}.yml') as file: - # configuration = file.read() - # configuration = configuration.replace( - # 'ghsl_urban_intersection: true', - # 'ghsl_urban_intersection: false', - # ) - # with open(f'./configuration/regions/{comparison}.yml', 'w') as file: - # file.write(configuration) - # r = ghsci.Region(comparison) - # r.analysis() - # r.generate() - # r.compare(reference) + def test_4_create_db(self): + """Load example region.""" + codename = 'example_ES_Las_Palmas_2023' + r = ghsci.Region(codename) + r._create_database() + + def test_5_example_analysis(self): + """Analyse example region.""" + codename = 'example_ES_Las_Palmas_2023' + r = ghsci.Region(codename) + r.analysis() + + def test_6_example_generate(self): + """Generate resources for example region.""" + codename = 'example_ES_Las_Palmas_2023' + r = ghsci.Region(codename) + r.generate() + + def test_7_sensitivity(self): + """Test sensitivity analysis of urban intersection parameter.""" + reference = 'example_ES_Las_Palmas_2023' + comparison = 'ES_Las_Palmas_2023_test_not_urbanx' + # create modified version of reference configuration + with open(f'./configuration/regions/{reference}.yml') as file: + configuration = file.read() + configuration = configuration.replace( + 'ghsl_urban_intersection: true', + 'ghsl_urban_intersection: false', + ) + with open(f'./configuration/regions/{comparison}.yml', 'w') as file: + file.write(configuration) + r_comparison = ghsci.Region(comparison) + # create output folder for comparison region + if not os.path.exists( + f'{ghsci.folder_path}/process/data/_study_region_outputs', + ): + os.makedirs( + f'{ghsci.folder_path}/process/data/_study_region_outputs', + ) + if not os.path.exists(r_comparison.config['region_dir']): + os.makedirs(r_comparison.config['region_dir']) + with open(f'./configuration/regions/{comparison}.yml', 'w') as file: + file.write(configuration) + r = ghsci.Region(reference) + df = r.get_df('indicators_region') + df.drop(columns=['geom'], inplace=True) + df[df.columns[(df.dtypes == 'float64').values]] = df[ + df.columns[(df.dtypes == 'float64').values] + ].astype(int) + df.to_csv( + f"{r_comparison.config['region_dir']}/{r_comparison.codename}_indicators_region.csv", + index=False, + ) + r.compare(comparison) def calculate_line_endings(path): @@ -105,20 +131,5 @@ def calculate_line_endings(path): return counts -def suite(): - suite = unittest.TestSuite() - for t in [ - 'test_global_indicators_shell', - 'test_project_setup', - 'test_load_example_region', - # 'test_example_analysis', - # 'test_example_generate', - # 'test_sensitivity', - ]: - suite.addTest(tests(t)) - return suite - - if __name__ == '__main__': - runner = unittest.TextTestRunner() - runner.run(suite()) + unittest.main(failfast=True)