create and load db "ports" with alembic & new ports db: UpdatedPub150…

….csv
dataforgoodfr · Feb 27, 2024 · 9b5e266 · 9b5e266
1 parent 0aa29f0
commit 9b5e266
Show file tree

Hide file tree

Showing 10 changed files with 15,773 additions and 23 deletions.
diff --git a/Makefile b/Makefile
@@ -1,22 +1,22 @@
 VERSION ?= 1.0.0
 
-BLOOM_DEV_DOCKER = @docker run --name bloom-test  --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net -p 8501:8501
-BLOOM_PRODUCTION_DOCKER = @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh
+BLOOM_DEV_DOCKER = @docker run --name bloom-test --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env --network=bloom_net -p 8501:8501
+BLOOM_PRODUCTION_DOCKER = @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.prod --log-driver json-file --log-opt max-size=10M --log-opt max-file=3 --entrypoint /entrypoint.sh
 
 build:
 	@docker build -t d4g/bloom:${VERSION} --platform linux/amd64  -f docker-env/Dockerfile .
 	@docker tag d4g/bloom:${VERSION} d4g/bloom:latest
 
 launch-dev-db:
 	@docker compose -f docker-env/docker-compose-db.yaml up -d
-	@sleep 20
+	@sleep 10
 	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} alembic upgrade head
-	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
+	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_ports_data.py
 
-load-amp-data:
+load-dev-db:
+	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_vessels_data.py
+	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_ports_data.py
 	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_amp_data.py
-
-load-test-positions-data:
 	$(BLOOM_DEV_DOCKER) --rm d4g/bloom:${VERSION} /venv/bin/python3 alembic/init_script/load_positions_data.py
 
 launch-dev-container:

diff --git a/alembic/init_script/load_ports_data.py b/alembic/init_script/load_ports_data.py
@@ -0,0 +1,38 @@
+import logging
+import os
+from pathlib import Path
+
+import pandas as pd
+from sqlalchemy import create_engine
+
+logging.basicConfig()
+logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
+
+postgres_user = os.environ.get("POSTGRES_USER")
+postgres_password = os.environ.get("POSTGRES_PASSWORD")
+postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
+postgres_db = os.environ.get("POSTGRES_DB")
+postgres_port = os.environ.get("POSTGRES_PORT")
+
+# The db url is configured with the db connexion variables declared in the db.yaml file.
+db_url = (
+    "postgresql://"
+    + postgres_user
+    + ":"
+    + postgres_password
+    + "@"
+    + postgres_hostname
+    + ":"s
+    + postgres_port
+    + "/"
+    + postgres_db
+)
+
+engine = create_engine(db_url)
+
+df = pd.read_csv(
+    Path(os.path.dirname(__file__)).joinpath("../../data/ports_rad3000_res10.csv"),
+    sep=";",
+)
+
+df.to_sql("ports", engine, if_exists="append", index=False)
diff --git a/alembic/init_script/load_vessels_data.py b/alembic/init_script/load_vessels_data.py
@@ -27,7 +27,9 @@
     + "/"
     + postgres_db
 )
+
 engine = create_engine(db_url)
+
 df = pd.read_csv(
     Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"),
     sep=";",

diff --git a/alembic/versions/7962eee40abe_create_port_table.py b/alembic/versions/7962eee40abe_create_port_table.py
@@ -0,0 +1,49 @@
+"""create ports table
+
+Revision ID: 7962eee40abe
+Revises: 961cee5426d6
+Create Date: 2024-02-26 18:38:37.726130
+
+"""
+import sqlalchemy as sa
+from geoalchemy2 import Geometry
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7962eee40abe"
+down_revision = "961cee5426d6"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """
+    ports.csv is a file that contains the data of the ports.
+    url;country;port;locode;latitude;longitude;geometry_point;geometry_buffer
+    https://www.vesselfinder.com/ports/ALSAR001;Albania;Sarande;ALSAR;39.8701;20.0062;POINT (20.0062 39.8701
+    """
+    op.create_table(
+        "ports",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("country", sa.String(255), nullable=False),
+        sa.Column("port", sa.String(255), nullable=False),
+        sa.Column("url", sa.String(255), nullable=False),
+        sa.Column("locode", sa.String(255), nullable=False),
+        sa.Column("latitude", sa.String(255), nullable=False),
+        sa.Column("longitude", sa.String(255), nullable=False),
+        sa.Column(
+            "geometry_point",
+            Geometry(geometry_type="POINT", srid=4326),
+            nullable=False,
+        ),
+        sa.Column(
+            "geometry_buffer",
+            Geometry(geometry_type="POLYGON", srid=4326),
+            nullable=False,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("ports")
diff --git a/alembic/versions/961cee5426d6_create_amp_table.py b/alembic/versions/961cee5426d6_create_amp_table.py
@@ -5,30 +5,38 @@
 Create Date: 2024-02-11 22:10:19.010986
 
 """
-from alembic import op
-import sqlalchemy as sa
 import geoalchemy2
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+from sqlalchemy import Inspector
+
+from alembic import op
 
 # revision identifiers, used by Alembic.
-revision = '961cee5426d6'
-down_revision = '1fd83d22bd1e'
+revision = "961cee5426d6"
+down_revision = "1fd83d22bd1e"
 branch_labels = None
 depends_on = None
 
 
 def upgrade() -> None:
-    op.create_table("mpa_fr_with_mn",
-                    sa.Column("index", sa.Integer, primary_key=True),
-                    sa.Column("wdpaid", sa.Integer),
-                    sa.Column("name", sa.String, nullable=False),
-                    sa.Column("desig_eng", sa.String),
-                    sa.Column("desig_type", sa.String),
-                    sa.Column("iucn_cat", sa.String),
-                    sa.Column("parent_iso", sa.String),
-                    sa.Column("iso3", sa.String),
-                    sa.Column("geometry", geoalchemy2.types.Geometry(geometry_type="GEOMETRY", srid=4326)),
-                    sa.Column("benificiaries", sa.String)
-                    )
+    op.create_table(
+        "mpa_fr_with_mn",
+        sa.Column("index", sa.Integer, primary_key=True),
+        sa.Column("wdpaid", sa.Integer),
+        sa.Column("name", sa.String, nullable=False),
+        sa.Column("desig_eng", sa.String),
+        sa.Column("desig_type", sa.String),
+        sa.Column("iucn_cat", sa.String),
+        sa.Column("parent_iso", sa.String),
+        sa.Column("iso3", sa.String),
+        sa.Column(
+            "geometry",
+            geoalchemy2.types.Geometry(geometry_type="GEOMETRY", srid=4326),
+        ),
+        sa.Column("benificiaries", sa.String),
+    )
 
 
 def downgrade() -> None: