feat: add class task management + csv path from config

.env.template

@@ -17,6 +17,21 @@ DATA_FOLDER=
 LOGGING_LEVEL=INFO
 
 
+AMP_DATA_CSV_PATH=
+PORT_DATA_CSV_PATH=
+PORT_POLYGON_DATA_CSV_PATH=
+VESSEL_DATA_CSV_PATH=
+
+VESSEL_POSITIONS_DATA_CSV_PATH=
+SPIRE_DATA_CSV_PATH=
+
+#PORT_RADIUS_M=3000
+#PORT_RESOLUTION=10
+
+
+#LOGGING_FORMAT="[%(name)s %(levelname)s @ %(asctime)s] %(pathname)s:%(lineno)s: %(message)s"
+
+
 ###############################################################################################
 # DOCKER SPECIFIC VARIABLES
 ###############################################################################################

README.md

@@ -156,10 +156,8 @@ password, database name, port to use)
     # * data/spire_positions_subset.csv
     # * data/vessels_subset.csv
     # * data/zones_subset.csv
-    $ python3 src/bloom/tasks/load_dim_vessel_from_csv.py 
-    $ python3 src/bloom/tasks/load_dim_port_from_csv.py
-    $ python3 src/bloom/tasks/load_dim_zone_amp_from_csv.py
-    $ python3 src/bloom/tasks/compute_port_geometry_buffer.py
+    $ python3 src/tasks/init.py &&
+    $ python3 src/tasks/pipeline_ais_data.py
 ```
 
 #### Starting the application

docker-compose-load-data.yaml

@@ -10,16 +10,14 @@ x-common-infos:
 services:
   load-data:
     container_name: bloom-load-data
-    image: d4g/bloom:${VERSION:-latest}
+    image: ghcr.io/dataforgoodfr/12_bloom:${VERSION:-latest}
     entrypoint: /bin/bash
     # Nominal start:
     command:
       - -c
-        /venv/bin/python3 src/bloom/tasks/load_dim_vessel_from_csv.py &&
-        /venv/bin/python3 src/alembic/init_script/load_positions_data.py &&
-        /venv/bin/python3 src/alembic/init_script/load_amp_data.py &&
-        /venv/bin/python3 src/bloom/tasks/load_dim_port_from_csv.py &&
-        /venv/bin/python3 src/bloom/tasks/compute_port_geometry_buffer.py
+      - |
+        /venv/bin/python3 src/tasks/init.py &&
+        /venv/bin/python3 src/tasks/pipeline_ais_data.py
     volumes:
       - ./:/project/
       - ./data:/project/data

docker-compose.yaml

@@ -10,7 +10,7 @@ x-common-infos:
 services:
   bloom:
     container_name: bloom
-    image: d4g/bloom:${VERSION:-latest}
+    image: ghcr.io/dataforgoodfr/12_bloom:${VERSION:-latest}
     build:
       context: .
       dockerfile: ./docker/Dockerfile
@@ -60,7 +60,7 @@ services:
   init:
     container_name: init_bloom
     hostname: init_bloom
-    image: d4g/bloom:${VERSION:-latest}
+    image: ghcr.io/dataforgoodfr/12_bloom:${VERSION:-latest}
 
     # Nominal start:
     # As postgres+postgis gt available, then unavialable, then avialable at database init

src/alembic/versions/72af814ca33d_add_batch_reference.py

@@ -0,0 +1,31 @@
+"""add batch reference
+
+Revision ID: 72af814ca33d
+Revises: 4e912be8a176
+Create Date: 2024-04-03 21:45:21.431765
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '72af814ca33d'
+down_revision = '4e912be8a176'
+branch_labels = None
+depends_on = '4e912be8a176'
+
+
+def upgrade() -> None:
+    op.add_column('dim_vessel',sa.Column("batch", sa.String),schema='public')
+    op.add_column('spire_ais_data',sa.Column("batch", sa.String),
+                                             schema='public')
+    op.rename_table('spire_ais_data','stg_spire_ais_data',schema='public')
+    pass
+
+
+def downgrade() -> None:
+    op.rename_table('stg_spire_ais_data','spire_ais_data',schema='public')
+    op.drop_column('spire_ais_data','batch',schema='public')
+    op.drop_column('dim_vessel','batch',schema='public')
+    pass

src/bloom/config.py

@@ -30,6 +30,8 @@ class Settings(BaseSettings):
         env_file_encoding = 'utf-8',
         extra='ignore'
         )
+
+    logging_format:str="[%(name)s %(levelname)s @ %(asctime)s] %(message)s"
 
     # Déclaration des attributs/paramètres disponibles au sein de la class settings
     postgres_user:str = Field(default='')
@@ -49,6 +51,17 @@ class Settings(BaseSettings):
                                 default="INFO",
                                 pattern=r'NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL'
                             )
+
+    amp_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/zones_subset.csv")
+    port_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/ports.csv")
+    port_polygon_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/ports_rad3000_res10.csv")
+    port_radius_m:int = 3000    # Radius in meters
+    port_resolution:int = 10    # Number of points in the resulting polygon
+    vessel_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/vessels_subset.csv")
+    spire_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/spire_positions_subset.csv")
+
+
+    segment_data_csv_path:Path = Path(__file__).parent.joinpath("../../data/segments_subset.csv")
 
     @model_validator(mode='after')
     def update_db_url(self)->dict:

src/bloom/domain/excursion.py

@@ -7,7 +7,7 @@
 
 class Excursion(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
-    id: int | None = None
+    id: Union[int , None] = None
     vessel_id: int
     departure_port_id: Union[int, None] = None
     departure_at: Union[datetime, None] = None
@@ -27,3 +27,4 @@ class Excursion(BaseModel):
     total_time_extincting_amp: Union[timedelta, None] = None
     created_at: Union[datetime, None] = None
     updated_at: Union[datetime, None] = None
+    batch: Union[str, None] = None

src/bloom/domain/port.py

@@ -21,3 +21,4 @@ class Port(BaseModel):
     has_excursion: bool = False
     created_at: Union[datetime, None] = None
     updated_at: Union[datetime, None] = None
+    batch: Union[str, None] = None

src/bloom/domain/rel_segment_zone.py

@@ -9,3 +9,4 @@ class RelSegmentZone(BaseModel):
     segment_id: int
     zone_id: int
     created_at: Union[datetime, None] = None
+    batch: Union[str, None] = None

src/bloom/domain/segment.py

@@ -1,30 +1,36 @@
 from datetime import datetime, timedelta
+<<<<<<< HEAD
+=======
+
+from pydantic import BaseModel
+
+>>>>>>> 576cbd6 (fix/feat: load SpireDataFromCsv task)
 from typing import Union
 
 from pydantic import BaseModel, ConfigDict
 from shapely import Point
 
-
 class Segment(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
-    id: int | None = None
+    id: Union[ int , None ] = None
     excursion_id: int
     timestamp_start: Union[datetime, None] = None
     timestamp_end: Union[datetime, None] = None
     segment_duration: Union[timedelta, None] = None
     start_position: Union[Point, None] = None
     end_position: Union[Point, None] = None
-    heading: Union[float | None] = None
-    distance: Union[float | None] = None
-    average_speed: Union[float | None] = None
-    speed_at_start: Union[float | None] = None
-    speed_at_end: Union[float | None] = None
-    heading_at_start: Union[float | None] = None
-    heading_at_end: Union[float | None] = None
-    type: Union[str | None] = None
-    in_amp_zone: Union[bool | None] = None
-    in_territorial_waters: Union[bool | None] = None
-    in_costal_waters: Union[bool | None] = None
-    last_vessel_segment: Union[bool | None] = None
+    heading: Union[float , None] = None
+    distance: Union[float , None] = None
+    average_speed: Union[float , None] = None
+    speed_at_start: Union[float , None] = None
+    speed_at_end: Union[float , None] = None
+    heading_at_start: Union[float , None] = None
+    heading_at_end: Union[float , None] = None
+    type: Union[str , None] = None
+    in_amp_zone: Union[bool , None] = None
+    in_territorial_waters: Union[bool , None] = None
+    in_costal_waters: Union[bool , None] = None
+    last_vessel_segment: Union[bool , None] = None
     created_at: Union[datetime, None] = None
     updated_at: Union[datetime, None] = None
+    batch: Union[str, None] = None

src/bloom/domain/spire_ais_data.py

@@ -8,38 +8,39 @@
 
 
 class SpireAisData(BaseModel):
-    id: Union[int | None] = None  # noqa: UP007
+    id: Union[int , None] = None  # noqa: UP007
     spire_update_statement: datetime
-    vessel_ais_class: Union[str | None] = None  # noqa: UP007
-    vessel_flag: Union[str | None] = None  # noqa: UP007
-    vessel_name: Union[str | None] = None  # noqa: UP007
-    vessel_callsign: Union[str | None] = None  # noqa: UP007
-    vessel_timestamp: Union[datetime | None] = None  # noqa: UP007
-    vessel_update_timestamp: Union[datetime | None] = None  # noqa: UP007
-    vessel_ship_type: Union[str | None] = None  # noqa: UP007
-    vessel_sub_ship_type: Union[str | None] = None  # noqa: UP007
-    vessel_mmsi: Union[int | None] = None  # noqa: UP007
-    vessel_imo: Union[int | None] = None  # noqa: UP007
-    vessel_width: Union[int | None] = None  # noqa: UP007
-    vessel_length: Union[int | None] = None  # noqa: UP007
-    position_accuracy: Union[str | None] = None  # noqa: UP007
-    position_collection_type: Union[str | None] = None  # noqa: UP007
-    position_course: Union[float | None] = None  # noqa: UP007
-    position_heading: Union[float | None] = None  # noqa: UP007
-    position_latitude: Union[float | None] = None  # noqa: UP007
-    position_longitude: Union[float | None] = None  # noqa: UP007
-    position_maneuver: Union[str | None] = None  # noqa: UP007
-    position_navigational_status: Union[str | None] = None  # noqa: UP007
-    position_rot: Union[float | None] = None  # noqa: UP007
-    position_speed: Union[float | None] = None  # noqa: UP007
-    position_timestamp: Union[datetime | None] = None  # noqa: UP007
-    position_update_timestamp: Union[datetime | None] = None  # noqa: UP007
-    voyage_destination: Union[str | None] = None  # noqa: UP007
-    voyage_draught: Union[float | None] = None  # noqa: UP007
-    voyage_eta: Union[datetime | None] = None  # noqa: UP007
-    voyage_timestamp: Union[datetime | None] = None  # noqa: UP007
-    voyage_update_timestamp: Union[datetime | None] = None  # noqa: UP007
+    vessel_ais_class: Union[str , None] = None  # noqa: UP007
+    vessel_flag: Union[str , None] = None  # noqa: UP007
+    vessel_name: Union[str , None] = None  # noqa: UP007
+    vessel_callsign: Union[str , None] = None  # noqa: UP007
+    vessel_timestamp: Union[datetime , None] = None  # noqa: UP007
+    vessel_update_timestamp: Union[datetime , None] = None  # noqa: UP007
+    vessel_ship_type: Union[str , None] = None  # noqa: UP007
+    vessel_sub_ship_type: Union[str , None] = None  # noqa: UP007
+    vessel_mmsi: Union[int , None] = None  # noqa: UP007
+    vessel_imo: Union[int , None] = None  # noqa: UP007
+    vessel_width: Union[int , None] = None  # noqa: UP007
+    vessel_length: Union[int , None] = None  # noqa: UP007
+    position_accuracy: Union[str , None] = None  # noqa: UP007
+    position_collection_type: Union[str , None] = None  # noqa: UP007
+    position_course: Union[float , None] = None  # noqa: UP007
+    position_heading: Union[float , None] = None  # noqa: UP007
+    position_latitude: Union[float , None] = None  # noqa: UP007
+    position_longitude: Union[float , None] = None  # noqa: UP007
+    position_maneuver: Union[str , None] = None  # noqa: UP007
+    position_navigational_status: Union[str , None] = None  # noqa: UP007
+    position_rot: Union[float , None] = None  # noqa: UP007
+    position_speed: Union[float , None] = None  # noqa: UP007
+    position_timestamp: Union[datetime , None] = None  # noqa: UP007
+    position_update_timestamp: Union[datetime , None] = None  # noqa: UP007
+    voyage_destination: Union[str , None] = None  # noqa: UP007
+    voyage_draught: Union[float , None] = None  # noqa: UP007
+    voyage_eta: Union[datetime , None] = None  # noqa: UP007
+    voyage_timestamp: Union[datetime , None] = None  # noqa: UP007
+    voyage_update_timestamp: Union[datetime , None] = None  # noqa: UP007
     created_at: Union[datetime, None] = None  # noqa: UP007
+    batch: Union[str, None] = None
 
     def map_from_spire(spire_update_timestamp: datetime, vessel: dict[str, Any]):  # noqa: ANN201
         def deep_get(dictionary: dict[str, Any], *keys) -> str:

src/bloom/domain/vessel_data.py

@@ -6,18 +6,22 @@
 
 
 class VesselData(BaseModel):
-    id: int | None = None
+    id: Union[ int , None ]
     timestamp: datetime
-    ais_class: str | None
-    flag: str | None
-    name: str | None
-    callsign: str | None
-    timestamp: datetime | None
-    ship_type: str | None
-    sub_ship_type: str | None
-    mmsi: int | None
-    imo: int | None
-    width: int | None
-    length: int | None
+    ais_class:  Union[str , None]
+    flag:  Union[str , None]
+    name:  Union[str , None]
+    callsign:  Union[str , None]
+    timestamp:  Union[datetime , None]
+    ship_type:  Union[str , None]
+    sub_ship_type:  Union[str , None]
+    mmsi:  Union[int , None]
+    imo:  Union[int , None]
+    width:  Union[int , None]
+    length:  Union[int , None]
     vessel_id: int
-    created_at: Union[datetime, None] = None
+    created_at: Union[datetime, None]
+
+    def __init__(self):
+        self.id = None
+        self.created_at = None

src/bloom/domain/vessel_position.py

@@ -9,18 +9,18 @@
 class VesselPosition(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    id: int | None = None
+    id: Union[int , None] = None
     timestamp: datetime
     accuracy: Union[str, None] = None
     collection_type: Union[str, None] = None
     course: Union[float, None] = None
     heading: Union[float, None] = None
     position: Union[Point, None] = None
-    latitude: Union[float | None] = None
-    longitude: Union[float | None] = None
-    maneuver: Union[str | None] = None
-    navigational_status: Union[str | None] = None
-    rot: Union[float | None] = None
-    speed: Union[float | None] = None
+    latitude: Union[float , None] = None
+    longitude: Union[float , None] = None
+    maneuver: Union[str , None] = None
+    navigational_status: Union[str , None] = None
+    rot: Union[float , None] = None
+    speed: Union[float , None] = None
     vessel_id: int
     created_at: Union[datetime, None] = None

src/bloom/domain/vessel_voyage.py

@@ -6,7 +6,7 @@
 
 
 class VesselVoyage(BaseModel):
-    id: int | None = None
+    id: Union[ int , None ] = None
     timestamp: datetime
     destination: str | None
     draught: float | None

src/bloom/domain/white_zone.py

@@ -6,7 +6,7 @@
 
 
 class WhiteZone(BaseModel):
-    id: int | None = None
+    id: Union[ int | None ] = None
     geometry: Union[Geometry, None] = None
     created_at: Union[datetime, None] = None
     updated_at: Union[datetime, None] = None

src/bloom/domain/zone.py

@@ -7,7 +7,7 @@
 
 class Zone(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
-    id: int | None = None
+    id: Union[int , None] = None
     category: str
     sub_category: Union[str, None] = None
     name: str
@@ -16,3 +16,4 @@ class Zone(BaseModel):
     json_data: Union[dict, None] = None
     created_at: Union[datetime, None] = None
     updated_at: Union[datetime, None] = None
+    batch: Union[str, None] = None