fix and update get_schema lambda (#2261)

* fix get schema
* use DataProductSchema to fetch schema

containers/daap-get-schema/CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.2.0] - 2023-11-08
+
+- refactor get_schema to use `DataProductSchema` to fetch schema
 - update Dockerfile COPY command fixing `LAMBDA_TASK_ROOT` typo
 
 ## [1.1.0] - 2023-10-18

containers/daap-get-schema/config.json

@@ -1,6 +1,6 @@
 {
     "name": "daap-get-schema",
-    "version": "1.1.0",
+    "version": "1.2.0",
     "registry": "ecr",
     "ecr": {
         "role": "arn:aws:iam::013433889002:role/modernisation-platform-oidc-cicd",

containers/daap-get-schema/src/var/task/get_schema.py

@@ -1,13 +1,9 @@
-import json
 import os
 from http import HTTPStatus
 
-import botocore
-from data_platform_api_responses import format_response_json, response_status_404
+from data_platform_api_responses import format_error_response, format_response_json
 from data_platform_logging import DataPlatformLogger
-from data_platform_paths import DataProductConfig
-from data_product_metadata import format_table_schema
-from dataengineeringutils3.s3 import read_json_from_s3
+from data_product_metadata import DataProductSchema, format_table_schema
 
 
 def handler(event, context):
@@ -23,21 +19,22 @@ def handler(event, context):
         },
     )
 
-    config = DataProductConfig(data_product_name)
-    schema_path = config.schema_path(table_name)
-
     try:
-        registered_glue_schema = read_json_from_s3(schema_path.uri)
-    except botocore.exceptions.ClientError as e:
-        if e.response["Error"]["Code"] == "NoSuchKey":
-            message = f"Schema not found for data product '{data_product_name}', table '{table_name}'"
-            logger.error(f"{message} ({schema_path.uri})")
-            return response_status_404(message)
-        else:
-            raise
+        registered_glue_schema = (
+            DataProductSchema(
+                data_product_name=data_product_name,
+                table_name=table_name,
+                logger=logger,
+                input_data=None,
+            )
+            .load()
+            .latest_version_saved_data
+        )
+    except Exception:
+        message = f"no existing table schema found in S3 for {table_name=} {data_product_name=}"
+        logger.info(message)
+        return format_error_response(HTTPStatus.NOT_FOUND, event, message)
 
     registered_schema = format_table_schema(registered_glue_schema)
 
-    return format_response_json(
-        status_code=HTTPStatus.OK, json_body=json.dumps(registered_schema)
-    )
+    return format_response_json(status_code=HTTPStatus.OK, body=registered_schema)

containers/daap-get-schema/tests/unit/mock_test.py

@@ -1,4 +1,5 @@
 import json
+from http import HTTPStatus
 from unittest.mock import patch
 
 import pytest
@@ -56,15 +57,18 @@ def test_valid(
             assert result == {
                 "body": json.dumps(out_schema),
                 "headers": {"Content-Type": "application/json"},
-                "statusCode": 200,
+                "statusCode": HTTPStatus.OK,
             }
 
     def test_missing(self, fake_context):
+        data_product_name = "abc"
+        table_name = "def"
+
         result = handler(
             {
                 "pathParameters": {
-                    "data-product-name": "abc",
-                    "table-name": "def",
+                    "data-product-name": data_product_name,
+                    "table-name": table_name,
                 }
             },
             fake_context,
@@ -74,10 +78,10 @@ def test_missing(self, fake_context):
             "body": json.dumps(
                 {
                     "error": {
-                        "message": "Schema not found for data product 'abc', table 'def'"
+                        "message": f"no existing table schema found in S3 for {table_name=} {data_product_name=}"
                     }
                 }
             ),
             "headers": {"Content-Type": "application/json"},
-            "statusCode": 404,
+            "statusCode": HTTPStatus.NOT_FOUND,
         }