Gwright99/87 streamline number of locals

.githooks/data_external/generate_db_connection_string.py

@@ -3,13 +3,35 @@
 import json
 import sys
 from types import SimpleNamespace
-from typing import List
 
 sys.dont_write_bytecode = True
 
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
+
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
 # Much simpler way to get variable passed in (via Terraform sending to stdin)
 query = json.load(sys.stdin)
-data = SimpleNamespace(**query)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
 
 # Get engine_version depending on whether container DB or RDS instance is in play
 engine_version = data.db_container_engine_version if data.flag_use_container_db else data.db_engine_version
@@ -19,21 +41,11 @@
 v24plus_connstring = "permitMysqlScheme=true"
 
 
-def return_tf_payload(status: str, value: str):
-    payload = {'status': status, 'value': value}
-    print(json.dumps(payload))
-
-
 def generate_connection_string(mysql8: str, v24plus: str):
     connection_string = ""
-    add_mysql8 = False
-    add_v24plus = False
 
-    if mysql8.startswith("8."): 
-        add_mysql8 = True
-
-    if v24plus >= "v24":
-        add_v24plus = True
+    add_mysql8 = True if mysql8.startswith("8.") else False
+    add_v24plus = True if v24plus >= "v24" else False
 
     if add_mysql8 and add_v24plus:
         connection_string = f"?{mysql8_connstring}&{v24plus_connstring}"
@@ -42,11 +54,13 @@ def generate_connection_string(mysql8: str, v24plus: str):
     elif not add_mysql8 and add_v24plus:
         connection_string = f"?{v24plus_connstring}"
 
-    return connection_string
+    values = {
+        "connection_string": connection_string
+    }
+
+    return values
 
 
 if __name__ == '__main__':
-    connection_string = generate_connection_string(engine_version, data.tower_container_version)
-    return_tf_payload("0", connection_string)
-
-    exit(0)
+    values = generate_connection_string(engine_version, data.tower_container_version)
+    return_tf_payload("0", values)

.githooks/data_external/generate_dns_values.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
+
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+# Vars
+dns_zone_mappings = {
+    "flag_create_route53_private_zone":             ("zone_private_new",        [0, 'id']),
+    "flag_use_existing_route53_private_zone":       ("zone_private_existing",   [0, 'id']),
+    "flag_use_existing_route53_public_zone":        ("zone_public_existing",    [0, 'id']),
+}
+
+# Transformed dictionary nested into flat dicionary so value can be passed around.
+# I don't love this approach but can't find a cleaner/simpler way right now.
+dns_instance_ip_mappings = {
+    "flag_make_instance_private":                   ("tower_host_instance", ['private_ip']),
+    "flag_make_instance_private_behind_public_alb": ("tower_host_instance", ['private_ip']),
+    "flag_private_tower_without_eice":              ("tower_host_instance", ['private_ip']),
+    "flag_make_instance_public":                    ("tower_host_eip",      [0, 'public_ip'])
+}
+
+
+def populate_values(query):
+
+    external_logger.debug(f"Query is: {query}")
+
+    # Baseline variables
+    dns_zone_id = ""
+    dns_instance_ip = ""
+
+    for k,v in dns_zone_mappings.items():
+        external_logger.debug(f"k is : {k}; and v is: {v}")
+        tf_obj, dpath = v
+        tf_obj_json = json.loads(query[tf_obj])
+        dns_zone_id = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
+
+
+    for k,v in dns_instance_ip_mappings.items():
+        '''`aws_instance.ec2.private_ip` vs `aws_eip.towerhost[0].public_ip`'''
+        tf_obj, dpath = v
+        tf_obj_json = json.loads(query[tf_obj])
+        dns_instance_ip = getDVal(tf_obj_json, dpath) if data_dictionary[k] else dns_instance_ip
+
+    values = {
+        "dns_zone_id": dns_zone_id,
+        "dns_instance_ip": dns_instance_ip
+    }
+
+    return values
+
+
+if __name__ == '__main__':
+    values = populate_values(query)
+    return_tf_payload("0", values)

.githooks/data_external/generate_flags.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
+
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+def populate_values(query):
+
+    external_logger.debug(f"Query is: {query}")
+
+    # Determine kinda of DNS record to create
+    dns_create_alb_record = True if (data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+    dns_create_ec2_record = True if (not data.flag_create_load_balancer and not data.flag_create_hosts_file_entry) else False
+
+    values = {
+        "dns_create_alb_record": dns_create_alb_record,
+        "dns_create_ec2_record": dns_create_ec2_record,
+    }
+
+    return values
+
+
+if __name__ == '__main__':
+    values = populate_values(query)
+    return_tf_payload("0", values)

.githooks/data_external/template_get_tfvars_and_query.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+from types import SimpleNamespace
+
+sys.dont_write_bytecode = True
+
+
+## ------------------------------------------------------------------------------------
+## Extract TFvars and get query values
+## ------------------------------------------------------------------------------------
+# NOTE! This part is a bit convoluted but there's a method to the madness:
+#   - Script executes from `~/cx-field-tools-installer`. 
+#   - Change into .githooks (to avoid the `.` import problem), import, return to project root to extract tfvars.
+#   - Error thrown without workaround: `ImportError: attempted relative import with no known parent package`
+#   - Query object to receive objects via TF data call (i.e. resources)
+project_root = os.getcwd()
+os.chdir(f"{project_root}/.githooks")
+sys.path.append(".")
+
+from utils.extractors import get_tfvars_as_json #convert_tfvars_to_dictionary
+from utils.logger import external_logger
+from utils.common_data_external_functions import getDVal, return_tf_payload
+
+os.chdir(project_root)
+data_dictionary = get_tfvars_as_json()
+data = SimpleNamespace(**data_dictionary)
+
+# Much simpler way to get variable passed in (via Terraform sending to stdin)
+query = json.load(sys.stdin)
+# query = SimpleNamespace(**query)
+## ------------------------------------------------------------------------------------
+
+
+def populate_values(query):
+
+    external_logger.debug(f"Query is: {query}")
+
+    # Add logic here
+
+    values = {
+        "key": "value"
+    }
+
+    return values
+
+
+if __name__ == '__main__':
+    values = populate_values(query)
+    return_tf_payload("0", values)
+

.githooks/utils/common_data_external_functions.py

@@ -0,0 +1,45 @@
+# Centralizes common functions which will be needed across scripts called by TF `data.external`.
+import json
+from typing import Any
+
+from utils.logger import external_logger
+
+
+# https://www.reddit.com/r/learnpython/comments/y02net/is_there_a_better_way_to_store_full_dictionary/
+def getDVal(d : dict, listPath : list) -> Any:
+    '''
+    Recursively loop through a dictionary object to get to the target nested key.
+    This allows us to specify a TF objecy and the exact subpath to slice from.
+    Note: The path must be defined as discrete elements in a list.
+    '''
+    for key in listPath:
+        d = d[key]
+    return d
+
+
+def convert_booleans_to_strings(payload: dict) -> dict: 
+    '''
+    TF data.external freaks out if JSON true/false values not returned as strings.
+    Unfortunately, json.dumps will convert True/False to true/false, but not turn to string.
+    This function fixes the json.dumps payload so it's acceptable to TF.
+    '''
+    for k,v in payload.items():
+        external_logger.debug(f"k is {k} and v is {v}.")
+        if isinstance(v, bool):
+            payload[k] = "true" if v == True else "false"
+    return payload
+
+
+def return_tf_payload(status: str, values: dict):
+    '''
+    Package the payload to return to TF data.external.
+    Note: Using external_logger due to the stdout/stderr problem.
+    '''
+    external_logger.debug(f"Payload is: {values}")
+
+    payload = {'status': status, **values}
+    payload = convert_booleans_to_strings(payload)
+    print(json.dumps(payload))
+
+    external_logger.error("Flushing.")
+    exit(0)

.githooks/utils/extractors.py

@@ -65,7 +65,8 @@ def convert_tfvars_to_dictionary(file):
             elif flag_skip_block_comment:
                 indices_to_pop.append(i)
 
-        logger.debug(f"Indices to pop: {indices_to_pop}")
+        # This breaks my `external.data` implementation when logging set to debug. Commenting out.
+        # logger.debug(f"Indices to pop: {indices_to_pop}")
         purge_indices_in_reverse(indices_to_pop)