Feature/approve wildcards

src/confcom/HISTORY.rst

@@ -2,6 +2,11 @@
 
 Release History
 ===============
+0.2.12
+* adding ability for mixed-mode OCI image pulling, e.g. using tar files and remote registries in the same template
+* adding option to use allow-all regex for environment variables
+* tar file bug fixes
+
 0.2.11
 * bug fix for clean room scenario where non-existent docker client connection attempted to be closed
 * adding ability for ARM Template workflows to use regex for environment variables

src/confcom/README.md

@@ -4,6 +4,7 @@
   - [Repository](#repository)
   - [Prerequisites](#prerequisites)
   - [Installation Instructions (End User)](#installation-instructions-end-user)
+  - [Current Limitations](#current-limitations)
   - [Trademarks](#trademarks)
 
 ## Repository
@@ -48,6 +49,13 @@
    az extension add -n confcom
    ```
 
+## Current Limitations
+
+The `confcom` extension does not currently support:
+
+- [ARM Template functions](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/template-functions) other than `variables` and `parameters`.
+- Variables and Parameters with non-primitive data types e.g. objects and arrays
+
 ## Trademarks
 
 This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft

src/confcom/azext_confcom/_help.py

@@ -49,6 +49,10 @@
           type: boolean
           short-summary: 'When enabled, the generated security policy adds the ability to use /bin/sh or /bin/bash to debug the container. It also enabled stdio access, ability to dump stack traces, and enables runtime logging. It is recommended to only use this option for debugging purposes.'
 
+        - name: --approve-wildcards -y
+          type: boolean
+          short-summary: 'When enabled, all prompts for using wildcards in environment variables are automatically approved.'
+
         - name: --disable-stdio
           type: boolean
           short-summary: 'When enabled, the containers in the container group do not have access to stdio.'

src/confcom/azext_confcom/_params.py

@@ -64,6 +64,12 @@ def load_arguments(self, _):
             required=False,
             help="Debug mode will enable processes in a container group that are helpful for debugging",
         )
+        c.argument(
+            "approve_wildcards",
+            options_list=("--approve-wildcards", "-y"),
+            required=False,
+            help="Approving wildcards by default will get rid of the prompts during the wildcard environment variable use case and auto-approve the use of wildcards",
+        )
         c.argument(
             "disable_stdio",
             options_list=("--disable-stdio",),

src/confcom/azext_confcom/config.py

@@ -70,6 +70,8 @@
 POLICY_FIELD_CONTAINERS_ELEMENTS_COMMANDS = "command"
 POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS = "env_rules"
 POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_STRATEGY = "strategy"
+POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_VALUE = "value"
+POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_NAME = "name"
 POLICY_FIELD_CONTAINERS_ELEMENTS_ENVS_RULE = "pattern"
 POLICY_FIELD_CONTAINERS_ELEMENTS_REQUIRED = "required"
 POLICY_FIELD_CONTAINERS_ELEMENTS_LAYERS = "layers"

src/confcom/azext_confcom/container.py

@@ -7,7 +7,7 @@
 import json
 import os
 from typing import Any, List, Dict
-from azext_confcom.template_util import case_insensitive_dict_get
+from azext_confcom.template_util import case_insensitive_dict_get, replace_params_and_vars
 from azext_confcom import config
 from azext_confcom.errors import eprint
 
@@ -368,6 +368,27 @@ def get_mounts(self) -> List:
     def set_extra_environment_rules(self, rules: Dict) -> None:
         self._extraEnvironmentRules = rules
 
+    def parse_all_parameters_and_variables(self, params, vars_dict) -> None:
+        field_names = [
+            "containerImage",
+            "_environmentRules",
+            "_command",
+            "_workingDir",
+            "_mounts",
+            "_identifier",
+            "_exec_processes",
+            "_extraEnvironmentRules",
+        ]
+        for field_name in field_names:
+            attribute = getattr(self, field_name)
+            out = replace_params_and_vars(params, vars_dict, attribute)
+            setattr(self, field_name, out)
+        # set these at the end since they're derived from containerImage, which could have been altered
+        if ":" in self.containerImage:
+            self.base, self.tag = self.containerImage.split(":", 1)
+        else:
+            self.base, self.tag = self.containerImage, "latest"
+
     def _get_environment_rules(self) -> List[Dict[str, Any]]:
         out_rules = copy.deepcopy(self._environmentRules)
         env_var_names = [
@@ -469,7 +490,7 @@ def from_json(
     def __init__(
         self,
         containerImage: str,
-        environmentRules: Dict,
+        environmentRules: List[Dict],
         command: List[str],
         mounts: List[Dict],
         workingDir: str,

src/confcom/azext_confcom/custom.py

@@ -26,6 +26,7 @@ def acipolicygen_confcom(
     image_name: str,
     infrastructure_svn: str,
     tar_mapping_location: str,
+    approve_wildcards: str = False,
     use_json: bool = False,
     outraw: bool = False,
     outraw_pretty_print: bool = False,
@@ -97,6 +98,7 @@ def acipolicygen_confcom(
             arm_template_parameters,
             debug_mode=debug_mode,
             disable_stdio=disable_stdio,
+            approve_wildcards=approve_wildcards,
         )
     elif image_name:
         container_group_policies = security_policy.load_policy_from_image_name(

src/confcom/azext_confcom/data/internal_config.json

@@ -1,5 +1,5 @@
 {
-    "version": "0.2.11",
+    "version": "0.2.12",
     "hcsshim_config": {
         "maxVersion": "1.0.0",
         "minVersion": "0.0.1"

src/confcom/azext_confcom/os_util.py

@@ -98,7 +98,7 @@ def map_image_from_tar(image_name: str, tar: TarFile, tar_location: str):
     ]
     info_file = None
     # if there's more than one image in the tarball, we need to do some more logic
-    if len(info_file_name) > 1:
+    if len(info_file_name) > 0:
         # extract just the manifest file and see if any of the RepoTags match the image_name we're searching for
         # the manifest.json should have a list of all the image tags
         # and what json files they map to to get env vars, startup cmd, etc.
@@ -114,13 +114,11 @@ def map_image_from_tar(image_name: str, tar: TarFile, tar_location: str):
                 break
         # remove the extracted manifest file to clean up
         os.remove(manifest_path)
-    elif len(info_file_name) == 0:
-        eprint(f"Tarball at {tar_location} contains no images")
     else:
-        info_file = info_file_name[0]
+        eprint(f"Tarball at {tar_location} contains no images")
 
     if not info_file:
-        eprint(f"Image {image_name} is not found in tarball at {tar_location}")
+        return None
     tar.extract(info_file.name, path=tar_dir)
 
     # get the path of the json file and read it in

src/confcom/azext_confcom/rootfs_proxy.py

@@ -85,11 +85,13 @@ def get_policy_image_layers(
             output = [output[j * 2 + 1] for j in range(len(output) // 2)]
             output = [i.rstrip("\n").split(": ", 1)[1] for i in output]
         else:
-            eprint(
-                "Cannot get layer hashes. Please check whether the image exists in local repository/daemon."
-            )
+            output = []
+            # eprint(
+            #     "Cannot get layer hashes. Please check whether the image exists in local repository/daemon."
+            # )
 
         if err.decode("utf8") != "":
-            eprint(err.decode("utf8"))
+            output = []
+            # eprint(err.decode("utf8"))
 
         return output

src/confcom/azext_confcom/security_policy.py

@@ -20,7 +20,6 @@
 from azext_confcom.template_util import (
     extract_confidential_properties,
     is_sidecar,
-    parse_template,
     pretty_print_func,
     print_func,
     readable_diff,
@@ -31,7 +30,8 @@
     process_mounts,
     extract_probe,
     process_env_vars_from_template,
-    get_image_info
+    get_image_info,
+    get_tar_location_from_mapping
 )
 from azext_confcom.rootfs_proxy import SecurityPolicyProxy
 
@@ -45,6 +45,9 @@ class OutputType(Enum):
 
 
 class AciPolicy:  # pylint: disable=too-many-instance-attributes
+    all_params = {}
+    all_vars = {}
+
     def __init__(
         self,
         deserialized_config: Any,
@@ -393,12 +396,9 @@ def populate_policy_content_for_all_images(
             action="ignore", message="unclosed", category=ResourceWarning
         )
 
-        client = None
         tar_location = ""
         layer_cache = {}
-        if not tar_mapping:
-            client = self._get_docker_client()
-        elif isinstance(tar_mapping, str):
+        if isinstance(tar_mapping, str):
             tar_location = tar_mapping
         proxy = self._get_rootfs_proxy()
         container_images = self.get_images()
@@ -419,9 +419,9 @@ def populate_policy_content_for_all_images(
             message_queue = []
             # populate regular container images(s)
             for image in container_images:
-
+                image.parse_all_parameters_and_variables(AciPolicy.all_params, AciPolicy.all_vars)
                 image_name = f"{image.base}:{image.tag}"
-                image_info = get_image_info(progress, message_queue, client, tar_mapping, image)
+                image_info, tar = get_image_info(progress, message_queue, tar_mapping, image)
 
                 # verify and populate the working directory property
                 if not image.get_working_dir() and image_info:
@@ -471,12 +471,15 @@ def populate_policy_content_for_all_images(
                                 }
                             )
 
+                # populate tar location
+                if isinstance(tar_mapping, dict):
+                    tar_location = get_tar_location_from_mapping(tar_mapping, image_name)
                 # populate layer info
                 if layer_cache.get(image_name):
                     image.set_layers(layer_cache.get(image_name))
                 else:
                     image.set_layers(proxy.get_policy_image_layers(
-                        image.base, image.tag, tar_location=tar_location
+                        image.base, image.tag, tar_location=tar_location if tar else ""
                     ))
                     layer_cache[image_name] = image.get_layers()
                 progress.update()
@@ -487,7 +490,7 @@ def populate_policy_content_for_all_images(
             for message in message_queue:
                 logger.warning(message)
 
-    def get_images(self) -> List[Any]:
+    def get_images(self) -> List[ContainerImage]:
         return self._images
 
     def pull_image(self, image: ContainerImage) -> Any:
@@ -501,6 +504,7 @@ def load_policy_from_arm_template_str(
     infrastructure_svn: str = None,
     debug_mode: bool = False,
     disable_stdio: bool = False,
+    approve_wildcards: bool = False,
 ) -> List[AciPolicy]:
     """Function that converts ARM template string to an ACI Policy"""
     input_arm_json = os_util.load_json_from_str(template_data)
@@ -537,9 +541,8 @@ def load_policy_from_arm_template_str(
 
     get_values_for_params(input_parameter_json, all_params)
 
-    input_arm_json = parse_template(all_params,
-                                    case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_VARIABLES)
-                                    or {}, input_arm_json)
+    AciPolicy.all_params = all_params
+    AciPolicy.all_vars = case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_VARIABLES) or {}
 
     container_groups = []
 
@@ -612,7 +615,8 @@ def load_policy_from_arm_template_str(
                 {
                     config.ACI_FIELD_CONTAINERS_ID: image_name,
                     config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE: image_name,
-                    config.ACI_FIELD_CONTAINERS_ENVS: process_env_vars_from_template(image_properties),
+                    config.ACI_FIELD_CONTAINERS_ENVS: process_env_vars_from_template(
+                        AciPolicy.all_params, AciPolicy.all_vars, image_properties, approve_wildcards),
                     config.ACI_FIELD_CONTAINERS_COMMAND: case_insensitive_dict_get(
                         image_properties, config.ACI_FIELD_TEMPLATE_COMMAND
                     )
@@ -651,6 +655,7 @@ def load_policy_from_arm_template_file(
     parameter_path: str,
     debug_mode: bool = False,
     disable_stdio: bool = False,
+    approve_wildcards: bool = False,
 ) -> List[AciPolicy]:
     """Utility function: generate policy object from given arm template and parameter file paths"""
     input_arm_json = os_util.load_str_from_file(template_path)
@@ -659,7 +664,7 @@ def load_policy_from_arm_template_file(
         input_parameter_json = os_util.load_str_from_file(parameter_path)
     return load_policy_from_arm_template_str(
         input_arm_json, input_parameter_json, infrastructure_svn,
-        debug_mode=debug_mode, disable_stdio=disable_stdio
+        debug_mode=debug_mode, disable_stdio=disable_stdio, approve_wildcards=approve_wildcards,
     )