samconfig debug level logging fixed; documentation updated

designs/sam-config.md

@@ -9,7 +9,7 @@ Today users of SAM CLI need to invoke the CLI directly with all parameters suppl
 
 for e.g: `sam build --use-container --debug`
 
-But often, during the lifecycle of building and deploying a serverless application. the same commands get run repeatedly to build, package and deploy, before solidifying into the final application. 
+But often, during the lifecycle of building and deploying a serverless application. The same commands get run repeatedly to build, package and deploy, before solidifying into the final application. 
 
 These CLI commands are often long and have many changing parts.
 
@@ -45,16 +45,33 @@ The suite of commands supported by SAM CLI would be aided by looking for a confi
 
 This configuration would be used for specifiying the parameters that each of SAM CLI commands use and would be in TOML format.
 
-Running a SAM CLI command now automatically looks for `samconfig.toml` file and if its finds it goes ahead with parameter passthroughs to the CLI.
+Running a SAM CLI command now automatically looks for `samconfig.toml` file and if it finds it, it passes parameter through to the CLI.
+
+Every command which uses parameters from the configuration file, prints out the location of `samconfig.toml` file it parses.
 
 ```
 sam build
-Default Config file location: samconfig.toml
+Config file location: /home/xxxxxxxxxx/projects/app-samconfig/samconfig.toml
 ..
 ..
 ..
 ```
 
+If no configuration file can be found at the project root directory, the command output will contain a warning.
+
+```
+sam local invoke -t ./out/build/template.yaml 
+Config file '/home/xxxxxxxxx/projects/app-samconfig/out/build/samconfig.toml' does not exist
+..
+..
+..
+```
+
+Where is my `samconfig.toml`?
+---------------------------------
+
+SAM CLI always expects `samconfig.toml` to be in the project root directory (where the template file is located). When neither template nor config file is specified through cli options, both of them are expected to be in the current working directory where SAM CLI command is running. However, if you use `--template-file` to point to the directory without config file, you need to use `--config-file` option to be able to use `samconfig.toml`.
+
 Why `samconfig.toml` not under `.aws-sam` directory?
 ---------------------------------
 

samcli/cli/cli_config_file.py

@@ -57,14 +57,12 @@ def __call__(self, config_path, config_env, cmd_names):
 
         samconfig = SamConfig(config_file_dir, config_file_name)
 
-        # Enable debug level logging by environment variable "SAM_DEBUG"
-        if os.environ.get("SAM_DEBUG", "").lower() == "true":
-            LOG.setLevel(logging.DEBUG)
-
-        LOG.debug("Config file location: %s", samconfig.path())
-
-        if not samconfig.exists():
-            LOG.debug("Config file '%s' does not exist", samconfig.path())
+        # bringing samconfig file location up to info level,
+        # to improve UX and make it clear where we're looking for samconfig file
+        if samconfig.exists():
+            click.echo(f"Config file location: {samconfig.path()}")
+        else:
+            click.secho(f"Config file '{samconfig.path()}' does not exist", fg="yellow")
             return resolved_config
 
         try:
@@ -236,8 +234,10 @@ def decorator_customize_config_file(f):
     config_file_param_decls = ("--config-file",)
     config_file_attrs["help"] = (
         "The path and file name of the configuration file containing default parameter values to use. "
-        "Its default value is 'samconfig.toml' in project directory. For more information about configuration files, "
-        "see: "
+        "Its default value is 'samconfig.toml' in project root directory. Project root directory is defined by the "
+        "template file location. When you use config file and specify --template-file SAM CLI expects samconfig.toml "
+        "and the template file to be in the same directory. Alternatively, if --config-file is explicitly specified, "
+        "it can point to a custom samconfig.toml location. For more information about configuration files, see "
         "https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-config.html."
     )
     config_file_attrs["default"] = "samconfig.toml"

samcli/cli/main.py

@@ -74,10 +74,6 @@ def print_cmdline_args(func):
     """
 
     def wrapper(*args, **kwargs):
-        if kwargs.get("config_file") and kwargs.get("config_env"):
-            config_file = kwargs["config_file"]
-            config_env = kwargs["config_env"]
-            LOG.debug("Using config file: %s, config environment: %s", config_file, config_env)
         LOG.debug("Expand command line arguments to:")
         cmdline_args_log = ""
         for key, value in kwargs.items():
@@ -115,8 +111,17 @@ def cli(ctx):
 
     The AWS Serverless Application Model extends AWS CloudFormation to provide a simplified way of defining the
     Amazon API Gateway APIs, AWS Lambda functions, and Amazon DynamoDB tables needed by your serverless application.
+
+    SAM CLI commands run in the project root directory which is the directory with SAM template file
+    (template.{yml|yaml|json}). If no template file is specified explicitly, SAM CLI looks it up in the current
+    working directory (where SAM CLI is running).
+
+    SAM CLI options can be either passed directly to the commands and/or stored in the config file (samconfig.toml),
+    which is expected to be in the project root directory by default. It is also possible to specify a custom directory
+    for the config file if necessary.
+
     You can find more in-depth guide about the SAM specification here:
-    https://github.com/awslabs/serverless-application-model.
+    https://github.com/aws/serverless-application-model.
     """
     if global_cfg.telemetry_enabled is None:
         enabled = True

samcli/cli/options.py

@@ -20,8 +20,14 @@ def callback(ctx, param, value):
         state.debug = value
         return value
 
+    # NOTE: --debug option should be eager to be evaluated before other parameters and to set log level to DEBUG
+    # before any other option/parameter processing will require to output debug info.
+    # Otherwise parameters are evaluated according to their order and if --debug is specified at the end of the command
+    # some debug output can be lost
+    # https://click.palletsprojects.com/en/7.x/advanced/#callback-evaluation-order
     return click.option(
         "--debug",
+        is_eager=True,
         expose_value=False,
         is_flag=True,
         envvar="SAM_DEBUG",

samcli/commands/_utils/options.py

@@ -217,7 +217,12 @@ def template_click_option(include_build=True):
         callback=partial(get_or_default_template_file_name, include_build=include_build),
         show_default=True,
         is_eager=True,
-        help="AWS SAM template which references built artifacts for resources in the template. (if applicable)"
+        help="AWS SAM template which references built artifacts for resources in the template (if applicable). "
+        "Template file defines the root directory of the project and allows to point SAM CLI to the directory "
+        "for build, local invocation etc. If template file is not specified explicitly SAM CLI expects it to be "
+        "in the current working directory (where it is running). When you use config file and specify --template-file "
+        "SAM CLI expects samconfig.toml and the template file to be in the same directory."
+        "Alternatively, if --config-file is explicitly specified, it can point to a custom samconfig.toml location."
         if include_build
         else "AWS SAM template file.",
     )

tests/integration/buildcmd/build_integ_base.py

@@ -172,6 +172,8 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over
         process_execute.process.wait()
 
         process_stdout = process_execute.stdout.decode("utf-8")
+        if process_stdout.startswith("Config file"):
+            *_, process_stdout = process_stdout.partition("\n")
         self.assertEqual(json.loads(process_stdout), expected_result)
 
 

tests/integration/buildcmd/test_build_cmd.py

@@ -1,32 +1,34 @@
-import re
-import shutil
-import sys
-import os
+"""
+Integration tests for Build comand
+"""
+
 import logging
+import os
 import random
-from unittest import skipIf
+import shutil
+import sys
 from pathlib import Path
-from parameterized import parameterized, parameterized_class
-from subprocess import Popen, PIPE, TimeoutExpired
+from unittest import skipIf
 
 import pytest
-
+from parameterized import parameterized, parameterized_class
 from samcli.lib.utils import osutils
-from .build_integ_base import (
-    BuildIntegBase,
-    DedupBuildIntegBase,
-    CachedBuildIntegBase,
-    BuildIntegRubyBase,
-    NestedBuildIntegBase,
-    IntrinsicIntegBase,
-)
 from tests.testing_utils import (
+    CI_OVERRIDE,
     IS_WINDOWS,
     RUNNING_ON_CI,
-    CI_OVERRIDE,
-    run_command,
-    SKIP_DOCKER_TESTS,
     SKIP_DOCKER_MESSAGE,
+    SKIP_DOCKER_TESTS,
+    run_command,
+)
+
+from .build_integ_base import (
+    BuildIntegBase,
+    BuildIntegRubyBase,
+    CachedBuildIntegBase,
+    DedupBuildIntegBase,
+    IntrinsicIntegBase,
+    NestedBuildIntegBase,
 )
 
 LOG = logging.getLogger(__name__)
@@ -188,8 +190,8 @@ def test_unsupported_runtime(self):
         LOG.info(cmdlist)
         process_execute = run_command(cmdlist, cwd=self.working_dir)
         self.assertEqual(1, process_execute.process.returncode)
-
-        self.assertIn("Build Failed", str(process_execute.stdout))
+        output = "\n".join(process_execute.stdout.decode("utf-8").strip().splitlines())
+        self.assertIn("Build Failed", output)
 
 
 @skipIf(
@@ -1141,7 +1143,8 @@ def test_with_wrong_builder_specified_python_runtime(self, use_container):
         # This will error out.
         command = run_command(cmdlist, cwd=self.working_dir)
         self.assertEqual(command.process.returncode, 1)
-        self.assertEqual(command.stdout.strip(), b"Build Failed")
+        output = "\n".join(command.stdout.decode("utf-8").strip().splitlines())
+        self.assertIn("Build Failed", output)
 
     def _verify_built_artifact(self, build_dir, function_logical_id, expected_files):
 

tests/integration/init/test_init_command.py

@@ -1,13 +1,15 @@
-from unittest import TestCase
-
-from parameterized import parameterized
-from subprocess import Popen, TimeoutExpired, PIPE
+"""
+Integration tests for init command
+"""
 import os
 import shutil
 import tempfile
-from samcli.lib.utils.packagetype import IMAGE, ZIP
-
 from pathlib import Path
+from subprocess import PIPE, Popen, TimeoutExpired
+from unittest import TestCase
+
+from parameterized import parameterized
+from samcli.lib.utils.packagetype import IMAGE, ZIP
 
 TIMEOUT = 300
 
@@ -225,7 +227,7 @@ def test_init_command_no_interactive_missing_name(self):
 You can also re-run without the --no-interactive flag to be prompted for required values.
             """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_apptemplate_location(self):
         stderr = None
@@ -263,7 +265,7 @@ def test_init_command_no_interactive_apptemplate_location(self):
     --location
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_runtime_location(self):
         stderr = None
@@ -301,7 +303,7 @@ def test_init_command_no_interactive_runtime_location(self):
     --location
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_base_image_location(self):
         stderr = None
@@ -339,7 +341,7 @@ def test_init_command_no_interactive_base_image_location(self):
     --location
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_base_image_no_dependency(self):
         stderr = None
@@ -379,7 +381,7 @@ def test_init_command_no_interactive_base_image_no_dependency(self):
 You can also re-run without the --no-interactive flag to be prompted for required values.
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_packagetype_location(self):
         stderr = None
@@ -417,7 +419,7 @@ def test_init_command_no_interactive_packagetype_location(self):
     --location
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_no_interactive_base_image_no_packagetype(self):
         stderr = None
@@ -455,7 +457,7 @@ def test_init_command_no_interactive_base_image_no_packagetype(self):
 You can also re-run without the --no-interactive flag to be prompted for required values.
                         """
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
     def test_init_command_wrong_packagetype(self):
         stderr = None
@@ -489,7 +491,7 @@ def test_init_command_wrong_packagetype(self):
                 _get_command()
             )
 
-            self.assertEqual(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
+            self.assertIn(errmsg.strip(), "\n".join(stderr.strip().splitlines()))
 
 
 class TestInitWithArbitraryProject(TestCase):