official release of v3.2.1 (#1236)

* bumping version to 3.2.1

* add two lambda layers to improve deploy time (#1235)

* adding support for lambda layers for precompiled dependencies

* update to necessary lambdas to use lambda layers (apps)

* removing logic for precompiled packages

* trying not to suck

streamalert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '3.2.0'
+__version__ = '3.2.1'

streamalert/_vendored/README.rst → streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/README.rst

@@ -1,6 +1,10 @@
 How to Update Precompiled Dependencies
 ######################################
 
+For dependencies included in zips to be usable with Lambda Layers, files must reside within a ``python`` directory. See the
+AWS `documentation <https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path>`_
+for more information.
+
 
 Building Dependencies Using EC2
 ===============================
@@ -36,10 +40,10 @@ On EC2 Instance
   $ pip install --upgrade pip setuptools
 
   # Make a temp build directory and temp pip install directory
-  $ mkdir $HOME/build_temp $HOME/pip_temp
+  $ mkdir -p $HOME/build_temp $HOME/pip_temp/python
 
   # Install all of the dependencies to this directory
-  $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp
+  $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp/python
 
   # Replace the `boxsdk[jwt]==2.6.1` below with the desired package & version
   # For example, the following would update the aliyun dependencies:
@@ -104,8 +108,8 @@ SSH and Build Dependencies
   # upgrade pip and setuptools if neccessary
   $ pip install --upgrade pip setuptools
 
-  $ mkdir $HOME/build_temp $HOME/pip_temp
-  $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp
+  $ mkdir -p $HOME/build_temp $HOME/pip_temp/python
+  $ pip install boxsdk[jwt]==2.6.1 --build $HOME/build_temp/ --target $HOME/pip_temp/python
 
   # Replace the `boxsdk[jwt]==2.6.1` below with the desired package & version
   # For example, the following would update the aliyun dependencies:
@@ -126,7 +130,7 @@ Copy the `pip.zip` file from the virtual machine to the local host.
 
 .. code-block:: bash
 
-  $ cp pip.zip /vagrant/streamalert/_vendored/boxsdk[jwt]==2.6.1_dependencies.zip
+  $ cp pip.zip /vagrant/streamalert_cli/_infrastructure/modules/tf_globals/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip
   $ exit  # exit the session
 
 

streamalert_cli/_infrastructure/modules/tf_globals/main.tf

@@ -71,3 +71,15 @@ resource "aws_dynamodb_table" "rules_table" {
     Name = "StreamAlert"
   }
 }
+
+resource "aws_lambda_layer_version" "aliyun_dependencies" {
+  filename            = "${path.module}/lambda_layers/aliyun-python-sdk-actiontrail==2.0.0_dependencies.zip"
+  layer_name          = "aliyun"
+  compatible_runtimes = ["python3.7"]
+}
+
+resource "aws_lambda_layer_version" "box_dependencies" {
+  filename            = "${path.module}/lambda_layers/boxsdk[jwt]==2.6.1_dependencies.zip"
+  layer_name          = "box"
+  compatible_runtimes = ["python3.7"]
+}

streamalert_cli/_infrastructure/modules/tf_globals/output.tf

@@ -13,3 +13,10 @@ output "classifier_sqs_queue_arn" {
 output "classifier_sqs_sse_kms_key_arn" {
   value = module.classifier_queue.sqs_sse_kms_key_arn
 }
+
+output "lamdba_layer_arns" {
+  value = [
+    aws_lambda_layer_version.aliyun_dependencies.arn,
+    aws_lambda_layer_version.box_dependencies.arn,
+  ]
+}

streamalert_cli/_infrastructure/modules/tf_lambda/main.tf

@@ -34,6 +34,8 @@ resource "aws_lambda_function" "function" {
     security_group_ids = var.vpc_security_group_ids
   }
 
+  layers = var.layers
+
   tags = local.tags
 }
 

streamalert_cli/_infrastructure/modules/tf_lambda/variables.tf

@@ -12,6 +12,12 @@ variable "runtime" {
   description = "Function runtime environment"
 }
 
+variable "layers" {
+  type        = list(string)
+  default     = []
+  description = "List of Lambda Layer ARNs to use with this function"
+}
+
 variable "handler" {
   description = "Entry point for the function"
 }

streamalert_cli/manage_lambda/package.py

@@ -16,7 +16,6 @@
 import os
 import shutil
 import tempfile
-import zipfile
 
 from streamalert.shared.logger import get_logger
 from streamalert_cli.helpers import run_command
@@ -54,11 +53,6 @@ class LambdaPackage:
         'pymsteams==0.1.12',
     }
 
-    PRECOMPILED_LIBS = {               # Default precompiled dependencies
-        'aliyun-python-sdk-actiontrail==2.0.0',
-        'boxsdk[jwt]==2.6.1',
-    }
-
     def __init__(self, config):
         self.config = config
         self.temp_package_path = os.path.join(tempfile.gettempdir(), self.package_name)
@@ -78,7 +72,7 @@ def create(self):
             shutil.rmtree(self.temp_package_path)
 
         # Copy all of the default package files
-        self._copy_files(self.DEFAULT_PACKAGE_FILES, ignores={'*dependencies.zip'})
+        self._copy_files(self.DEFAULT_PACKAGE_FILES)
 
         # Copy in any user-specified files
         self._copy_user_config_files()
@@ -87,11 +81,6 @@ def create(self):
             LOGGER.error('Failed to install necessary libraries')
             return False
 
-        # Extract any precompiled libs for this package
-        if not self._extract_precompiled_libs():
-            LOGGER.error('Failed to extract precompiled libraries')
-            return False
-
         # Zip it all up
         # Build these in the top-level of the terraform directory as streamalert.zip
         result = shutil.make_archive(
@@ -119,41 +108,6 @@ def _copy_files(self, paths, ignores=None):
             kwargs = {'ignore': shutil.ignore_patterns(*ignores)} if ignores else dict()
             shutil.copytree(path, os.path.join(self.temp_package_path, path), **kwargs)
 
-    def _extract_precompiled_libs(self):
-        """Extract any precompiled libraries into the deployment package folder
-
-        Returns:
-            bool: True if precompiled libs were extracted successfully, False if some are missing
-        """
-        vendored_dir = os.path.join('streamalert', '_vendored')
-        found_libs = set()
-        for zipped_file in os.listdir(vendored_dir):
-            # Skip files that are not explicitly deps
-            if not zipped_file.endswith('_dependencies.zip'):
-                continue
-
-            value = zipped_file.rstrip('_dependencies.zip')
-            if value not in self.PRECOMPILED_LIBS:
-                LOGGER.error(
-                    'Found precompiled libraries not included in PRECOMPILED_LIBS: %s', value
-                )
-                return False
-
-            LOGGER.info('Extracting precompiled library: %s', zipped_file)
-
-            # Copy the contents of the dependency zip to the package directory
-            with zipfile.ZipFile(os.path.join(vendored_dir, zipped_file), 'r') as libs_file:
-                libs_file.extractall(self.temp_package_path)
-
-            found_libs.add(value)
-
-        diff = self.PRECOMPILED_LIBS.difference(found_libs)
-        if diff:
-            LOGGER.error('Missing required precompiled libraries: %s', ', '.join(diff))
-            return False
-
-        return True
-
     def _resolve_libraries(self):
         """Install all libraries into the deployment package folder
 

streamalert_cli/terraform/apps.py

@@ -59,5 +59,6 @@ def generate_apps(cluster_name, cluster_dict, config):
             'streamalert.apps.main.handler',
             config['clusters'][cluster_name]['modules']['streamalert_apps'][function_name],
             config,
-            input_event=app_config
+            input_event=app_config,
+            include_layers=True,
         )

streamalert_cli/terraform/lambda_module.py

@@ -52,7 +52,7 @@ def _tf_vpc_config(lambda_config):
 
 
 def generate_lambda(function_name, handler, lambda_config, config, environment=None,
-                    input_event=None, tags=None, zip_file=None):
+                    input_event=None, tags=None, **kwargs):
     """Generate an instance of the Lambda Terraform module.
 
     Args:
@@ -63,6 +63,9 @@ def generate_lambda(function_name, handler, lambda_config, config, environment=N
         environment (dict): Optional environment variables to specify.
             ENABLE_METRICS and LOGGER_LEVEL are included automatically.
         tags (dict): Optional tags to be added to this Lambda resource.
+
+    Keyword Args:
+        include_layers (bool): Optionally include the default Lambda Layers (default: False)
         zip_file (str): Optional name for the .zip of deployment package (default: streamalert.zip)
 
     Example Lambda config:
@@ -121,9 +124,12 @@ def generate_lambda(function_name, handler, lambda_config, config, environment=N
         'tags': tags or {},
     }
 
+    if kwargs.get('include_layers', False):
+        lambda_module['layers'] = '${module.globals.lamdba_layer_arns}'
+
     # The lambda module defaults to using the 'streamalert.zip' file that is created
-    if zip_file:
-        lambda_module['filename'] = zip_file
+    if kwargs.get('zip_file'):
+        lambda_module['filename'] = kwargs.get('zip_file')
 
     # Add Classifier input config from the loaded cluster file
     input_config = lambda_config.get('inputs')