Setup CloudWatch alarm for ClamAV notifications (#3895)

UPGRADING.rst

@@ -20,6 +20,16 @@ reverted. This is all fairly informal and loosely defined. Hopefully we won't
 have too many entries in this file.
 
 
+#3895 Setup CloudWatch alarm for ClamAV notifications
+=====================================================
+
+Operator
+~~~~~~~~
+
+Manually deploy the ``shared`` and ``gitlab`` components of any main deployment
+just before pushing the merge commit to the GitLab instance in that deployment.
+
+
 #5975 Upgrade ES domain for Hammerbox
 =====================================
 

terraform/gitlab/gitlab.tf.json.template.py

@@ -1813,7 +1813,8 @@ def qq(*words):
                                 '[Unit]',
                                 'Description=Scheduled ClamAV malware scan of entire file system',
                                 '[Timer]',
-                                'OnCalendar=Sun *-*-* 8:0:0',
+                                # Run a clamscan twice daily at 1am and 1pm PST
+                                'OnCalendar=*-*-* 9,21:0:0',
                                 '[Install]',
                                 'WantedBy=timers.target'
                             )

terraform/shared/shared.tf.json.template.py

@@ -524,6 +524,25 @@ def paren(s: str) -> str:
                     'value': 1,
                     'default_value': 0,
                 }
+            },
+            **{
+                name: {
+                    'name': config.qualified_resource_name(name, suffix='.filter'),
+                    'pattern': pattern,
+                    'log_group_name': '/aws/cwagent/azul-gitlab',
+                    'metric_transformation': {
+                        'name': config.qualified_resource_name(name),
+                        'namespace': 'LogMetrics',
+                        'value': 1,
+                        'default_value': 0,
+                    }
+                }
+                for name, pattern in [
+                    ('clamscan', '"clamscan succeeded"'),
+                    ('freshclam', '"freshclam succeeded"'),
+                    # Using '?' to create an "a OR b" filter pattern
+                    ('clam_fail', '?"clamscan failed" ?"freshclam failed"'),
+                ]
             }
         },
         'aws_cloudwatch_metric_alarm': {
@@ -546,8 +565,26 @@ def paren(s: str) -> str:
                 for a in cis_alarms
             },
             **{
-                'trail_logs': {
-                    'alarm_name': config.qualified_resource_name('trail_logs', suffix='.alarm'),
+                'clam_fail': {
+                    'alarm_name': config.qualified_resource_name('clam_fail', suffix='.alarm'),
+                    'comparison_operator': 'GreaterThanOrEqualToThreshold',
+                    'evaluation_periods': 1,
+                    'metric_name': 'clam_fail',
+                    'namespace': 'LogMetrics',
+                    'statistic': 'Sum',
+                    'treat_missing_data': 'notBreaching',
+                    'threshold': 1,
+                    # With ClamScan running twice a day we've got a 12h period,
+                    # plus 8h upper bound on running time, minus 2h lower bound
+                    # on running time, giving us an 18h evaluation period.
+                    'period': 18 * 60 * 60,
+                    'alarm_actions': ['${aws_sns_topic.monitoring.arn}'],
+                    'ok_actions': ['${aws_sns_topic.monitoring.arn}']
+                }
+            },
+            **{
+                resource_name: {
+                    'alarm_name': config.qualified_resource_name(resource_name, suffix='.alarm'),
                     'comparison_operator': 'LessThanThreshold',
                     'threshold': 1,
                     'datapoints_to_alarm': 1,
@@ -570,14 +607,18 @@ def paren(s: str) -> str:
                         {
                             'id': 'log_count_raw',
                             'metric': {
-                                'metric_name': config.qualified_resource_name('trail_logs'),
+                                'metric_name': config.qualified_resource_name(resource_name),
                                 'namespace': 'LogMetrics',
-                                'period': 10 * 60,
+                                'period': period,
                                 'stat': 'Sum',
                             }
                         }
                     ]
-                }
+                } for resource_name, period in [
+                    ('trail_logs', 10 * 60),
+                    ('clamscan', 18 * 60 * 60),
+                    ('freshclam', 18 * 60 * 60)
+                ]
             }
         },
         'aws_iam_role': {