Python/Flask - Replaces gunicorn with uWSGI (GoogleCloudPlatform#463)

* Init

* Contacts logging

* userservice

* Frontend

* pylint

* restore userservice default log level

* pylint

* pylint

* unit test fix

* Try catch log level env var

* remove exception

* updates

* Fix duplicate logging bug

.pylintrc

@@ -5,4 +5,5 @@ disable=F0401,  # import loading
         E1101,  # function members
         R0801,  # duplicate code
         R0903,  # public methods
-        R0915   # too many statements
+        R0915,   # too many statements
+        R0914   # too many local variables

dev-kubernetes-manifests/frontend.yaml

@@ -43,7 +43,7 @@ spec:
           value: "true"
         - name: SCHEME
           value: "http"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
         - name: LOG_LEVEL
           value: "info"
         # Set to "true" to enable the CymbalBank logo + title

dev-kubernetes-manifests/userservice.yaml

@@ -48,7 +48,7 @@ spec:
           value: "3600"
         - name: PRIV_KEY_PATH
           value: "/root/.ssh/privatekey"
-        # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
         - name: LOG_LEVEL
           value: "info"
         envFrom:

extras/cloudsql/kubernetes-manifests/frontend.yaml

@@ -46,7 +46,7 @@ spec:
           value: "true"
         - name: SCHEME
           value: "http"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
         - name: LOG_LEVEL
           value: "info"
         - name: DEFAULT_USERNAME

extras/cloudsql/kubernetes-manifests/userservice.yaml

@@ -48,7 +48,7 @@ spec:
           value: "3600"
         - name: PRIV_KEY_PATH
           value: "/root/.ssh/privatekey"
-        # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
         - name: LOG_LEVEL
           value: "info"
         envFrom:

src/contacts/Dockerfile

@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@
 #   https://github.com/GoogleCloudPlatform/python-docker
 FROM gcr.io/google-appengine/python
 
+
 # show python logs as they occur
 ENV PYTHONUNBUFFERED=0
 
@@ -37,11 +38,11 @@ ENV LOG_LEVEL info
 ADD requirements.txt /app/requirements.txt
 RUN pip install -r /app/requirements.txt
 
+# Copy uwsgi conf
+COPY uwsgi-conf.ini /uwsgi-conf.ini
+
 # Add application code.
 ADD *.py /app/
 
-# Copy logging configuration for gunicorn
-COPY logging.conf /logging.conf
-
-# Start server using gunicorn
-CMD gunicorn -b :$PORT --threads 4 --log-config /logging.conf --log-level=$LOG_LEVEL "contacts:create_app()"
+# Start server with user-provided PORT
+CMD uwsgi --ini /uwsgi-conf.ini  --http :${PORT}

src/contacts/contacts.py

@@ -189,9 +189,24 @@ def _shutdown():
         """Executed when web app is terminated."""
         app.logger.info("Stopping contacts service.")
 
-    # set up logger
-    app.logger.handlers = logging.getLogger("gunicorn.error").handlers
-    app.logger.setLevel(logging.getLogger("gunicorn.error").level)
+    # set log formatting
+    date_format = "%Y-%m-%d %H:%M:%S"
+    message_format = '%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s'
+    logging.basicConfig(format= message_format, datefmt= date_format, stream=sys.stdout)
+
+    # set log level
+    log_levels = {
+        "DEBUG": logging.DEBUG,
+        "WARNING": logging.WARNING,
+        "INFO": logging.INFO,
+        "ERROR": logging.ERROR,
+        "CRITICAL": logging.CRITICAL
+    }
+    level = logging.INFO #default
+    user_log_level = os.environ.get("LOG_LEVEL")
+    if user_log_level is not None and user_log_level.upper() in log_levels:
+        level = log_levels.get(user_log_level.upper())
+    app.logger.setLevel(level)
     app.logger.info("Starting contacts service.")
 
     # Set up tracing and export spans to Cloud Trace.

src/contacts/requirements.in

@@ -15,7 +15,6 @@ google-cloud-core==1.4.1
 google-cloud-trace==0.24.0
 googleapis-common-protos==1.52.0
 grpcio==1.30.0
-gunicorn==20.0.4
 idna==2.10
 importlib-metadata==1.7.0
 itsdangerous==1.1.0
@@ -47,6 +46,7 @@ requests==2.24.0
 rsa==4.6
 six==1.14.0
 urllib3==1.25.9
+uwsgi==2.0.19.1
 wcwidth==0.2.5
 webencodings==0.5.1
 Werkzeug==1.0.1

src/contacts/requirements.txt

@@ -70,8 +70,6 @@ grpcio==1.30.0
     # via
     #   -r requirements.in
     #   google-api-core
-gunicorn==20.0.4
-    # via -r requirements.in
 idna==2.10
     # via
     #   -r requirements.in
@@ -202,6 +200,8 @@ urllib3==1.25.9
     # via
     #   -r requirements.in
     #   requests
+uwsgi==2.0.19.1
+    # via -r requirements.in
 wcwidth==0.2.5
     # via
     #   -r requirements.in

src/contacts/tests/test_contacts.py

@@ -36,6 +36,8 @@
     INVALID_ROUTING_NUMS,
 )
 
+LOG_LEVEL="info"
+
 def create_new_contact(**kwargs):
     """Helper method for creating new contacts from template"""
     example_contact = EXAMPLE_CONTACT.copy()

src/contacts/uwsgi-conf.ini

@@ -0,0 +1,10 @@
+[uwsgi]
+uid = root
+gid = root
+
+processes = 4
+master = true
+module = contacts:create_app()
+die-on-term = true
+
+disable-logging=True

src/frontend/Dockerfile

@@ -47,8 +47,11 @@ ADD static /app/static
 ADD templates /app/templates
 ADD *.py /app/
 
-# Copy logging configuration for gunicorn
-COPY logging.conf /logging.conf
+# Copy uwsgi conf
+COPY uwsgi-conf.ini /uwsgi-conf.ini
 
-# Start server using gunicorn
-CMD gunicorn -b :$PORT --threads 4 --log-config /logging.conf --log-level=$LOG_LEVEL "frontend:create_app()"
+# Add application code.
+ADD *.py /app/
+
+# Start server with user-provided PORT
+CMD uwsgi --ini /uwsgi-conf.ini  --http :${PORT}

src/frontend/frontend.py

@@ -20,6 +20,7 @@
 import logging
 import os
 import socket
+import sys
 from decimal import Decimal
 
 import requests
@@ -547,10 +548,25 @@ def format_currency(int_amount):
     app.jinja_env.globals.update(format_timestamp_month=format_timestamp_month)
     app.jinja_env.globals.update(format_timestamp_day=format_timestamp_day)
 
-    # Set up logging
-    app.logger.handlers = logging.getLogger('gunicorn.error').handlers
-    app.logger.setLevel(logging.getLogger('gunicorn.error').level)
-    app.logger.info('Starting frontend service.')
+    # set log formatting
+    date_format = "%Y-%m-%d %H:%M:%S"
+    message_format = '%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s'
+    logging.basicConfig(format= message_format, datefmt= date_format, stream=sys.stdout)
+
+    # set log level
+    log_levels = {
+        "DEBUG": logging.DEBUG,
+        "WARNING": logging.WARNING,
+        "INFO": logging.INFO,
+        "ERROR": logging.ERROR,
+        "CRITICAL": logging.CRITICAL
+    }
+    level = logging.INFO #default
+    user_log_level = os.environ.get("LOG_LEVEL")
+    if user_log_level is not None and user_log_level.upper() in log_levels:
+        level = log_levels.get(user_log_level.upper())
+    app.logger.setLevel(level)
+    app.logger.info("Starting frontend.")
 
     # Set up tracing and export spans to Cloud Trace.
     if os.environ['ENABLE_TRACING'] == "true":

src/frontend/requirements.in

@@ -2,11 +2,11 @@ flask==1.1.2
 requests==2.23.0
 pyjwt==1.7.1
 cryptography==3.3.2
-gunicorn==20.0.4
 opentelemetry-sdk==0.13b0
 opentelemetry-exporter-google-cloud==0.13b0
 opentelemetry-tools-google-cloud==0.13b0
 opentelemetry-instrumentation-flask==0.13b0
 opentelemetry-instrumentation-sqlalchemy==0.13b0
 opentelemetry-instrumentation-jinja2==0.13b0
-opentelemetry-instrumentation-requests==0.13b0
+opentelemetry-instrumentation-requests==0.13b0
+uwsgi==2.0.19.1