Revert Python services from uWSGI to Gunicorn + call Locust directly (#…

…501) * Revert "Python/Flask - Replaces gunicorn with uWSGI (#463)" This reverts commit b5b0024. * bump gunicorn and adjust dockerfiles * call locust directly in loadgenerator * remove escapes from entrypoint * update copyright year in touched files Co-authored-by: Megan O'Keefe <[email protected]>
GoogleCloudPlatform · Jun 22, 2021 · 368549b · 368549b
1 parent 55206c6
commit 368549b
Show file tree

Hide file tree

Showing 27 changed files with 154 additions and 170 deletions.
diff --git a/.pylintrc b/.pylintrc
@@ -5,5 +5,4 @@ disable=F0401,  # import loading
         E1101,  # function members
         R0801,  # duplicate code
         R0903,  # public methods
-        R0915,   # too many statements
-        R0914   # too many local variables
+        R0915   # too many statements
diff --git a/dev-kubernetes-manifests/frontend.yaml b/dev-kubernetes-manifests/frontend.yaml
@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -43,7 +43,7 @@ spec:
           value: "true"
         - name: SCHEME
           value: "http"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
         - name: LOG_LEVEL
           value: "info"
         # Set to "true" to enable the CymbalBank logo + title

diff --git a/dev-kubernetes-manifests/userservice.yaml b/dev-kubernetes-manifests/userservice.yaml
@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,7 +48,7 @@ spec:
           value: "3600"
         - name: PRIV_KEY_PATH
           value: "/root/.ssh/privatekey"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
+        # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
         - name: LOG_LEVEL
           value: "info"
         envFrom:

diff --git a/extras/cloudsql/kubernetes-manifests/frontend.yaml b/extras/cloudsql/kubernetes-manifests/frontend.yaml
@@ -1,4 +1,4 @@
-# Copyright 2019 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -46,7 +46,7 @@ spec:
           value: "true"
         - name: SCHEME
           value: "http"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
+         # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
         - name: LOG_LEVEL
           value: "info"
         - name: DEFAULT_USERNAME

diff --git a/extras/cloudsql/kubernetes-manifests/userservice.yaml b/extras/cloudsql/kubernetes-manifests/userservice.yaml
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,7 +48,7 @@ spec:
           value: "3600"
         - name: PRIV_KEY_PATH
           value: "/root/.ssh/privatekey"
-         # Valid levels are debug, info, warning, error, critical. If no valid level is set, defaults to info.
+        # Valid levels are debug, info, warning, error, critical. If no valid level is set, gunicorn will default to info.
         - name: LOG_LEVEL
           value: "info"
         envFrom:

diff --git a/src/contacts/Dockerfile b/src/contacts/Dockerfile
@@ -18,7 +18,6 @@
 #   https://github.com/GoogleCloudPlatform/python-docker
 FROM gcr.io/google-appengine/python
 
-
 # show python logs as they occur
 ENV PYTHONUNBUFFERED=0
 
@@ -38,11 +37,11 @@ ENV LOG_LEVEL info
 ADD requirements.txt /app/requirements.txt
 RUN pip install -r /app/requirements.txt
 
-# Copy uwsgi conf
-COPY uwsgi-conf.ini /uwsgi-conf.ini
-
 # Add application code.
 ADD *.py /app/
 
-# Start server with user-provided PORT
-CMD uwsgi --ini /uwsgi-conf.ini  --http :${PORT}
+# Copy logging configuration for gunicorn
+COPY logging.conf /logging.conf
+
+# Start server using gunicorn
+CMD gunicorn -b :$PORT --threads 4 --log-config /logging.conf --log-level=$LOG_LEVEL "contacts:create_app()"
diff --git a/src/contacts/contacts.py b/src/contacts/contacts.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -189,24 +189,9 @@ def _shutdown():
         """Executed when web app is terminated."""
         app.logger.info("Stopping contacts service.")
 
-    # set log formatting
-    date_format = "%Y-%m-%d %H:%M:%S"
-    message_format = '%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s'
-    logging.basicConfig(format= message_format, datefmt= date_format, stream=sys.stdout)
-
-    # set log level
-    log_levels = {
-        "DEBUG": logging.DEBUG,
-        "WARNING": logging.WARNING,
-        "INFO": logging.INFO,
-        "ERROR": logging.ERROR,
-        "CRITICAL": logging.CRITICAL
-    }
-    level = logging.INFO #default
-    user_log_level = os.environ.get("LOG_LEVEL")
-    if user_log_level is not None and user_log_level.upper() in log_levels:
-        level = log_levels.get(user_log_level.upper())
-    app.logger.setLevel(level)
+    # set up logger
+    app.logger.handlers = logging.getLogger("gunicorn.error").handlers
+    app.logger.setLevel(logging.getLogger("gunicorn.error").level)
     app.logger.info("Starting contacts service.")
 
     # Set up tracing and export spans to Cloud Trace.

diff --git a/src/contacts/logging.conf b/src/contacts/logging.conf
@@ -0,0 +1,33 @@
+[loggers]
+keys=root, gunicorn.error, contacts
+
+[handlers]
+keys=console
+
+[formatters]
+keys=generic
+
+[logger_root]
+handlers=console
+
+[logger_gunicorn.error]
+formatter=generic
+handlers=console
+propagate=0
+qualname=gunicorn.error
+
+[logger_contacts]
+formatter=generic
+handlers=console
+propagate=0
+qualname=contacts
+
+[handler_console]
+class=StreamHandler
+formatter=generic
+args=(sys.stdout, )
+
+[formatter_generic]
+format=%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
diff --git a/src/contacts/requirements.in b/src/contacts/requirements.in
@@ -15,6 +15,7 @@ google-cloud-core==1.4.1
 google-cloud-trace==0.24.0
 googleapis-common-protos==1.52.0
 grpcio==1.30.0
+gunicorn==20.1.0
 idna==2.10
 importlib-metadata==1.7.0
 itsdangerous==1.1.0
@@ -46,7 +47,6 @@ requests==2.24.0
 rsa==4.7
 six==1.14.0
 urllib3==1.25.9
-uwsgi==2.0.19.1
 wcwidth==0.2.5
 webencodings==0.5.1
 Werkzeug==1.0.1

diff --git a/src/contacts/requirements.txt b/src/contacts/requirements.txt
@@ -70,6 +70,8 @@ grpcio==1.30.0
     # via
     #   -r requirements.in
     #   google-api-core
+gunicorn==20.1.0
+    # via -r requirements.in
 idna==2.10
     # via
     #   -r requirements.in
@@ -200,8 +202,6 @@ urllib3==1.25.9
     # via
     #   -r requirements.in
     #   requests
-uwsgi==2.0.19.1
-    # via -r requirements.in
 wcwidth==0.2.5
     # via
     #   -r requirements.in

diff --git a/src/contacts/tests/test_contacts.py b/src/contacts/tests/test_contacts.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,8 +36,6 @@
     INVALID_ROUTING_NUMS,
 )
 
-LOG_LEVEL="info"
-
 def create_new_contact(**kwargs):
     """Helper method for creating new contacts from template"""
     example_contact = EXAMPLE_CONTACT.copy()

diff --git a/src/contacts/uwsgi-conf.ini b/src/contacts/uwsgi-conf.ini
diff --git a/src/frontend/Dockerfile b/src/frontend/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -47,11 +47,8 @@ ADD static /app/static
 ADD templates /app/templates
 ADD *.py /app/
 
-# Copy uwsgi conf
-COPY uwsgi-conf.ini /uwsgi-conf.ini
+# Copy logging configuration for gunicorn
+COPY logging.conf /logging.conf
 
-# Add application code.
-ADD *.py /app/
-
-# Start server with user-provided PORT
-CMD uwsgi --ini /uwsgi-conf.ini  --http :${PORT}
+# Start server using gunicorn
+CMD gunicorn -b :$PORT --threads 4 --log-config /logging.conf --log-level=$LOG_LEVEL "frontend:create_app()"
diff --git a/src/frontend/frontend.py b/src/frontend/frontend.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@
 import logging
 import os
 import socket
-import sys
 from decimal import Decimal
 
 import requests
@@ -548,25 +547,10 @@ def format_currency(int_amount):
     app.jinja_env.globals.update(format_timestamp_month=format_timestamp_month)
     app.jinja_env.globals.update(format_timestamp_day=format_timestamp_day)
 
-    # set log formatting
-    date_format = "%Y-%m-%d %H:%M:%S"
-    message_format = '%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s'
-    logging.basicConfig(format= message_format, datefmt= date_format, stream=sys.stdout)
-
-    # set log level
-    log_levels = {
-        "DEBUG": logging.DEBUG,
-        "WARNING": logging.WARNING,
-        "INFO": logging.INFO,
-        "ERROR": logging.ERROR,
-        "CRITICAL": logging.CRITICAL
-    }
-    level = logging.INFO #default
-    user_log_level = os.environ.get("LOG_LEVEL")
-    if user_log_level is not None and user_log_level.upper() in log_levels:
-        level = log_levels.get(user_log_level.upper())
-    app.logger.setLevel(level)
-    app.logger.info("Starting frontend.")
+    # Set up logging
+    app.logger.handlers = logging.getLogger('gunicorn.error').handlers
+    app.logger.setLevel(logging.getLogger('gunicorn.error').level)
+    app.logger.info('Starting frontend service.')
 
     # Set up tracing and export spans to Cloud Trace.
     if os.environ['ENABLE_TRACING'] == "true":

diff --git a/src/frontend/logging.conf b/src/frontend/logging.conf
@@ -0,0 +1,33 @@
+[loggers]
+keys=root, gunicorn.error, frontend
+
+[handlers]
+keys=console
+
+[formatters]
+keys=generic
+
+[logger_root]
+handlers=console
+
+[logger_gunicorn.error]
+formatter=generic
+handlers=console
+propagate=0
+qualname=gunicorn.error
+
+[logger_frontend]
+formatter=generic
+handlers=console
+propagate=0
+qualname=frontend
+
+[handler_console]
+class=StreamHandler
+formatter=generic
+args=(sys.stdout, )
+
+[formatter_generic]
+format=%(asctime)s | [%(levelname)s] | %(funcName)s | %(message)s
+datefmt=%Y-%m-%d %H:%M:%S
+class=logging.Formatter
diff --git a/src/frontend/requirements.in b/src/frontend/requirements.in
@@ -2,11 +2,11 @@ flask==1.1.2
 requests==2.23.0
 pyjwt==1.7.1
 cryptography==3.3.2
+gunicorn==20.1.0
 opentelemetry-sdk==0.13b0
 opentelemetry-exporter-google-cloud==0.13b0
 opentelemetry-tools-google-cloud==0.13b0
 opentelemetry-instrumentation-flask==0.13b0
 opentelemetry-instrumentation-sqlalchemy==0.13b0
 opentelemetry-instrumentation-jinja2==0.13b0
-opentelemetry-instrumentation-requests==0.13b0
-uwsgi==2.0.19.1
+opentelemetry-instrumentation-requests==0.13b0
diff --git a/src/frontend/requirements.txt b/src/frontend/requirements.txt
@@ -39,6 +39,8 @@ googleapis-common-protos==1.52.0
     # via google-api-core
 grpcio==1.30.0
     # via google-api-core
+gunicorn==20.1.0
+    # via -r requirements.in
 idna==2.9
     # via requests
 itsdangerous==1.1.0
@@ -121,8 +123,6 @@ sqlalchemy==1.3.19
     # via opentelemetry-instrumentation-sqlalchemy
 urllib3==1.25.8
     # via requests
-uwsgi==2.0.19.1
-    # via -r requirements.in
 werkzeug==1.0.1
     # via flask
 wrapt==1.12.1

diff --git a/src/frontend/uwsgi-conf.ini b/src/frontend/uwsgi-conf.ini
diff --git a/src/loadgenerator/Dockerfile b/src/loadgenerator/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2020 Google LLC
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,6 +30,9 @@ RUN virtualenv -p python3.7 /env
 ENV VIRTUAL_ENV /env
 ENV PATH /env/bin:$PATH
 
+# enable gevent support in debugger
+ENV GEVENT_SUPPORT=True
+
 # explicitly set a fallback log level in case no log level is defined by Kubernetes
 ENV LOG_LEVEL info
 
@@ -38,9 +41,7 @@ ADD requirements.txt /app/requirements.txt
 RUN pip install -r /app/requirements.txt
 
 # Add application code.
-ADD loadgen.sh /app
 ADD locustfile.py /app
-RUN chmod +x /app/loadgen.sh
 
 # start loadgenerator
-ENTRYPOINT /app/loadgen.sh
+ENTRYPOINT locust --host="http://${FRONTEND_ADDR}" --loglevel $LOG_LEVEL --no-web -c "${USERS:-10}" 2>&1