Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adding load config from module or py-file #823

Merged
merged 15 commits into from
Nov 6, 2023
Merged

adding load config from module or py-file #823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
92b4190
adding load config from module or py-file
ali96343 Nov 1, 2023
6d1824f
adding info messages
ali96343 Nov 1, 2023
ecf0487
adding sa_show_msg
ali96343 Nov 1, 2023
23e064a
adding2 sa_show_msg
ali96343 Nov 1, 2023
a503040
done-1, adding test example_config.py
ali96343 Nov 2, 2023
fc2cd4e
done-2, black sa
ali96343 Nov 2, 2023
c9b44d9
done-3, comment fix
ali96343 Nov 3, 2023
6f1f83d
adding var use_python_config eq use_native_config
ali96343 Nov 3, 2023
d474979
rm last comma from dict
ali96343 Nov 4, 2023
e1cf2cc
+ func check_kv
ali96343 Nov 4, 2023
e738875
rm print from get_gunicorn_vars
ali96343 Nov 4, 2023
fe6902b
mv config to sa_config
ali96343 Nov 4, 2023
69033c2
-Q test
ali96343 Nov 4, 2023
3279540
adding gunicorn.rst
ali96343 Nov 5, 2023
214c056
fix gunicorn.rst
ali96343 Nov 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 119 additions & 0 deletions py4web/gunicorn.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
====================
gunicorn with py4web
====================


The gunicorn server starts in the usual way for the py4web

::

$./py4web.py run apps -s gunicorn --watch=off
$
$./py4web.py run apps -s gunicornGevent --watch=off


gunicornGevent === gunicorn + monkey.patch_all()

It is possible to use several methods to configure gunicorn options with py4web

Let's show examples

* set gunicorn options via bash environment variables

::

$export GUNICORN_worker_class=sync
$ ./py4web.py run apps -s gunicorn -L 20 -w 4 --watch=off
$
$export GUNICORN_worker_class=gthread
$ ./py4web.py run apps -s gunicorn -L 20 -w 4 --watch=off
$
$export GUNICORN_worker_class=gevent
$ ./py4web.py run apps -s gunicornGevent -L 20 -w 4 --watch=off
$
$export GUNICORN_worker_class=eventlet
$ ./py4web.py run apps -s gunicornGevent -L 20 -w 4 --watch=off




* set gunicorn options via config file gunicorn.saenv

::

# example gunicorn.saenv
#
export GUNICORN_worker_tmp_dir=/dev/shm
export GUNICORN_max_requests=1200
worker_class=gthread
threads=2

# guncornGevent
#worker_class=gevent
#worker_class=eventlet

# for use python-config-file
# use_python_config=myguni.conf.py

# for use python-config-mod_name
# use_python_config=python:mod_name


* set gunicorn options via python file myguni.conf.py

::

set the env variable use_python_config=myguni.conf.py

.. code:: bash

$ # via env
$export GUNCORN_use_python_config=myguni.conf.py
$
$ # via gunicorn.saenv
$echo use_python_config=mmyguni.conf.py >> gunicorn.saenv

::

write file myguni.conf.py

.. code:: python

# Gunicorn configuration file
# https://docs.gunicorn.org/en/stable/settings.html
import multiprocessing

max_requests = 1000
max_requests_jitter = 50

log_file = "-"

workers = multiprocessing.cpu_count() * 2 + 1

::

$ ./py4web.py run apps -s gunicorn --watch=off


* set gunicorn options via python module

::

write python module

.. code:: bash


$ mkdir mod_name && cp myguni.conf.py mod_name/__init__.py
$
$ # via env
$export GUNCORN_use_python_config=python:mod_name
$
$ # via gunicorn.saenv
$echo use_python_config=python:mod_name >> gunicorn.saenv


::

$ ./py4web.py run apps -s gunicorn --watch=off

149 changes: 78 additions & 71 deletions py4web/server_adapters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,26 +128,29 @@ def gunicorn():
from gevent import local # pip install gevent gunicorn
import threading

# To use gevent monkey.patch_all()
# run ./py4web.py run apps -s gunicornGevent ......
if isinstance(threading.local(), local.local):
print("gunicorn: monkey.patch_all() applied")

class GunicornServer(ServerAdapter):
""" https://docs.gunicorn.org/en/stable/settings.html """

# https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
# ./py4web.py run apps -s gunicorn --watch=off --port=8000 --ssl_cert=cert.pem --ssl_key=key.pem -w 6 -L 20
# ./py4web.py run apps -s gunicornGevent --watch=off --port=8000 --ssl_cert=cert.pem --ssl_key=key.pem -w 6 -L 20
# ./py4web.py run apps -s gunicorn --watch=off --ssl_cert=cert.pem --ssl_key=key.pem -w 6 -L 20
# ./py4web.py run apps -s gunicornGevent --watch=off --ssl_cert=cert.pem --ssl_key=key.pem -w 6 -L 20
# time seq 1 5000 | xargs -I % -P 0 curl http://localhost:8000/todo &>/dev/null

def run(self, app_handler):
from gunicorn.app.base import BaseApplication
from gunicorn.app.base import Application
import re

config = {
logger = None

sa_config = {
"bind": f"{self.host}:{self.port}",
"workers": get_workers(self.options),
"certfile": self.options.get("certfile", None),
"keyfile": self.options.get("keyfile", None),
"accesslog": None,
"errorlog": None,
}

if not self.quiet:
Expand All @@ -158,7 +161,7 @@ def run(self, app_handler):
logger = logging_conf(level)
log_to = "-" if log_file is None else log_file

config.update(
sa_config.update(
{
"loglevel": logging.getLevelName(level),
"access_log_format": '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"',
Expand All @@ -167,91 +170,95 @@ def run(self, app_handler):
}
)

class GunicornApplication(BaseApplication):
class GunicornApplication(Application):
def logger_info(self, msg="msg"):
logger and logger.info(str(msg))

def get_gunicorn_vars(self, env_file="gunicorn.saenv"):
def check_kv(kx, vx):
if kx and vx and kx != "bind":
if vx.startswith("{") and vx.endswith("}"):
vt = re.sub(",\s*\}", "}", vx)
vx = json.loads(vt.replace("'", '"'))
if vx == "None":
vx = None
return kx, vx
return None, None

result = dict()
if os.path.isfile(env_file):
try:
with open(env_file, "r") as f:
lines = f.read().splitlines()
for line in lines:
line = line.strip()
if not line or line.startswith(("#",'[')):
if not line or line.startswith(("#", "[")):
continue
for k in ("export ", "GUNICORN_"):
line = line.replace(k, "",1)
for e in ("export ", "GUNICORN_"):
line = line.replace(e, "", 1)
k, v = None, None
try:
k, v = line.split("=", 1)
except ValueError:
continue
v = v.strip()
result[k.strip().lower()] = None if v == 'None' else v
k, v = k.strip().lower(), v.strip()
except (ValueError, AttributeError):
continue
k, v = check_kv(k, v)
if k is None:
continue
result[k] = v
if result:
print(f"gunicorn: read {env_file}")
result["config"] = "./" + env_file
return result
except OSError as ex:
print(f"{ex} gunicorn: cannot read {env_file}")
self.logger_info(f"gunicorn: cannot read {env_file}; {ex}")

for k, v in os.environ.items():
if k.startswith("GUNICORN_") and v:
key = k.split("_", 1)[1].lower()
result[key] = v
if k.startswith("GUNICORN_"):
k = k.split("_", 1)[1].lower()
k, v = check_kv(k, v)
if k is None:
continue
result[k] = v
result["config"] = "os.environ.items"
return result

def load_config(self):

"""
gunicorn.saenv

# example
# export GUNICORN_max_requests=1200
export GUNICORN_worker_tmp_dir=/dev/shm

# run as -s gunicornGevent
#worker_class=gevent
#worker_class=eventlet

# run as -s gunicorn
#worker_class=sync
[hello any text]
worker_class=gthread
workers=4
threads=8

"""


# export GUNICORN_BACKLOG=4096
# export GUNICORN_worker_connections=100

# export GUNICORN_worker_class=sync
# export GUNICORN_worker_class=gthread
# export GUNICORN_worker_tmp_dir=/dev/shm
# export GUNICORN_threads=8
# export GUNICORN_timeout=10
# export GUNICORN_max_requests=1200

#
# tested with ssep4w https://github.com/ali96343/lvsio
#
# To use gevent monkey.patch_all()
# run ./py4web.py run apps -s gunicornGevent ......
# export GUNICORN_worker_class=gevent
# export GUNICORN_worker_class=gunicorn.workers.ggevent.GeventWorker
#
# pip install gunicorn[eventlet]
# export GUNICORN_worker_class=eventlet
#
# time seq 1 5000 | xargs -I % -P 0 curl http://localhost:8000/todo &>/dev/null

gunicorn_vars = self.get_gunicorn_vars()

if gunicorn_vars:
config.update(gunicorn_vars)
print("gunicorn config:", config)

for key, value in config.items():
self.cfg.set(key, value)
# test https://github.com/benoitc/gunicorn/blob/master/examples/example_config.py
for e in ("use_python_config", "use_native_config"):
try:
location = gunicorn_vars[e]
Application.load_config_from_module_name_or_filename(
self, location
)
self.cfg.set("config", "./" + location)
self.logger_info(f"gunicorn: used {location}")
return
except KeyError:
pass

#if gunicorn_vars:
sa_config.update(gunicorn_vars)
location = gunicorn_vars["config"]
self.logger_info(f"gunicorn: used {location} {sa_config}")

for k, v in sa_config.items():
if k not in self.cfg.settings:
continue
try:
self.cfg.set(k, v)
except Exception:
self.logger_info(f"gunicorn: Invalid value for {k}:{v}")
raise

try:
gunicorn_vars["print_config"] == "True" and self.logger_info(
self.cfg
)
except KeyError:
pass

def load(self):
return app_handler
Expand Down
Loading