From a6e0f1b75a5b60dbb81288b22d78d9686c3b565c Mon Sep 17 00:00:00 2001 From: Bogdan Date: Fri, 3 Feb 2017 09:37:08 -0800 Subject: [PATCH] Add an option to configure celery workers size. (#2085) --- superset/cli.py | 7 +++++-- superset/config.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/superset/cli.py b/superset/cli.py index 3b64c450634c7..ba143594e7d4c 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -144,8 +144,10 @@ def refresh_druid(datasource, merge): session.commit() -@manager.command -def worker(): +@manager.option( + '-w', '--workers', default=config.get("SUPERSET_CELERY_WORKERS", 32), + help="Number of celery server workers to fire up") +def worker(workers): """Starts a Superset worker for async SQL query execution.""" # celery -A tasks worker --loglevel=info print("Starting SQL Celery worker.") @@ -159,5 +161,6 @@ def worker(): 'broker': config.get('CELERY_CONFIG').BROKER_URL, 'loglevel': 'INFO', 'traceback': True, + 'concurrency': int(workers), } c_worker.run(**options) diff --git a/superset/config.py b/superset/config.py index 4a32b2c5dc0f6..d35d9e3a2b25f 100644 --- a/superset/config.py +++ b/superset/config.py @@ -32,6 +32,7 @@ ROW_LIMIT = 50000 VIZ_ROW_LIMIT = 10000 SUPERSET_WORKERS = 2 +SUPERSET_CELERY_WORKERS = 32 SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0' SUPERSET_WEBSERVER_PORT = 8088