From 26bc0063be0db6e54d90e95274b508248e0e77a9 Mon Sep 17 00:00:00 2001 From: Tobias Genannt Date: Tue, 10 Nov 2020 15:23:07 +0100 Subject: [PATCH] Gunicorn is replaced with nginx-unit --- Dockerfile | 53 +++++++++++++++++-------------------- build.sh | 14 +++++----- docker-compose.test.yml | 14 ---------- docker-compose.yml | 22 +++------------ docker/docker-entrypoint.sh | 17 +++++++++--- docker/gunicorn_config.py | 8 ------ docker/nginx-unit.json | 40 ++++++++++++++++++++++++++++ requirements-container.txt | 4 +++ 8 files changed, 93 insertions(+), 79 deletions(-) delete mode 100644 docker/gunicorn_config.py create mode 100644 docker/nginx-unit.json create mode 100644 requirements-container.txt diff --git a/Dockerfile b/Dockerfile index 8db4cac6f..19e00259f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,27 +12,15 @@ RUN apk add --no-cache \ libffi-dev \ libxslt-dev \ openldap-dev \ - postgresql-dev - -WORKDIR /install - -RUN pip install --prefix="/install" --no-warn-script-location \ -# gunicorn is used for launching netbox - gunicorn \ - greenlet \ - eventlet \ -# napalm is used for gathering information from network devices - napalm \ -# ruamel is used in startup_scripts - 'ruamel.yaml>=0.15,<0.16' \ -# django_auth_ldap is required for ldap - django_auth_ldap \ -# django-storages was introduced in 2.7 and is optional - django-storages + postgresql-dev \ + python3-dev \ + py3-pip \ + && python3 -m venv /opt/netbox/venv \ + && /opt/netbox/venv/bin/python3 -m pip install --upgrade pip setuptools ARG NETBOX_PATH -COPY ${NETBOX_PATH}/requirements.txt / -RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt +COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt / +RUN /opt/netbox/venv/bin/pip install -r /requirements.txt -r /requirements-container.txt ### # Main stage @@ -43,6 +31,7 @@ FROM ${FROM} as main RUN apk add --no-cache \ bash \ + curl \ ca-certificates \ graphviz \ libevent \ @@ -51,35 +40,43 @@ RUN apk add --no-cache \ libressl \ libxslt \ postgresql-libs \ - ttf-ubuntu-font-family + ttf-ubuntu-font-family \ + python3 \ + py3-pip \ + unit \ + unit-python3 WORKDIR /opt -COPY --from=builder /install /usr/local +COPY --from=builder /opt/netbox/venv /opt/netbox/venv ARG NETBOX_PATH COPY ${NETBOX_PATH} /opt/netbox COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py -COPY docker/gunicorn_config.py /etc/netbox/ -COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh COPY startup_scripts/ /opt/netbox/startup_scripts/ COPY initializers/ /opt/netbox/initializers/ COPY configuration/ /etc/netbox/config/ +COPY docker/nginx-unit.json /etc/unit/ WORKDIR /opt/netbox/netbox -# Must set permissions for '/opt/netbox/netbox/static' directory -# to g+w so that `./manage.py collectstatic` can be executed during -# container startup. # Must set permissions for '/opt/netbox/netbox/media' directory # to g+w so that pictures can be uploaded to netbox. -RUN mkdir static && chmod -R g+w static media +RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \ + && chmod -R g+w media /opt/unit/ \ + && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ] -CMD ["gunicorn", "-c /etc/netbox/gunicorn_config.py", "netbox.wsgi"] +CMD [ "unitd", \ + "--no-daemon", \ + "--control", "unix:/opt/unit/unit.sock", \ + "--pid", "/opt/unit/unit.pid", \ + "--log", "/dev/stdout", \ + "--state", "/opt/unit/state/", \ + "--tmp", "/opt/unit/tmp/" ] LABEL ORIGINAL_TAG="" \ NETBOX_GIT_BRANCH="" \ diff --git a/build.sh b/build.sh index 19c56c81e..628934bdc 100755 --- a/build.sh +++ b/build.sh @@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " DOCKERFILE The name of Dockerfile to use." echo " Default: Dockerfile" echo " DOCKER_FROM The base image to use." - echo " Default: 'python:3.9-alpine'" + echo " Default: 'alpine:edge'" echo " DOCKER_TARGET A specific target to build." echo " It's currently not possible to pass multiple targets." echo " Default: main ldap" @@ -106,7 +106,7 @@ else fi ### -# Variables for fetching the source +# Variables for fetching the Netbox source ### SRC_ORG="${SRC_ORG-netbox-community}" SRC_REPO="${SRC_REPO-netbox}" @@ -115,10 +115,10 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}" NETBOX_PATH="${NETBOX_PATH-.netbox}" ### -# Fetching the source +# Fetching the Netbox source ### if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then - echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'" + echo "🌐 Checking out '${NETBOX_BRANCH}' of Netbox from the url '${URL}' into '${NETBOX_PATH}'" if [ ! -d "${NETBOX_PATH}" ]; then $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}" fi @@ -135,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then $DRY git checkout -qf FETCH_HEAD $DRY git prune ) - echo "✅ Checked out netbox" + echo "✅ Checked out Netbox" fi ### @@ -157,7 +157,7 @@ fi # Determining the value for DOCKER_FROM ### if [ -z "$DOCKER_FROM" ]; then - DOCKER_FROM="python:3.9-alpine" + DOCKER_FROM="alpine:edge" fi ### @@ -271,7 +271,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then SHOULD_BUILD="true" - BUILD_REASON="${BUILD_REASON} python" + BUILD_REASON="${BUILD_REASON} alpine" fi if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then SHOULD_BUILD="true" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 23f74ca99..a7fa7a35e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -14,19 +14,9 @@ services: - ./configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro - ./scripts:/etc/netbox/scripts:z,ro - - netbox-nginx-config:/etc/netbox-nginx:z - - netbox-static-files:/opt/netbox/netbox/static:z - netbox-media-files:/opt/netbox/netbox/media:z - nginx: - command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.19-alpine - depends_on: - - netbox ports: - 8080 - volumes: - - netbox-static-files:/opt/netbox/netbox/static:ro - - netbox-nginx-config:/etc/netbox-nginx/:ro postgres: image: postgres:12-alpine env_file: env/postgres.env @@ -45,9 +35,5 @@ services: - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose env_file: env/redis-cache.env volumes: - netbox-static-files: - driver: local - netbox-nginx-config: - driver: local netbox-media-files: driver: local diff --git a/docker-compose.yml b/docker-compose.yml index 8a2c575cd..fa7ac9499 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,31 +15,19 @@ services: - ./configuration:/etc/netbox/config:z,ro - ./reports:/etc/netbox/reports:z,ro - ./scripts:/etc/netbox/scripts:z,ro - - netbox-nginx-config:/etc/netbox-nginx:z - - netbox-static-files:/opt/netbox/netbox/static:z - netbox-media-files:/opt/netbox/netbox/media:z + ports: + - 8080 netbox-worker: <<: *netbox depends_on: - redis entrypoint: - - python3 + - /opt/netbox/venv/bin/python - /opt/netbox/netbox/manage.py command: - rqworker - # nginx - nginx: - command: nginx -c /etc/netbox-nginx/nginx.conf - image: nginx:1.19-alpine - depends_on: - - netbox - ports: - - 8080 - volumes: - - netbox-static-files:/opt/netbox/netbox/static:ro - - netbox-nginx-config:/etc/netbox-nginx/:ro - # postgres postgres: image: postgres:12-alpine @@ -66,10 +54,6 @@ services: env_file: env/redis-cache.env volumes: - netbox-static-files: - driver: local - netbox-nginx-config: - driver: local netbox-media-files: driver: local netbox-postgres-data: diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4887d9664..f0a783398 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -7,6 +7,9 @@ set -e # Allows Netbox to be run as non-root users umask 002 +# Load correct Python3 env +source /opt/netbox/venv/bin/activate + # Try to connect to the DB DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3} MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30} @@ -60,11 +63,19 @@ else echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python fi -# Copy static files -./manage.py collectstatic --no-input - echo "✅ Initialisation is done." +( + UNIT_SOCKET="/opt/unit/unit.sock" + UNIT_CONFIG="/etc/unit/nginx-unit.json" + while [ ! -S $UNIT_SOCKET ]; do echo "$0: Waiting for control socket to be created..."; sleep 1; done + # even when the control socket exists, it does not mean unit has finished initialisation + # this curl call will get a reply once unit is fully launched + curl --silent --request GET --unix-socket $UNIT_SOCKET http://localhost/ + echo "$0: Applying configuration from $UNIT_CONFIG"; + curl --silent --request PUT --data-binary @$UNIT_CONFIG --unix-socket $UNIT_SOCKET http://localhost/config +)& + # Launch whatever is passed by docker # (i.e. the RUN instruction in the Dockerfile) # diff --git a/docker/gunicorn_config.py b/docker/gunicorn_config.py deleted file mode 100644 index 4ce776317..000000000 --- a/docker/gunicorn_config.py +++ /dev/null @@ -1,8 +0,0 @@ -command = '/usr/bin/gunicorn' -pythonpath = '/opt/netbox/netbox' -bind = '0.0.0.0:8001' -workers = 3 -errorlog = '-' -accesslog = '-' -capture_output = False -loglevel = 'info' diff --git a/docker/nginx-unit.json b/docker/nginx-unit.json new file mode 100644 index 000000000..3c4228ecb --- /dev/null +++ b/docker/nginx-unit.json @@ -0,0 +1,40 @@ +{ + "listeners": { + "*:8080": { + "pass": "routes" + } + }, + + "routes": [ + { + "match": { + "uri": "/static/*" + }, + "action": { + "share": "/opt/netbox/netbox" + } + }, + + { + "action": { + "pass": "applications/netbox" + } + } + ], + + "applications": { + "netbox": { + "type": "python 3", + "path": "/opt/netbox/netbox/", + "module": "netbox.wsgi", + "home": "/opt/netbox/venv", + "processes": { + "max": 4, + "spare": 2, + "idle_timeout": 120 + } + } + }, + + "access_log": "/dev/stdout" +} diff --git a/requirements-container.txt b/requirements-container.txt new file mode 100644 index 000000000..e011042e6 --- /dev/null +++ b/requirements-container.txt @@ -0,0 +1,4 @@ +napalm==3.2.0 +ruamel.yaml>=0.15,<0.16 +django-auth-ldap==2.2.0 +django-storages==1.10.1 \ No newline at end of file