# This is a combination of 7 commits.

# This is the 1st commit message: build(docker): Split builder image out for better caching This is an experimental new approach which creates a separate builder image, that is highly cacheable (and allows caching for yarn install step) to generate the Python wheel. It should save us multiple minutes by leveraging our builds' incremental nature much better. # This is the commit message #2: remove redundant op, trigger subsequent build # This is the commit message #3: add SOURCE_COMMIT env to builder run # This is the commit message #4: oops # This is the commit message #5: add revision and license labels to builder # This is the commit message #6: use kaniko 0.22 as it should fix the weird failures # This is the commit message #7: Simpler and stronger .dockerignore
getsentry · May 26, 2020 · 6828052 · 6828052
1 parent be31b59
commit 6828052
Show file tree

Hide file tree

Showing 7 changed files with 203 additions and 247 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,66 +1,8 @@
-**/__pycache__
-.git
-.gitignore
-.gitattributes
-.dockerignore
-**/cloudbuild.yaml
-Gemfile.lock
-.idea/
-*.iml
-.pytest_cache/
-.vscode
-.tx
-.travis
-.github
-.mailmap
-.npmrc
-.travis.yml
-.pre-commit-config.yaml
-.eslint*
-.env
-.envrc
-coverage.xml
-conftest.py
-netlify.toml
-static
-junit.xml
-*.codestyle.xml
-package-lock.json
-.cache/
-.coverage
-.storybook-out/
-.DS_Store
-.venv
-*.egg-info
-*.pyc
-*.log
-*.egg
-*.db
-*.pid
-Brewfile
-Makefile
-MANIFEST
-test.conf
-pip-log.txt
-celerybeat-schedule
-sentry-package.json
-/.artifacts
-/coverage/
-/cover
-/build
-/env
-/tests
-/tmp
-/node_modules/
-/docs-ui/node_modules/
-/scripts
-/src/sentry/assets.json
-/src/sentry/static/version
-/src/sentry/static/sentry/dist/
-/src/sentry/static/sentry/vendor/
-/src/sentry/static/sentry/admin/
-/src/sentry/static/sentry/rest_framework/
-/src/sentry/integration-docs
-/src/sentry/loader/_registry.json
-/wheelhouse
-/test_cli/
+# Ignore everything
+*
+
+!/docker
+!/package.json
+!/yarn.lock
+!/dist/requirements.txt
+!/dist/*.whl
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,4 +2,4 @@ include setup.py README.md MANIFEST.in LICENSE AUTHORS
 recursive-include ./ requirements*.txt
 recursive-include ./config/relay *
 graft src/sentry
-global-exclude *~
+global-exclude *.pyc
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,95 +1,3 @@
-FROM python:2.7.16-slim-buster as sdist
-
-LABEL maintainer="[email protected]"
-LABEL org.opencontainers.image.title="Sentry PyPI Wheel"
-LABEL org.opencontainers.image.description="PyPI Wheel Builder for Sentry"
-LABEL org.opencontainers.image.url="https://sentry.io/"
-LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
-LABEL org.opencontainers.image.vendor="Functional Software, Inc."
-LABEL org.opencontainers.image.authors="[email protected]"
-
-# Sane defaults for pip
-ENV PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=1
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    # Needed for GPG
-    dirmngr \
-    gnupg \
-    # Needed for fetching stuff
-    wget \
-    && rm -rf /var/lib/apt/lists/* \
-    # Needed to extract final dependencies from the whl
-    && pip install pkginfo==1.5.0.1
-
-# Fetch trusted keys
-RUN for key in \
-      # gosu
-      B42F6819007F00F88E364FD4036A9C25BF357DD4 \
-      # tini
-      595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
-      # Node - gpg keys listed at https://github.com/nodejs/node
-      94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-      FD3A5288F042B6850C66B31F09FE44734EB7990E \
-      71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-      DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-      C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-      B9AE9905FFD7803F25714661B63B535A4C206CA9 \
-      77984A986EBC2AA786BC0F66B01FBB92821C587A \
-      8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
-      4ED778F539E3634C779C87C6D7062848A1AB005C \
-      A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
-      B9E2F5981AA6E0CD28160D9FF13993A75599653C \
-    ; do \
-      # TODO(byk): Replace the keyserver below w/ something owned by Sentry
-      gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
-    done
-
-# grab gosu for easy step-down from root
-ENV GOSU_VERSION 1.11
-RUN set -x \
-    && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
-    && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
-    && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
-    && rm -r /usr/local/bin/gosu.asc \
-    && chmod +x /usr/local/bin/gosu
-
-# grab tini for signal processing and zombie killing
-ENV TINI_VERSION 0.18.0
-RUN set -x \
-    && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
-    && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
-    && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
-    && rm /usr/local/bin/tini.asc \
-    && chmod +x /usr/local/bin/tini
-
-# Get and set up Node for front-end asset building
-COPY .nvmrc /usr/src/sentry/
-RUN cd /usr/src/sentry \
-    && export NODE_VERSION="$(cat .nvmrc)" \
-    && wget "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
-    && wget "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
-    && gpg --batch --verify SHASUMS256.txt.asc \
-    && grep " node-v$NODE_VERSION-linux-x64.tar.gz\$" SHASUMS256.txt.asc | sha256sum -c - \
-    && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
-    && rm -r "node-v$NODE_VERSION-linux-x64.tar.gz" SHASUMS256.txt.asc
-
-ARG SOURCE_COMMIT
-ENV SENTRY_BUILD=${SOURCE_COMMIT:-unknown}
-LABEL org.opencontainers.image.revision=$SOURCE_COMMIT
-LABEL org.opencontainers.image.licenses="https://github.com/getsentry/sentry/blob/${SOURCE_COMMIT:-master}/LICENSE"
-
-COPY . /usr/src/sentry/
-RUN export YARN_CACHE_FOLDER="$(mktemp -d)" \
-    && cd /usr/src/sentry \
-    && python setup.py bdist_wheel \
-    && rm -r "$YARN_CACHE_FOLDER" \
-    && mv /usr/src/sentry/dist /dist \
-    # Dump the dependencies of our wheel as a separate requirements.txt file
-    # so we can install them first, leveraging Docker's caching when they
-    # don't change across versions.
-    && pkginfo -f requires_dist --single --sequence-delim=! /dist/*.whl | tr ! \\n > /dist/requirements.txt
-
 # This is the image to be run
 FROM python:2.7.16-slim-buster
 
@@ -102,91 +10,123 @@ LABEL org.opencontainers.image.source="https://github.com/getsentry/sentry"
 LABEL org.opencontainers.image.vendor="Functional Software, Inc."
 LABEL org.opencontainers.image.authors="[email protected]"
 
-
 # add our user and group first to make sure their IDs get assigned consistently
 RUN groupadd -r sentry && useradd -r -m -g sentry sentry
 
-COPY --from=sdist /usr/local/bin/gosu /usr/local/bin/tini /usr/local/bin/
+ENV GOSU_VERSION=1.11 \
+  TINI_VERSION=0.18.0
+
+RUN set -x \
+  && buildDeps=" \
+  dirmngr \
+  gnupg \
+  wget \
+  " \
+  && apt-get update && apt-get install -y --no-install-recommends $buildDeps \
+  && rm -rf /var/lib/apt/lists/* \
+  # Fetch trusted keys
+  && for key in \
+  # gosu
+  B42F6819007F00F88E364FD4036A9C25BF357DD4 \
+  # tini
+  595E85A6B1B4779EA4DAAEC70B588DFF0527A9B7 \
+  ; do \
+  # TODO(byk): Replace the keyserver below w/ something owned by Sentry
+  gpg --batch --keyserver hkps://mattrobenolt-keyserver.global.ssl.fastly.net:443 --recv-keys "$key"; \
+  done \
+  # grab gosu for easy step-down from root
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
+  && wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
+  && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
+  && rm -r /usr/local/bin/gosu.asc \
+  && chmod +x /usr/local/bin/gosu \
+  # grab tini for signal processing and zombie killing
+  && wget -O /usr/local/bin/tini "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini" \
+  && wget -O /usr/local/bin/tini.asc "https://github.com/krallin/tini/releases/download/v$TINI_VERSION/tini.asc" \
+  && gpg --batch --verify /usr/local/bin/tini.asc /usr/local/bin/tini \
+  && rm /usr/local/bin/tini.asc \
+  && chmod +x /usr/local/bin/tini \
+  && apt-get purge -y --auto-remove $buildDeps
 
 # Sane defaults for pip
 ENV PIP_NO_CACHE_DIR=off \
-    PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    # Sentry config params
-    SENTRY_CONF=/etc/sentry \
-    # Disable some unused uWSGI features, saving dependencies
-    # Thank to https://stackoverflow.com/a/25260588/90297
-    UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
-    # UWSGI dogstatsd plugin
-    UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
+  PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  # Sentry config params
+  SENTRY_CONF=/etc/sentry \
+  # Disable some unused uWSGI features, saving dependencies
+  # Thank to https://stackoverflow.com/a/25260588/90297
+  UWSGI_PROFILE_OVERRIDE=ssl=false;xml=false;routing=false \
+  # UWSGI dogstatsd plugin
+  UWSGI_NEED_PLUGIN=/var/lib/uwsgi/dogstatsd
 
 # Copy and install dependencies first to leverage Docker layer caching.
-COPY --from=sdist /dist/requirements.txt /tmp/dist/requirements.txt
+COPY /dist/requirements.txt /tmp/dist/requirements.txt
 RUN set -x \
-    && buildDeps="" \
-    # uwsgi
-    && buildDeps="$buildDeps \
-      gcc \
-      g++ \
-      wget \
-    " \
-    # maxminddb
-    && buildDeps="$buildDeps \
-      libmaxminddb-dev \
-    "\
-    # librabbitmq
-    && buildDeps="$buildDeps \
-      make \
-    " \
-       # xmlsec
-    && buildDeps="$buildDeps \
-      libxmlsec1-dev \
-      pkg-config \
-    " \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends $buildDeps \
-    && pip install -r /tmp/dist/requirements.txt \
-    # Separate these due to https://git.io/fjyz6
-    # Otherwise librabbitmq will install the latest amqp version,
-    # violating kombu's amqp<2.0 constraint.
-    && pip install librabbitmq==1.6.1 \
-    && mkdir /tmp/uwsgi-dogstatsd \
-    && wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
-       tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
-    && UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
-    && mkdir -p /var/lib/uwsgi \
-    && mv dogstatsd_plugin.so /var/lib/uwsgi/ \
-    && rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
-    && apt-get purge -y --auto-remove $buildDeps \
-    # We install run-time dependencies strictly after
-    # build dependencies to prevent accidental collusion.
-    # These are also installed last as they are needed
-    # during container run and can have the same deps w/
-    # build deps such as maxminddb.
-    && apt-get install -y --no-install-recommends \
-      # pillow
-      libjpeg-dev \
-      # rust bindings
-      libffi-dev \
-      # maxminddb bindings
-      libmaxminddb-dev \
-      # SAML needs these run-time
-      libxmlsec1-dev \
-      libxslt-dev \
-      # pyyaml needs this run-time
-      libyaml-dev \
-      # other
-      pkg-config \
-    \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && python -c 'import librabbitmq' \
-    # Fully verify that the C extension is correctly installed, it unfortunately
-    # requires a full check into maxminddb.extension.Reader
-    && python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
-    && mkdir -p $SENTRY_CONF
-
-COPY --from=sdist /dist/*.whl /tmp/dist/
-RUN pip install /tmp/dist/*.whl && pip check
+  && buildDeps="" \
+  # uwsgi
+  && buildDeps="$buildDeps \
+  gcc \
+  g++ \
+  wget \
+  " \
+  # maxminddb
+  && buildDeps="$buildDeps \
+  libmaxminddb-dev \
+  "\
+  # librabbitmq
+  && buildDeps="$buildDeps \
+  make \
+  " \
+  # xmlsec
+  && buildDeps="$buildDeps \
+  libxmlsec1-dev \
+  pkg-config \
+  " \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends $buildDeps \
+  && pip install -r /tmp/dist/requirements.txt \
+  # Separate these due to https://git.io/fjyz6
+  # Otherwise librabbitmq will install the latest amqp version,
+  # violating kombu's amqp<2.0 constraint.
+  && pip install librabbitmq==1.6.1 \
+  && mkdir /tmp/uwsgi-dogstatsd \
+  && wget -O - https://github.com/eventbrite/uwsgi-dogstatsd/archive/filters-and-tags.tar.gz | \
+  tar -xzf - -C /tmp/uwsgi-dogstatsd --strip-components=1 \
+  && UWSGI_NEED_PLUGIN="" uwsgi --build-plugin /tmp/uwsgi-dogstatsd \
+  && mkdir -p /var/lib/uwsgi \
+  && mv dogstatsd_plugin.so /var/lib/uwsgi/ \
+  && rm -rf /tmp/dist /tmp/uwsgi-dogstatsd .uwsgi_plugins_builder \
+  && apt-get purge -y --auto-remove $buildDeps \
+  # We install run-time dependencies strictly after
+  # build dependencies to prevent accidental collusion.
+  # These are also installed last as they are needed
+  # during container run and can have the same deps w/
+  # build deps such as maxminddb.
+  && apt-get install -y --no-install-recommends \
+  # pillow
+  libjpeg-dev \
+  # rust bindings
+  libffi-dev \
+  # maxminddb bindings
+  libmaxminddb-dev \
+  # SAML needs these run-time
+  libxmlsec1-dev \
+  libxslt-dev \
+  # pyyaml needs this run-time
+  libyaml-dev \
+  # other
+  pkg-config \
+  \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* \
+  && python -c 'import librabbitmq' \
+  # Fully verify that the C extension is correctly installed, it unfortunately
+  # requires a full check into maxminddb.extension.Reader
+  && python -c 'import maxminddb.extension; maxminddb.extension.Reader' \
+  && mkdir -p $SENTRY_CONF
+
+COPY /dist/*.whl /tmp/dist/
+RUN pip install /tmp/dist/*.whl && pip check && rm -rf /tmp/dist
 RUN sentry help | sed '1,/Commands:/d' | awk '{print $1}' >  /sentry-commands.txt
 
 COPY ./docker/sentry.conf.py ./docker/config.yml $SENTRY_CONF/