From 99cbd80d28206e52e234fd952f0368b06bdf9432 Mon Sep 17 00:00:00 2001 From: phanikumv Date: Mon, 19 Sep 2022 20:43:14 +0530 Subject: [PATCH] Remove staging env deployment --- .circleci/integration-tests/Dockerfile.gen_1 | 81 -------------------- .circleci/integration-tests/script.sh | 7 +- 2 files changed, 1 insertion(+), 87 deletions(-) delete mode 100644 .circleci/integration-tests/Dockerfile.gen_1 diff --git a/.circleci/integration-tests/Dockerfile.gen_1 b/.circleci/integration-tests/Dockerfile.gen_1 deleted file mode 100644 index 1d9eeb36a..000000000 --- a/.circleci/integration-tests/Dockerfile.gen_1 +++ /dev/null @@ -1,81 +0,0 @@ -FROM quay.io/astronomer/ap-airflow:2.3.3-2 - -ENV AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False -ENV AWS_NUKE_VERSION=v2.17.0 - -USER root - -RUN apt-get update -y \ - && apt-get install -y software-properties-common \ - && apt-get install -y wget procps gnupg2 - -# Install openjdk-8 -RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' \ - && apt-get update && apt-get install -y openjdk-8-jdk -ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ - -RUN apt-get update -y \ - && apt-get install -y git \ - && apt-get install -y unzip \ - && apt-get install -y --no-install-recommends \ - build-essential \ - libsasl2-2 \ - libsasl2-dev \ - libsasl2-modules \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && apt-get install -y curl gnupg && \ - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && \ - apt-get update -y && \ - apt-get install google-cloud-sdk -y - -# Set Hive and Hadoop versions. -ENV HIVE_LIBRARY_VERSION=hive-2.3.9 -ENV HADOOP_LIBRARY_VERSION=hadoop-2.10.1 - -# install AWS CLI -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ - && unzip awscliv2.zip \ - && ./aws/install; - -# install AWS nuke -RUN wget --quiet \ - "https://github.com/rebuy-de/aws-nuke/releases/download/${AWS_NUKE_VERSION}/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz" \ - && tar -xzvf aws-nuke-${AWS_NUKE_VERSION}-linux-amd64.tar.gz -C /usr/local/bin \ - && mv /usr/local/bin/aws-nuke-${AWS_NUKE_VERSION}-linux-amd64 /usr/local/bin/aws-nuke - -# install eksctl -RUN curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp \ - && mv /tmp/eksctl /usr/local/bin - -# install kubectl -RUN curl -o kubectl https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl \ - && chmod +x ./kubectl \ - && mv ./kubectl /usr/local/bin - -# Install Apache Hive (Download same version of library to match EMR Hive version created by DAG). -RUN curl "https://downloads.apache.org/hive/$HIVE_LIBRARY_VERSION/apache-$HIVE_LIBRARY_VERSION-bin.tar.gz" -o "/tmp/apache-$HIVE_LIBRARY_VERSION-bin.tar.gz" \ - && tar -xf /tmp/apache-$HIVE_LIBRARY_VERSION-bin.tar.gz -C /usr/local -ENV HIVE_HOME=/usr/local/apache-$HIVE_LIBRARY_VERSION-bin - -# Install Apache Hadoop (Download same version of library to match EMR Hadoop version created by DAG). -RUN curl "https://archive.apache.org/dist/hadoop/common/$HADOOP_LIBRARY_VERSION/$HADOOP_LIBRARY_VERSION.tar.gz" -o "/tmp/$HADOOP_LIBRARY_VERSION.tar.gz" \ - && tar -xf /tmp/$HADOOP_LIBRARY_VERSION.tar.gz -C /usr/local -ENV HADOOP_HOME=/usr/local/$HADOOP_LIBRARY_VERSION -ENV HADOOP_CONF_DIR=$HADOOP_HOME/conf - -ENV PATH $PATH:$JAVA_HOME/bin:$HIVE_HOME/bin:$HADOOP_HOME/bin - -COPY astronomer-providers /tmp/astronomer-providers -RUN pip install /tmp/astronomer-providers[all] -RUN pip install apache-airflow[slack] - -RUN mkdir -p ${AIRFLOW_HOME}/dags - -COPY . . -RUN cp -r example_* ${AIRFLOW_HOME}/dags -RUN cp master_dag.py ${AIRFLOW_HOME}/dags/ -RUN cp nuke-config.yml ${AIRFLOW_HOME}/dags/ - -USER astro diff --git a/.circleci/integration-tests/script.sh b/.circleci/integration-tests/script.sh index 8da381320..da19e5404 100644 --- a/.circleci/integration-tests/script.sh +++ b/.circleci/integration-tests/script.sh @@ -95,12 +95,7 @@ for dag in $(find "${PROVIDER_PATH}" -type f -name 'example_*'); do cp "${dag}" # Build image and deploy BUILD_NUMBER=$(awk 'BEGIN {srand(); print srand()}') -if [[ ${DEPLOYMENT_INSTANCE} == "staging" ]]; then - IMAGE_NAME=registry.${DOCKER_REGISTRY}/${RELEASE_NAME}/airflow:ci-${BUILD_NUMBER} - docker build -t "${IMAGE_NAME}" -f "${SCRIPT_PATH}"/Dockerfile.gen_1 "${SCRIPT_PATH}" - docker login registry."${DOCKER_REGISTRY}" -u _ -p "${ASTRONOMER_API_KEY}" - docker push "${IMAGE_NAME}" -elif [[ ${DEPLOYMENT_INSTANCE} == "astro-cloud" ]]; then +if [[ ${DEPLOYMENT_INSTANCE} == "astro-cloud" ]]; then IMAGE_NAME=${DOCKER_REGISTRY}/${ORGANIZATION_ID}/${DEPLOYMENT_ID}:ci-${BUILD_NUMBER} docker build -t "${IMAGE_NAME}" -f "${SCRIPT_PATH}"/Dockerfile.astro_cloud "${SCRIPT_PATH}" docker login "${DOCKER_REGISTRY}" -u "${ASTRONOMER_KEY_ID}" -p "${ASTRONOMER_KEY_SECRET}"