diff --git a/.github/workflows/canary-tests.yml b/.github/workflows/canary-tests.yml new file mode 100644 index 0000000..9884961 --- /dev/null +++ b/.github/workflows/canary-tests.yml @@ -0,0 +1,43 @@ +name: Canary Test + +on: + schedule: + - cron: "0 0 * * 0" # Run every Sunday at midnight + +jobs: + run-containers-and-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Run Containers + run: | + cd docker-hadoop + chmod +x ./setup-compose.sh + ./setup-compose.sh + + echo "Waiting for namenode to be ready..." + max_attempts=10 + attempt_num=1 + until docker exec namenode curl -sf http://localhost:9870 || [ "$attempt_num" -gt "$max_attempts" ]; do + echo "namenode is not ready yet. Retrying in 5 seconds..." + sleep 5 + attempt_num=$((attempt_num + 1)) + done + + if [ "$attempt_num" -le "$max_attempts" ]; then + echo "namenode is ready." + else + echo "namenode did not become ready in time." + exit 1 + fi + + - name: Run tests + run: | + # Run the commands now that namenode is ready + docker exec nodemanager1 /bin/sh -c "cd \$DISH_TOP && hdfs dfs -put README.md /README.md" + docker exec nodemanager1 /bin/sh -c "cd \$DISH_TOP && ./di.sh ./scripts/sample.sh" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5881581..20306cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Docker Build, Push Images +name: Docker Build, Push Images and Canary Test on: release: @@ -284,3 +284,47 @@ jobs: run: | docker push binpash/hadoop-submit:${{ env.RELEASE }} docker push binpash/hadoop-submit:latest + + run-containers-and-tests: + runs-on: ubuntu-latest + needs: + - build-hadoop-namenode + - build-hadoop-datanode + - build-hadoop-resourcemanager + - build-hadoop-nodemanager + - build-hadoop-historyserver + - build-hadoop-submit + + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + + - name: Run Containers + run: | + cd docker-hadoop + chmod +x ./setup-compose.sh + ./setup-compose.sh + + echo "Waiting for namenode to be ready..." + max_attempts=10 + attempt_num=1 + until docker exec namenode curl -sf http://localhost:9870 || [ "$attempt_num" -gt "$max_attempts" ]; do + echo "namenode is not ready yet. Retrying in 5 seconds..." + sleep 5 + attempt_num=$((attempt_num + 1)) + done + + if [ "$attempt_num" -le "$max_attempts" ]; then + echo "namenode is ready." + else + echo "namenode did not become ready in time." + exit 1 + fi + + - name: Run tests + run: | + # Run the commands now that namenode is ready + docker exec nodemanager1 /bin/sh -c "cd \$DISH_TOP && hdfs dfs -put README.md /README.md" + docker exec nodemanager1 /bin/sh -c "cd \$DISH_TOP && ./di.sh ./scripts/sample.sh" diff --git a/docker-hadoop b/docker-hadoop index 4e09b77..7d72f46 160000 --- a/docker-hadoop +++ b/docker-hadoop @@ -1 +1 @@ -Subproject commit 4e09b7761aebf4a058fbef5be96087da8065b70b +Subproject commit 7d72f46e041eff67c131b32dcbde7e89da06fbdc