diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5d5eba9..e923559 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,20 +33,13 @@ jobs: needs: generate-artifacts runs-on: ubuntu-latest container: ghcr.io/filtersheroes/expired_domains_image:latest + env: + GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v4 - run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" ./scripts/CI/getArtifactNamesForED.py KAD - npm install @actions/artifact - - uses: actions/github-script@v6 - with: - script: | - const artifact = require("@actions/artifact") - const artifactClient = artifact.create() - const artifactFiles = process.env.E_KAD_NAMES.split(" ") - for (artifactFile of artifactFiles) { - await artifactClient.downloadArtifact(artifactFile, "./expired-d", { createArtifactFolder: false }) - } - run: | + ./scripts/CI/downloadArtifacts.py "E_KAD_NAMES" ls -l ./expired-d diff --git a/Dockerfile_ED b/Dockerfile_ED index 2acd4c5..23913a7 100644 --- a/Dockerfile_ED +++ b/Dockerfile_ED @@ -3,7 +3,7 @@ FROM python:3-slim # make Apt non-interactive ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y locales git openssh-client ca-certificates wget tzdata whois curl nodejs npm --no-install-recommends && pip install GitPython dnspython aiohttp tldextract --no-cache-dir +RUN apt-get update && apt-get install -y locales git openssh-client ca-certificates wget tzdata whois curl --no-install-recommends && pip install GitPython dnspython aiohttp tldextract aiodns --no-cache-dir # uncomment chosen locale to enable it's generation RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && sed -i -e 's/# pl_PL.UTF-8 UTF-8/pl_PL.UTF-8 UTF-8/' /etc/locale.gen diff --git a/scripts/CI/downloadArtifacts.py b/scripts/CI/downloadArtifacts.py new file mode 100755 index 0000000..20d7b16 --- /dev/null +++ b/scripts/CI/downloadArtifacts.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# coding=utf-8 +# pylint: disable=C0103 +# pylint: disable=missing-module-docstring +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +import os +import sys +import shutil +import asyncio +import aiohttp + + +pj = os.path.join +pn = os.path.normpath + +script_path = os.path.dirname(os.path.realpath(__file__)) +main_path = os.getenv("GITHUB_WORKSPACE") + +expired_dir = pj(main_path, "expired-d") +if not os.path.isdir(expired_dir): + os.mkdir(expired_dir) + +async def download_artifact(session, limit, name): + async with limit: + try: + repo = os.getenv("GITHUB_REPOSITORY") + artifacts_url = f"https://api.github.com/repos/{repo}/actions/artifacts" + token = os.getenv("GIT_TOKEN") + headers = { "Authorization": f"Bearer {token}", "User-Agent": "Python"} + params = {'name': name} + resp = await session.get(artifacts_url, headers=headers, params=params) + if resp.status == 200: + data = await resp.json() + print(data) + if data["artifacts"]: + artifact = data["artifacts"][0] + try: + print(f"Downloading artifact {name} ...") + resp_adl = await session.get(artifact["archive_download_url"], headers=headers) + if resp_adl.status == 200: + file_path = pj(expired_dir, name + ".zip") + zip_file = await resp_adl.read() + with open(file_path, "wb") as f: + f.write(zip_file) + if os.path.isfile(file_path): + shutil.unpack_archive(file_path, expired_dir) + os.remove(file_path) + except Exception as e_adl: + print(e_adl) + except Exception as e: + print(e) + +async def download_artifacts(limit_value, names): + limit = asyncio.Semaphore(limit_value) + async with aiohttp.ClientSession() as session: + await asyncio.gather(*[download_artifact(session, limit, name) for name in names]) + +artifact_names = [] +if sys.argv[1]: + artifact_names = os.getenv(str(sys.argv[1])).split(" ") +asyncio.run(download_artifacts(5, artifact_names))