Test

.github/workflows/test.yml

@@ -33,20 +33,13 @@ jobs:
     needs: generate-artifacts
     runs-on: ubuntu-latest
     container: ghcr.io/filtersheroes/expired_domains_image:latest
+    env:
+      GIT_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
       - run: |
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
           ./scripts/CI/getArtifactNamesForED.py KAD
-          npm install @actions/artifact
-      - uses: actions/github-script@v6
-        with:
-          script: |
-            const artifact = require("@actions/artifact")
-            const artifactClient = artifact.create()
-            const artifactFiles = process.env.E_KAD_NAMES.split(" ")
-            for (artifactFile of artifactFiles) {
-              await artifactClient.downloadArtifact(artifactFile, "./expired-d", { createArtifactFolder: false })
-            }
       - run: |
+          ./scripts/CI/downloadArtifacts.py "E_KAD_NAMES"
           ls -l ./expired-d

Dockerfile_ED

@@ -3,7 +3,7 @@ FROM python:3-slim
 # make Apt non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y locales git openssh-client ca-certificates wget tzdata whois curl nodejs npm --no-install-recommends && pip install GitPython dnspython aiohttp tldextract --no-cache-dir
+RUN apt-get update && apt-get install -y locales git openssh-client ca-certificates wget tzdata whois curl --no-install-recommends && pip install GitPython dnspython aiohttp tldextract aiodns --no-cache-dir
 
 # uncomment chosen locale to enable it's generation
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && sed -i -e 's/# pl_PL.UTF-8 UTF-8/pl_PL.UTF-8 UTF-8/' /etc/locale.gen

scripts/CI/downloadArtifacts.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# coding=utf-8
+# pylint: disable=C0103
+# pylint: disable=missing-module-docstring
+# pylint: disable=missing-class-docstring
+# pylint: disable=missing-function-docstring
+import os
+import sys
+import shutil
+import asyncio
+import aiohttp
+
+
+pj = os.path.join
+pn = os.path.normpath
+
+script_path = os.path.dirname(os.path.realpath(__file__))
+main_path = os.getenv("GITHUB_WORKSPACE")
+
+expired_dir = pj(main_path, "expired-d")
+if not os.path.isdir(expired_dir):
+    os.mkdir(expired_dir)
+
+async def download_artifact(session, limit, name):
+    async with limit:
+        try:
+            repo = os.getenv("GITHUB_REPOSITORY")
+            artifacts_url = f"https://api.github.com/repos/{repo}/actions/artifacts"
+            token = os.getenv("GIT_TOKEN")
+            headers = { "Authorization": f"Bearer {token}", "User-Agent": "Python"}
+            params = {'name': name}
+            resp = await session.get(artifacts_url, headers=headers, params=params)
+            if resp.status == 200:
+                data = await resp.json()
+                print(data)
+                if data["artifacts"]:
+                    artifact = data["artifacts"][0]
+                    try:
+                        print(f"Downloading artifact {name} ...")
+                        resp_adl = await session.get(artifact["archive_download_url"], headers=headers)
+                        if resp_adl.status == 200:
+                            file_path = pj(expired_dir, name + ".zip")
+                            zip_file = await resp_adl.read()
+                            with open(file_path, "wb") as f:
+                                f.write(zip_file)
+                            if os.path.isfile(file_path):
+                                shutil.unpack_archive(file_path, expired_dir)
+                                os.remove(file_path)
+                    except Exception as e_adl:
+                        print(e_adl)
+        except Exception as e:
+            print(e)
+
+async def download_artifacts(limit_value, names):
+    limit = asyncio.Semaphore(limit_value)
+    async with aiohttp.ClientSession() as session:
+        await asyncio.gather(*[download_artifact(session, limit, name) for name in names])
+
+artifact_names = []
+if sys.argv[1]:
+    artifact_names = os.getenv(str(sys.argv[1])).split(" ")
+asyncio.run(download_artifacts(5, artifact_names))