Try to implement geosite and geoip db.

.github/workflows/main.yml

@@ -22,15 +22,79 @@ jobs:
           cache: 'pip'
       - name: Setup environment
         run: pip install -r requirements.txt
+      - name: Set up Go
+        uses: actions/setup-go@v4
       - name: Pull v2fly community domain list
         uses: actions/checkout@v3
         with:
           repository: 'v2fly/domain-list-community'
           path: 'domain-list-community'
+      - name: Pull Loyalsoldier domain-list-custom
+        uses: actions/checkout@v3
+        with:
+          repository: 'Loyalsoldier/domain-list-custom'
+          path: 'tool_geosite'
+      - name: Pull Loyalsoldier's geosite tool
+        uses: actions/checkout@v3
+        with:
+          repository: 'Loyalsoldier/domain-list-custom'
+          path: 'tool_geosite'
+      - name: Install Loyalsoldier's geoip tool
+        run: go install -v github.com/Loyalsoldier/geoip@latest
       - name: Generate rules
         run: |
               python generate.py
-              python generate_homepage.py
+      - name: Generate Geosite DB
+        run: |
+              cd tool_geosite
+              go run ./ --datapath=../dists/geosite --outputpath=../dists
+      - name: Generate GeoIP DB
+        run: |
+              cat <<EOF
+              {
+              "input": [
+                {
+                  "type": "text",
+                  "action": "add",
+                  "args": {
+                    "name": "cn",
+                    "uri": "./dists/text/domestic_ip.txt",
+                    "onlyIPType": "ipv4"
+                  }
+                },
+                {
+                  "type": "text",
+                  "action": "add",
+                  "args": {
+                    "name": "cn",
+                    "uri": "./dists/text/domestic_ip6.txt",
+                    "onlyIPType": "ipv6"
+                  }
+                },
+                {
+                  "type": "clashRuleSetClassical",
+                  "action": "add",
+                  "args": {
+                    "name": "apns",
+                    "uri": "./dists/clash-compatible/apns.txt"
+                  }
+                }
+              ],
+              "output": [
+                {
+                  "type": "v2rayGeoIPDat",
+                  "action": "output",
+                  "args": {
+                    "outputName": "geoip.dat",
+                    "outputDir": "./dists"
+                  }
+                }
+              ]
+              }
+              EOF > config.json
+              geoip -c config.json
+      - name: Remove tmp files
+        run: rm -rf dists/geosite
       - name: Publish to Cloudflare Pages
         uses: cloudflare/pages-action@v1
         with:

Utils/const.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-TARGETS = ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"]
+TARGETS = ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible", "geosite"]
 PATH_SOURCE_V2FLY = Path("./domain-list-community/data/")
 PATH_SOURCE_CUSTOM = Path("./Source/")
 PATH_SOURCE_PATCH = Path("./Patch/")

Utils/geosite.py

@@ -51,4 +51,7 @@ def batch_convert(categories: list, tools: list, exclusions=None) -> None:
             src_geosite = set(open(const.PATH_SOURCE_V2FLY/category, mode="r", encoding="utf-8").read().splitlines())
             set_geosite = parse(src_geosite, exclusions)
             list_geosite_sorted = rule.set_to_sorted_list(set_geosite)
-            rule.dump(list_geosite_sorted, tool, const.PATH_DIST/tool/(category + ".txt"))
+            if tool == "geosite":
+                rule.dump(list_geosite_sorted, tool, const.PATH_DIST/tool/category)
+            else:
+                rule.dump(list_geosite_sorted, tool, const.PATH_DIST/tool/(category + ".txt"))

Utils/rule.py

@@ -144,6 +144,15 @@ def dump(src: list, target: str, dst: Path) -> None:
                         dist.writelines(f"{rule.Payload},Policy\n")
                     case _:
                         raise TypeError(f'Unsupported rule type "{rule.Type}". File: {dst}.')
+        case "geosite":
+            for rule in src:
+                match rule.Type:
+                    case "DomainSuffix":
+                        dist.writelines(f"{rule.Payload}\n")
+                    case "DomainFull":
+                        dist.writelines(f"full:{rule.Payload}\n")
+                    case _:
+                        raise TypeError(f'Unsupported rule type "{rule.Type}". File: {dst}.')
         case _:
             raise TypeError("Target type unsupported, "
                             "only accept 'text', 'text-plus', 'yaml', 'surge-compatible' or 'clash-compatible'."
@@ -152,6 +161,8 @@ def dump(src: list, target: str, dst: Path) -> None:
 
 def batch_dump(src: list, targets: list, dst_path: Path, filename: str) -> None:
     for target in targets:
+        if target == "geosite" and ".txt" in filename:
+            filename = filename.split(".")[0]
         dump(src, target, dst_path/target/filename)
 
 

generate.py

@@ -125,7 +125,9 @@
     if not line.startswith("#"):
         list_cidr.append(rule.Rule("IPCIDR", line))
 logger.info(f"Generated {len(list_cidr)} domestic IPv4 rules.")
-rule.batch_dump(list_cidr, const.TARGETS, const.PATH_DIST, "domestic_ip.txt")
+rule.batch_dump(list_cidr, ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"],
+                const.PATH_DIST, "domestic_ip.txt")
+
 src_cidr6 = connection.get(const.URL_CHNROUTES_V6).text.splitlines()
 list_cidr6_raw = []
 for line in src_cidr6:
@@ -137,7 +139,8 @@
 for cidr in list_cidr6_raw:
     list_cidr6.append(rule.Rule("IPCIDR6", cidr))
 logger.info(f"Generated {len(list_cidr6)} domestic IPv6 rules.")
-rule.batch_dump(list_cidr6, const.TARGETS, const.PATH_DIST, "domestic_ip6.txt")
+rule.batch_dump(list_cidr6, ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"],
+                const.PATH_DIST, "domestic_ip6.txt")
 END_TIME = time_ns()
 logger.info(f"Finished. Total time: {format((END_TIME - START_TIME) / 1e9, '.3f')}s\n")
 
@@ -174,19 +177,25 @@
         logger.debug(f'Start converting "{filename.name}".')
         set_custom = rule.custom_convert(filename)
         is_classical = False
+        is_ipcidr = False
         for item in set_custom:
             if item.Type == "Classic":
                 is_classical = True  # If one rule's type is "Classic", this is a classical ruleset.
                 break
-            else:
+            if item.Type == "IPCIDR":
+                is_ipcidr = True
                 break
+            break
         list_custom_sorted = rule.set_to_sorted_list(set_custom)
         if is_classical:
-            targets = const.TARGETS
             rule.batch_dump(list_custom_sorted,
                             ["yaml", "surge-compatible", "clash-compatible"],
                             # Classical ruleset doesn't have plain text type.
                             const.PATH_DIST, filename.name)
+        elif is_ipcidr:
+            rule.batch_dump(list_custom_sorted,
+                            ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"],
+                            const.PATH_DIST, filename.name)
         else:
             rule.batch_dump(list_custom_sorted, const.TARGETS, const.PATH_DIST, filename.name)
         logger.debug(f"Converted {len(list_custom_sorted)} rules.")
@@ -197,7 +206,9 @@
     logger.debug(f'Start converting "{filename.name}".')
     set_personal = rule.custom_convert(filename)
     list_personal_sorted = rule.set_to_sorted_list(set_personal)
-    rule.batch_dump(list_personal_sorted, const.TARGETS, const.PATH_DIST, "personal/" + filename.name)
+    rule.batch_dump(list_personal_sorted, ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"],
+                    const.PATH_DIST, "personal/" + filename.name)
+    rule.dump(list_personal_sorted, "geosite", const.PATH_DIST/"geosite"/("personal-" + filename.stem))
     logger.debug(f"Converted {len(list_personal_sorted)} rules.")
 
 END_TIME = time_ns()