fix loading spiders

juniorguru · Feb 12, 2024 · 3f1b19e · 3f1b19e
1 parent f7b35ca
commit 3f1b19e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/juniorguru_plucker/actors.py b/juniorguru_plucker/actors.py
@@ -64,7 +64,7 @@ def configure_async():
 
 def iter_actor_paths(path: Path | str) -> Generator[Path, None, None]:
     for actor_spec in Path(path).rglob(".actor/actor.json"):
-        yield actor_spec.parent.parent.relative_to(path)
+        yield actor_spec.parent.parent.relative_to(".")
 
 
 def get_spider_module_name(actor_path: Path | str) -> str:
@@ -75,8 +75,9 @@ class SpiderLoader(BaseSpiderLoader):
     def __init__(self, settings: BaseSettings):
         super().__init__(settings)
         if not self.spider_modules:
+            spider_path = settings.get("SPIDER_LOADER_SPIDERS_PATH", ".")
             self.spider_modules = list(
-                map(get_spider_module_name, iter_actor_paths("."))
+                map(get_spider_module_name, iter_actor_paths(spider_path))
             )
         self._load_all_spiders()
 

diff --git a/juniorguru_plucker/settings.py b/juniorguru_plucker/settings.py
@@ -17,6 +17,8 @@
 
 SPIDER_LOADER_CLASS = "juniorguru_plucker.actors.SpiderLoader"
 
+SPIDER_LOADER_SPIDERS_PATH = "./juniorguru_plucker"
+
 ITEM_PIPELINES = {
     "juniorguru_plucker.pipelines.required_fields_filter.Pipeline": 50,
 }