Configurable database names

crawler4j/src/main/java/edu/uci/ics/crawler4j/crawler/CrawlController.java

@@ -88,16 +88,28 @@ public class CrawlController {
 
     public CrawlController(CrawlConfig config, PageFetcher pageFetcher,
                            RobotstxtServer robotstxtServer) throws Exception {
-        this(config, pageFetcher, null, robotstxtServer, null);
+        this(config, pageFetcher, null, robotstxtServer, null, null, null, null);
     }
 
     public CrawlController(CrawlConfig config, PageFetcher pageFetcher,
             RobotstxtServer robotstxtServer, TLDList tldList) throws Exception {
-        this(config, pageFetcher, null, robotstxtServer, tldList);
+        this(config, pageFetcher, null, robotstxtServer, tldList, null, null, null);
     }
 
     public CrawlController(CrawlConfig config, PageFetcher pageFetcher, Parser parser,
-                           RobotstxtServer robotstxtServer, TLDList tldList) throws Exception {
+            RobotstxtServer robotstxtServer, TLDList tldList) throws Exception {
+        this(config, pageFetcher, parser, robotstxtServer, tldList, null, null, null);
+    }
+
+    public CrawlController(CrawlConfig config, PageFetcher pageFetcher, Parser parser,
+            RobotstxtServer robotstxtServer, TLDList tldList,
+            String docIdDbName, String pendingDbName) throws Exception {
+        this(config, pageFetcher, parser, robotstxtServer, tldList, null, null, null);
+    }
+
+    public CrawlController(CrawlConfig config, PageFetcher pageFetcher, Parser parser,
+                           RobotstxtServer robotstxtServer, TLDList tldList,
+                           String docIdDbName, String pendingDbName, String inProcessDbName) throws Exception {
         config.validate();
         this.config = config;
 
@@ -140,8 +152,8 @@ public CrawlController(CrawlConfig config, PageFetcher pageFetcher, Parser parse
         }
 
         env = new Environment(envHome, envConfig);
-        docIdServer = new DocIDServer(env, config);
-        frontier = new Frontier(env, config);
+        docIdServer = new DocIDServer(env, config, docIdDbName);
+        frontier = new Frontier(env, config, pendingDbName, inProcessDbName);
 
         this.pageFetcher = pageFetcher;
         this.parser = parser == null ? new Parser(config, tldList) : parser;

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/DocIDServer.java

@@ -45,14 +45,18 @@ public class DocIDServer {
     private CrawlConfig config;
     private int lastDocID;
 
-    public DocIDServer(Environment env, CrawlConfig config) {
+    public DocIDServer(Environment env, CrawlConfig config, String dbName) {
         this.config = config;
         DatabaseConfig dbConfig = new DatabaseConfig();
         dbConfig.setAllowCreate(true);
         dbConfig.setTransactional(config.isResumableCrawling());
         dbConfig.setDeferredWrite(!config.isResumableCrawling());
         lastDocID = 0;
-        docIDsDB = env.openDatabase(null, DATABASE_NAME, dbConfig);
+        if (dbName == null) {
+            docIDsDB = env.openDatabase(null, DATABASE_NAME, dbConfig);
+        } else {
+            docIDsDB = env.openDatabase(null, dbName, dbConfig);
+        }
         if (config.isResumableCrawling()) {
             int docCount = getDocCount();
             if (docCount > 0) {
@@ -62,6 +66,10 @@ public DocIDServer(Environment env, CrawlConfig config) {
         }
     }
 
+    public DocIDServer(Environment env, CrawlConfig config) {
+        this(env, config, null);
+    }
+
     /**
      * Returns the docid of an already seen url.
      *

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/Frontier.java

@@ -52,13 +52,29 @@ public class Frontier {
     protected Counters counters;
 
     public Frontier(Environment env, CrawlConfig config) {
+        this(env, config, null);
+    }
+
+    public Frontier(Environment env, CrawlConfig config, String dbName) {
+        this(env, config, dbName, null);
+    }
+
+    public Frontier(Environment env, CrawlConfig config, String dbName, String inProcessDbName) {
         this.config = config;
         this.counters = new Counters(env, config);
         try {
-            workQueues = new WorkQueues(env, DATABASE_NAME, config.isResumableCrawling());
+            if (dbName == null) {
+                workQueues = new WorkQueues(env, DATABASE_NAME, config.isResumableCrawling());
+            } else {
+                workQueues = new WorkQueues(env, dbName, config.isResumableCrawling());
+            }
             if (config.isResumableCrawling()) {
                 scheduledPages = counters.getValue(Counters.ReservedCounterNames.SCHEDULED_PAGES);
-                inProcessPages = new InProcessPagesDB(env);
+                if (inProcessDbName == null) {
+                    inProcessPages = new InProcessPagesDB(env);
+                } else {
+                    inProcessPages = new InProcessPagesDB(env, inProcessDbName);
+                }
                 long numPreviouslyInProcessPages = inProcessPages.getLength();
                 if (numPreviouslyInProcessPages > 0) {
                     logger.info("Rescheduling {} URLs from previous crawl.",

crawler4j/src/main/java/edu/uci/ics/crawler4j/frontier/InProcessPagesDB.java

@@ -41,7 +41,11 @@ public class InProcessPagesDB extends WorkQueues {
     private static final String DATABASE_NAME = "InProcessPagesDB";
 
     public InProcessPagesDB(Environment env) {
-        super(env, DATABASE_NAME, true);
+        this(env, DATABASE_NAME);
+    }
+
+    public InProcessPagesDB(Environment env, String dbName) {
+        super(env, dbName, true);
         long docCount = getLength();
         if (docCount > 0) {
             logger.info("Loaded {} URLs that have been in process in the previous crawl.",