add: canalblog.com for domainsCrawlPass

internetarchive · Sep 20, 2024 · 52463f0 · 52463f0
1 parent 64cdfe7
commit 52463f0
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/internal/pkg/crawl/outlinks.go b/internal/pkg/crawl/outlinks.go
@@ -101,6 +101,7 @@ func (c *Crawl) domainsCrawlPass(origin, outlink *url.URL, originHop uint64) boo
 	// These are sites that host other sites. When --domains-crawl is used on one of these
 	// domains, we want to perform it on the subdomain, not the entire domain.
 	var excluded = []string{
+		"canalblog.com",
 		"blogspot.com",
 		"wordpress.com",
 		"tumblr.com",