Skip to content
This repository has been archived by the owner on Mar 9, 2021. It is now read-only.

Commit

Permalink
Check if Tumblr LikedBy is accessible (not offline)
Browse files Browse the repository at this point in the history
Checks if a Tumblr liked-by page is accessible (or not available/offline) before trying to crawl it.
  • Loading branch information
johanneszab committed Feb 18, 2020
1 parent 4f29ea2 commit edb3bc9
Showing 1 changed file with 8 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@ private async Task GetUrlsAsync()
semaphoreSlim = new SemaphoreSlim(shellService.Settings.ConcurrentScans);
trackedTasks = new List<Task>();

if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrLikedByCrawler:GetUrlsAsync: {0}", "User not logged in");
shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name);
postQueue.CompleteAdding();
return;
}

long pagination = CreateStartPagination();

// TODO: find way to parallelize without losing content.
Expand Down Expand Up @@ -114,14 +122,6 @@ private async Task CrawlPageAsync(long pagination, int crawlerNumber)

public override async Task IsBlogOnlineAsync()
{
if (!await CheckIfLoggedInAsync())
{
Logger.Error("TumblrLikedByCrawler:GetUrlsAsync: {0}", "User not logged in");
shellService.ShowError(new Exception("User not logged in"), Resources.NotLoggedIn, blog.Name);
postQueue.CompleteAdding();
return;
}

try
{
await GetRequestAsync(blog.Url);
Expand Down

0 comments on commit edb3bc9

Please sign in to comment.