diff --git a/VARIABLES.md b/VARIABLES.md index 3536069..c00da5e 100644 --- a/VARIABLES.md +++ b/VARIABLES.md @@ -10,4 +10,5 @@ You can configure some of BirdsiteLIVE's settings via environment variables (tho ## Instance customization * `Instance:Name` (default: BirdsiteLIVE) the name of the instance -* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it. \ No newline at end of file +* `Instance:ResolveMentionsInProfiles` (default: true) to enable or disable mentions parsing in profile's description. Resolving it will consume more User's API calls since newly discovered account can also contain references to others accounts as well. On a big instance it is recommended to disable it. +* `Instance:PublishReplies` (default: false) to enable or disable replies publishing. \ No newline at end of file diff --git a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs index fe303a7..1214002 100644 --- a/src/BirdsiteLive.Common/Settings/InstanceSettings.cs +++ b/src/BirdsiteLive.Common/Settings/InstanceSettings.cs @@ -6,5 +6,7 @@ public class InstanceSettings public string Domain { get; set; } public string AdminEmail { get; set; } public bool ResolveMentionsInProfiles { get; set; } + public bool PublishReplies { get; set; } + public int MaxUsersCapacity { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj index 5d93cb1..884af18 100644 --- a/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj +++ b/src/BirdsiteLive.Pipeline/BirdsiteLive.Pipeline.csproj @@ -17,4 +17,8 @@ + + + + diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index ef20cad..c381dcf 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -45,7 +46,13 @@ public async Task ProcessAsync(SyncTwitterUser[] syncTwi else if (tweets.Length > 0 && user.LastTweetPostedId == -1) { var tweetId = tweets.Last().Id; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, now); + } + else + { + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, now); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index f556831..ebb87fc 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -4,9 +4,11 @@ using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using BirdsiteLive.Common.Extensions; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Contracts; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; namespace BirdsiteLive.Pipeline.Processors @@ -14,13 +16,16 @@ namespace BirdsiteLive.Pipeline.Processors public class RetrieveTwitterUsersProcessor : IRetrieveTwitterUsersProcessor { private readonly ITwitterUserDal _twitterUserDal; + private readonly IMaxUsersNumberProvider _maxUsersNumberProvider; private readonly ILogger _logger; + public int WaitFactor = 1000 * 60; //1 min #region Ctor - public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, ILogger logger) + public RetrieveTwitterUsersProcessor(ITwitterUserDal twitterUserDal, IMaxUsersNumberProvider maxUsersNumberProvider, ILogger logger) { _twitterUserDal = twitterUserDal; + _maxUsersNumberProvider = maxUsersNumberProvider; _logger = logger; } #endregion @@ -33,7 +38,8 @@ public async Task GetTwitterUsersAsync(BufferBlock twitterUse try { - var users = await _twitterUserDal.GetAllTwitterUsersAsync(); + var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 5b305e7..c7cbc36 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -1,4 +1,5 @@ -using System.Linq; +using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -23,7 +24,8 @@ public async Task ProcessAsync(UserWithTweetsToSync userWithTweetsToSync, Cancel var userId = userWithTweetsToSync.User.Id; var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).Min(); - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync); + var now = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, now); } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs index 2fc93fe..a6f6982 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToInboxTask.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; @@ -21,15 +22,17 @@ public class SendTweetsToInboxTask : ISendTweetsToInboxTask private readonly IActivityPubService _activityPubService; private readonly IStatusService _statusService; private readonly IFollowersDal _followersDal; + private readonly InstanceSettings _settings; private readonly ILogger _logger; #region Ctor - public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger logger) + public SendTweetsToInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) { _activityPubService = activityPubService; _statusService = statusService; _followersDal = followersDal; + _settings = settings; _logger = logger; } #endregion @@ -52,8 +55,13 @@ public async Task ExecuteAsync(IEnumerable tweets, Follower foll { try { - var note = _statusService.GetStatus(user.Acct, tweet); - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + if (!tweet.IsReply || + tweet.IsReply && tweet.IsThread || + _settings.PublishReplies) + { + var note = _statusService.GetStatus(user.Acct, tweet); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), follower.Host, inbox); + } } catch (ArgumentException e) { diff --git a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs index dcc6aca..1abe183 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SubTasks/SendTweetsToSharedInboxTask.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Net; using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; @@ -20,14 +21,16 @@ public class SendTweetsToSharedInboxTask : ISendTweetsToSharedInboxTask private readonly IStatusService _statusService; private readonly IActivityPubService _activityPubService; private readonly IFollowersDal _followersDal; + private readonly InstanceSettings _settings; private readonly ILogger _logger; #region Ctor - public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, ILogger logger) + public SendTweetsToSharedInboxTask(IActivityPubService activityPubService, IStatusService statusService, IFollowersDal followersDal, InstanceSettings settings, ILogger logger) { _activityPubService = activityPubService; _statusService = statusService; _followersDal = followersDal; + _settings = settings; _logger = logger; } #endregion @@ -52,8 +55,13 @@ public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, st { try { - var note = _statusService.GetStatus(user.Acct, tweet); - await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + if (!tweet.IsReply || + tweet.IsReply && tweet.IsThread || + _settings.PublishReplies) + { + var note = _statusService.GetStatus(user.Acct, tweet); + await _activityPubService.PostNewNoteActivity(note, user.Acct, tweet.Id.ToString(), host, inbox); + } } catch (ArgumentException e) { @@ -66,7 +74,7 @@ public async Task ExecuteAsync(ExtractedTweet[] tweets, SyncTwitterUser user, st throw; } } - + syncStatus = tweet.Id; } } diff --git a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs index 8de272e..d2436f0 100644 --- a/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs +++ b/src/BirdsiteLive.Pipeline/StatusPublicationPipeline.cs @@ -20,15 +20,18 @@ public class StatusPublicationPipeline : IStatusPublicationPipeline private readonly IRetrieveTweetsProcessor _retrieveTweetsProcessor; private readonly IRetrieveFollowersProcessor _retrieveFollowersProcessor; private readonly ISendTweetsToFollowersProcessor _sendTweetsToFollowersProcessor; + private readonly ISaveProgressionProcessor _saveProgressionProcessor; private readonly ILogger _logger; #region Ctor - public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ILogger logger) + public StatusPublicationPipeline(IRetrieveTweetsProcessor retrieveTweetsProcessor, IRetrieveTwitterUsersProcessor retrieveTwitterAccountsProcessor, IRetrieveFollowersProcessor retrieveFollowersProcessor, ISendTweetsToFollowersProcessor sendTweetsToFollowersProcessor, ISaveProgressionProcessor saveProgressionProcessor, ILogger logger) { _retrieveTweetsProcessor = retrieveTweetsProcessor; _retrieveTwitterAccountsProcessor = retrieveTwitterAccountsProcessor; _retrieveFollowersProcessor = retrieveFollowersProcessor; _sendTweetsToFollowersProcessor = sendTweetsToFollowersProcessor; + _saveProgressionProcessor = saveProgressionProcessor; + _logger = logger; } #endregion @@ -41,7 +44,9 @@ public async Task ExecuteAsync(CancellationToken ct) var retrieveTweetsBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 1, CancellationToken = ct }); var retrieveFollowersBlock = new TransformManyBlock(async x => await _retrieveFollowersProcessor.ProcessAsync(x, ct)); var retrieveFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); - var sendTweetsToFollowersBlock = new ActionBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct }); + var sendTweetsToFollowersBlock = new TransformBlock(async x => await _sendTweetsToFollowersProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct }); + var sendTweetsToFollowersBufferBlock = new BufferBlock(new DataflowBlockOptions { BoundedCapacity = 20, CancellationToken = ct }); + var saveProgressionBlock = new ActionBlock(async x => await _saveProgressionProcessor.ProcessAsync(x, ct), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5, CancellationToken = ct }); // Link pipeline twitterUsersBufferBlock.LinkTo(retrieveTweetsBlock, new DataflowLinkOptions { PropagateCompletion = true }); @@ -49,14 +54,16 @@ public async Task ExecuteAsync(CancellationToken ct) retrieveTweetsBufferBlock.LinkTo(retrieveFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveFollowersBlock.LinkTo(retrieveFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); retrieveFollowersBufferBlock.LinkTo(sendTweetsToFollowersBlock, new DataflowLinkOptions { PropagateCompletion = true }); + sendTweetsToFollowersBlock.LinkTo(sendTweetsToFollowersBufferBlock, new DataflowLinkOptions { PropagateCompletion = true }); + sendTweetsToFollowersBufferBlock.LinkTo(saveProgressionBlock, new DataflowLinkOptions { PropagateCompletion = true }); // Launch twitter user retriever var retrieveTwitterAccountsTask = _retrieveTwitterAccountsProcessor.GetTwitterUsersAsync(twitterUsersBufferBlock, ct); // Wait - await Task.WhenAny(new[] { retrieveTwitterAccountsTask, sendTweetsToFollowersBlock.Completion }); + await Task.WhenAny(new[] { retrieveTwitterAccountsTask, saveProgressionBlock.Completion }); - var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : sendTweetsToFollowersBlock.Completion.Exception; + var ex = retrieveTwitterAccountsTask.IsFaulted ? retrieveTwitterAccountsTask.Exception : saveProgressionBlock.Completion.Exception; _logger.LogCritical(ex, "An error occurred, pipeline stopped"); } } diff --git a/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs new file mode 100644 index 0000000..c84b7b1 --- /dev/null +++ b/src/BirdsiteLive.Pipeline/Tools/MaxUsersNumberProvider.cs @@ -0,0 +1,49 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Pipeline.Tools +{ + public interface IMaxUsersNumberProvider + { + Task GetMaxUsersNumberAsync(); + } + + public class MaxUsersNumberProvider : IMaxUsersNumberProvider + { + private readonly InstanceSettings _instanceSettings; + private readonly ITwitterUserDal _twitterUserDal; + + private int _totalUsersCount = -1; + private int _warmUpIterations; + + #region Ctor + public MaxUsersNumberProvider(InstanceSettings instanceSettings, ITwitterUserDal twitterUserDal) + { + _instanceSettings = instanceSettings; + _twitterUserDal = twitterUserDal; + } + #endregion + + public async Task GetMaxUsersNumberAsync() + { + // Init data + if (_totalUsersCount == -1) + { + _totalUsersCount = await _twitterUserDal.GetTwitterUsersCountAsync(); + var warmUpMaxCapacity = _instanceSettings.MaxUsersCapacity / 4; + _warmUpIterations = warmUpMaxCapacity == 0 ? 0 : (int)(_totalUsersCount / (float)warmUpMaxCapacity); + } + + // Return if warm up ended + if (_warmUpIterations <= 0) return _instanceSettings.MaxUsersCapacity; + + // Calculate warm up value + var maxUsers = _warmUpIterations > 0 + ? _instanceSettings.MaxUsersCapacity / 4 + : _instanceSettings.MaxUsersCapacity; + _warmUpIterations--; + return maxUsers; + } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index f4bb8d1..573385c 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -24,7 +24,9 @@ public ExtractedTweet Extract(ITweet tweet) InReplyToAccount = tweet.InReplyToScreenName, MessageContent = ExtractMessage(tweet), Media = ExtractMedia(tweet.Media), - CreatedAt = tweet.CreatedAt.ToUniversalTime() + CreatedAt = tweet.CreatedAt.ToUniversalTime(), + IsReply = tweet.InReplyToUserId != null, + IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id }; return extractedTweet; } diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index 0363973..1f6bcea 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -11,5 +11,7 @@ public class ExtractedTweet public ExtractedMedia[] Media { get; set; } public DateTime CreatedAt { get; set; } public string InReplyToAccount { get; set; } + public bool IsReply { get; set; } + public bool IsThread { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs index ee104ae..0f82429 100644 --- a/src/BirdsiteLive.Twitter/TwitterTweetsService.cs +++ b/src/BirdsiteLive.Twitter/TwitterTweetsService.cs @@ -40,7 +40,7 @@ public ExtractedTweet GetTweet(long statusId) var tweet = Tweet.GetTweet(statusId); _statisticsHandler.CalledTweetApi(); if (tweet == null) return null; //TODO: test this - return _tweetExtractor.Extract(tweet); + return _tweetExtractor.Extract(tweet); } public ExtractedTweet[] GetTimeline(string username, int nberTweets, long fromTweetId = -1) diff --git a/src/BirdsiteLive.sln b/src/BirdsiteLive.sln index bf78d55..0a35bf6 100644 --- a/src/BirdsiteLive.sln +++ b/src/BirdsiteLive.sln @@ -39,6 +39,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Domain.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BirdsiteLive.Pipeline.Tests", "Tests\BirdsiteLive.Pipeline.Tests\BirdsiteLive.Pipeline.Tests.csproj", "{BF51CA81-5A7A-46F8-B4FB-861C6BE59298}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BirdsiteLive.DAL.Tests", "Tests\BirdsiteLive.DAL.Tests\BirdsiteLive.DAL.Tests.csproj", "{5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -101,6 +103,10 @@ Global {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Debug|Any CPU.Build.0 = Debug|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.ActiveCfg = Release|Any CPU {BF51CA81-5A7A-46F8-B4FB-861C6BE59298}.Release|Any CPU.Build.0 = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -119,6 +125,7 @@ Global {2A8CC30D-D775-47D1-9388-F72A5C32DE2A} = {DA3C160C-4811-4E26-A5AD-42B81FAF2D7C} {F544D745-89A8-4DEA-B61C-A7E6C53C1D63} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} {BF51CA81-5A7A-46F8-B4FB-861C6BE59298} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} + {5A1E3EB5-6CBB-470D-8A0D-10F8C18353D5} = {A32D3458-09D0-4E0A-BA4B-8C411B816B94} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {69E8DCAD-4C37-4010-858F-5F94E6FBABCE} diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index 4729487..df9c11b 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.10.1 + 0.11.0 diff --git a/src/BirdsiteLive/Services/FederationService.cs b/src/BirdsiteLive/Services/FederationService.cs index f2c2e94..9acab41 100644 --- a/src/BirdsiteLive/Services/FederationService.cs +++ b/src/BirdsiteLive/Services/FederationService.cs @@ -1,6 +1,8 @@ using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using BirdsiteLive.DAL; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.Pipeline; using Microsoft.Extensions.Hosting; @@ -9,36 +11,21 @@ namespace BirdsiteLive.Services { public class FederationService : BackgroundService { - private readonly IDbInitializerDal _dbInitializerDal; + private readonly IDatabaseInitializer _databaseInitializer; private readonly IStatusPublicationPipeline _statusPublicationPipeline; #region Ctor - public FederationService(IDbInitializerDal dbInitializerDal, IStatusPublicationPipeline statusPublicationPipeline) + public FederationService(IDatabaseInitializer databaseInitializer, IStatusPublicationPipeline statusPublicationPipeline) { - _dbInitializerDal = dbInitializerDal; + _databaseInitializer = databaseInitializer; _statusPublicationPipeline = statusPublicationPipeline; } #endregion protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await DbInitAsync(); + await _databaseInitializer.InitAndMigrateDbAsync(); await _statusPublicationPipeline.ExecuteAsync(stoppingToken); } - - private async Task DbInitAsync() - { - var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); - var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); - - if (currentVersion == null) - { - await _dbInitializerDal.InitDbAsync(); - } - else if (currentVersion != mandatoryVersion) - { - throw new NotImplementedException(); - } - } } } \ No newline at end of file diff --git a/src/BirdsiteLive/appsettings.json b/src/BirdsiteLive/appsettings.json index 57bf945..3dc47ff 100644 --- a/src/BirdsiteLive/appsettings.json +++ b/src/BirdsiteLive/appsettings.json @@ -13,7 +13,9 @@ "Name": "BirdsiteLIVE", "Domain": "domain.name", "AdminEmail": "me@domain.name", - "ResolveMentionsInProfiles": true + "ResolveMentionsInProfiles": true, + "PublishReplies": false, + "MaxUsersCapacity": 1400 }, "Db": { "Type": "postgres", diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index c6d5c81..ca883ff 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ internal class DbVersion public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(1,0); + private readonly Version _currentVersion = new Version(2, 0); private const string DbVersionType = "db-version"; #region Ctor @@ -32,7 +32,7 @@ public DbInitializerPostgresDal(PostgresSettings settings, PostgresTools tools) _tools = tools; } #endregion - + public async Task GetCurrentDbVersionAsync() { var query = $"SELECT * FROM {_settings.DbVersionTableName} WHERE type = @type"; @@ -65,17 +65,7 @@ public Version GetMandatoryDbVersion() return _currentVersion; } - public Tuple[] GetMigrationPatterns() - { - return new Tuple[0]; - } - - public Task MigrateDbAsync(Version from, Version to) - { - throw new NotImplementedException(); - } - - public async Task InitDbAsync() + public async Task InitDbAsync() { // Create version table var createVersion = $@"CREATE TABLE {_settings.DbVersionTableName} @@ -124,13 +114,53 @@ data JSONB await _tools.ExecuteRequestAsync(createCachedTweets); // Insert version to db + var firstVersion = new Version(1, 0); using (var dbConnection = Connection) { dbConnection.Open(); await dbConnection.ExecuteAsync( $"INSERT INTO {_settings.DbVersionTableName} (type,major,minor) VALUES(@type,@major,@minor)", - new { type = DbVersionType, major = _currentVersion.Major, minor = _currentVersion.Minor }); + new { type = DbVersionType, major = firstVersion.Major, minor = firstVersion.Minor }); + } + + return firstVersion; + } + + public Tuple[] GetMigrationPatterns() + { + return new[] + { + new Tuple(new Version(1,0), new Version(2,0)) + }; + } + + public async Task MigrateDbAsync(Version from, Version to) + { + if (from == new Version(1, 0) && to == new Version(2, 0)) + { + var addLastSync = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD lastSync TIMESTAMP (2) WITHOUT TIME ZONE"; + await _tools.ExecuteRequestAsync(addLastSync); + + var addIndex = $@"CREATE INDEX IF NOT EXISTS lastsync_twitteruser ON {_settings.TwitterUserTableName}(lastSync)"; + await _tools.ExecuteRequestAsync(addIndex); + + await UpdateDbVersionAsync(to); + return to; + } + + throw new NotImplementedException(); + } + + private async Task UpdateDbVersionAsync(Version newVersion) + { + using (var dbConnection = Connection) + { + dbConnection.Open(); + + await dbConnection.ExecuteAsync( + $"UPDATE {_settings.DbVersionTableName} SET major = @major, minor = @minor WHERE type = @type", + new { type = DbVersionType, major = newVersion.Major, minor = newVersion.Minor }); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 082229a..afbf7d1 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -62,32 +62,33 @@ public async Task GetTwitterUsersCountAsync() } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(int maxNumber) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC LIMIT @maxNumber"; using (var dbConnection = Connection) { dbConnection.Open(); - var result = await dbConnection.QueryAsync(query); + var result = await dbConnection.QueryAsync(query, new { maxNumber }); return result.ToArray(); } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); - - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId WHERE id = @id"; + if(lastSync == default) throw new ArgumentException("lastSync"); + + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, lastSync = @lastSync WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId }); + await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, lastSync = lastSync.ToUniversalTime() }); } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs index 223b1ea..32a379f 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/Tools/PostgresTools.cs @@ -18,18 +18,11 @@ public PostgresTools(PostgresSettings settings) public async Task ExecuteRequestAsync(string request) { - try + using (var conn = new NpgsqlConnection(_settings.ConnString)) + using (var cmd = new NpgsqlCommand(request, conn)) { - using (var conn = new NpgsqlConnection(_settings.ConnString)) - using (var cmd = new NpgsqlCommand(request, conn)) - { - await conn.OpenAsync(); - await cmd.ExecuteNonQueryAsync(); - } - } - catch (Exception e) - { - Console.WriteLine(e); + await conn.OpenAsync(); + await cmd.ExecuteNonQueryAsync(); } } } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs index b786386..9d7db56 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/IDbInitializerDal.cs @@ -9,7 +9,7 @@ public interface IDbInitializerDal Task GetCurrentDbVersionAsync(); Version GetMandatoryDbVersion(); Tuple[] GetMigrationPatterns(); - Task MigrateDbAsync(Version from, Version to); - Task InitDbAsync(); + Task MigrateDbAsync(Version from, Version to); + Task InitDbAsync(); } } \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index 48d5661..1fa8127 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using BirdsiteLive.DAL.Models; namespace BirdsiteLive.DAL.Contracts @@ -7,8 +8,8 @@ public interface ITwitterUserDal { Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); Task GetTwitterUserAsync(string acct); - Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId); + Task GetAllTwitterUsersAsync(int maxNumber); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, DateTime lastSync); Task DeleteTwitterUserAsync(string acct); Task GetTwitterUsersCountAsync(); } diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs new file mode 100644 index 0000000..39e1e84 --- /dev/null +++ b/src/DataAccessLayers/BirdsiteLive.DAL/DatabaseInitializer.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.DAL +{ + public interface IDatabaseInitializer + { + Task InitAndMigrateDbAsync(); + } + + public class DatabaseInitializer : IDatabaseInitializer + { + private readonly IDbInitializerDal _dbInitializerDal; + + #region Ctor + public DatabaseInitializer(IDbInitializerDal dbInitializerDal) + { + _dbInitializerDal = dbInitializerDal; + } + #endregion + + public async Task InitAndMigrateDbAsync() + { + var currentVersion = await _dbInitializerDal.GetCurrentDbVersionAsync(); + var mandatoryVersion = _dbInitializerDal.GetMandatoryDbVersion(); + + if (currentVersion == mandatoryVersion) return; + + // Init Db + if (currentVersion == null) + currentVersion = await _dbInitializerDal.InitDbAsync(); + + // Migrate Db + var migrationPatterns = _dbInitializerDal.GetMigrationPatterns(); + while (migrationPatterns.Any(x => x.Item1 == currentVersion)) + { + var migration = migrationPatterns.First(x => x.Item1 == currentVersion); + currentVersion = await _dbInitializerDal.MigrateDbAsync(migration.Item1, migration.Item2); + } + + if (currentVersion != mandatoryVersion) throw new Exception("Migrating DB failed"); + } + } +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8061fc8..59be0a5 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -1,4 +1,6 @@ -namespace BirdsiteLive.DAL.Models +using System; + +namespace BirdsiteLive.DAL.Models { public class SyncTwitterUser { @@ -7,5 +9,7 @@ public class SyncTwitterUser public long LastTweetPostedId { get; set; } public long LastTweetSynchronizedForAllFollowersId { get; set; } + + public DateTime LastSync { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs index 021784e..24672c3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/CachedTweetsPostgresDalTests.cs @@ -14,21 +14,15 @@ public class CachedTweetsPostgresDalTests : PostgresTestingBase public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs index 7fc5383..a186dc3 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/DbInitializerPostgresDalTests.cs @@ -17,17 +17,14 @@ public async Task CleanUp() { await dal.DeleteAllAsync(); } - catch (Exception e) - { - Console.WriteLine(e); - } + catch (Exception ) { } } [TestMethod] public async Task GetCurrentDbVersionAsync_UninitializedDb() { var dal = new DbInitializerPostgresDal(_settings, _tools); - + var current = await dal.GetCurrentDbVersionAsync(); Assert.IsNull(current); } @@ -35,11 +32,11 @@ public async Task GetCurrentDbVersionAsync_UninitializedDb() [TestMethod] public async Task InitDbAsync() { + var mandatory = new Version(1, 0); var dal = new DbInitializerPostgresDal(_settings, _tools); await dal.InitDbAsync(); var current = await dal.GetCurrentDbVersionAsync(); - var mandatory = dal.GetMandatoryDbVersion(); Assert.IsNotNull(current); Assert.AreEqual(mandatory.Minor, current.Minor); Assert.AreEqual(mandatory.Major, current.Major); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs index cd6162d..cf08856 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/FollowersPostgresDalTests.cs @@ -16,21 +16,15 @@ public class FollowersPostgresDalTests : PostgresTestingBase public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] @@ -38,7 +32,7 @@ public async Task CreateAndGetFollower() { var acct = "myhandle"; var host = "domain.ext"; - var following = new[] {12, 19, 23}; + var following = new[] { 12, 19, 23 }; var followingSync = new Dictionary() { {12, 165L}, @@ -47,7 +41,7 @@ public async Task CreateAndGetFollower() }; var inboxRoute = "/myhandle/inbox"; var sharedInboxRoute = "/inbox"; - + var dal = new FollowersPostgresDal(_settings); await dal.CreateFollowerAsync(acct, host, inboxRoute, sharedInboxRoute, following, followingSync); @@ -105,7 +99,7 @@ public async Task GetFollowersAsync() //User 1 var acct = "myhandle1"; var host = "domain.ext"; - var following = new[] { 1,2,3 }; + var following = new[] { 1, 2, 3 }; var followingSync = new Dictionary(); var inboxRoute = "/myhandle1/inbox"; var sharedInboxRoute = "/inbox"; @@ -202,7 +196,7 @@ public async Task CreateUpdateAndGetFollower_Add() }; result.Followings = updatedFollowing.ToList(); result.FollowingsSyncStatus = updatedFollowingSync; - + await dal.UpdateFollowerAsync(result); result = await dal.GetFollowerAsync(acct, host); diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index f900c8f..d71842f 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -14,21 +14,15 @@ public class TwitterUserPostgresDalTests : PostgresTestingBase public async Task TestInit() { var dal = new DbInitializerPostgresDal(_settings, _tools); - await dal.InitDbAsync(); + var init = new DatabaseInitializer(dal); + await init.InitAndMigrateDbAsync(); } [TestCleanup] public async Task CleanUp() { var dal = new DbInitializerPostgresDal(_settings, _tools); - try - { - await dal.DeleteAllAsync(); - } - catch (Exception e) - { - Console.WriteLine(e); - } + await dal.DeleteAllAsync(); } [TestMethod] @@ -70,13 +64,15 @@ public async Task CreateUpdateAndGetUser() var updatedLastTweetId = 1600L; var updatedLastSyncId = 1550L; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId); + var now = DateTime.Now; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, now); result = await dal.GetTwitterUserAsync(acct); Assert.AreEqual(acct, result.Acct); Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); } [TestMethod] @@ -108,7 +104,7 @@ public async Task GetAllTwitterUsers() await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(); + var result = await dal.GetAllTwitterUsersAsync(1000); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -116,6 +112,41 @@ public async Task GetAllTwitterUsers() Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } + [TestMethod] + public async Task GetAllTwitterUsers_Limited() + { + var now = DateTime.Now; + var oldest = now.AddDays(-3); + var newest = now.AddDays(-2); + + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 20; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + var allUsers = await dal.GetAllTwitterUsersAsync(100); + for (var i = 0; i < 20; i++) + { + var user = allUsers[i]; + var date = i % 2 == 0 ? oldest : newest; + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, date); + } + + var result = await dal.GetAllTwitterUsersAsync(10); + Assert.AreEqual(10, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var acc in result) + Assert.IsTrue(Math.Abs((acc.LastSync - oldest.ToUniversalTime()).TotalMilliseconds) < 1000); + } + [TestMethod] public async Task CountTwitterUsers() { diff --git a/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj new file mode 100644 index 0000000..0992b02 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/BirdsiteLive.DAL.Tests.csproj @@ -0,0 +1,21 @@ + + + + netcoreapp3.1 + + false + + + + + + + + + + + + + + + diff --git a/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs new file mode 100644 index 0000000..ba11321 --- /dev/null +++ b/src/Tests/BirdsiteLive.DAL.Tests/DatabaseInitializerTests.cs @@ -0,0 +1,240 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using BirdsiteLive.DAL.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.DAL.Tests +{ + [TestClass] + public class DatabaseInitializerTests + { + [TestMethod] + public async Task DbInitAsync_UpToDate_Test() + { + #region Stubs + var current = new Version(2, 3); + var mandatory = new Version(2, 3); + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(1, 0); + + var migrationPatterns = new Tuple[0]; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_NoDb_Migration_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task DbInitAsync_HasDb_Migration_Test() + { + #region Stubs + var current = new Version(1, 7); + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,3)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns.Skip(1)) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + await dbInitializer.InitAndMigrateDbAsync(); + + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(Exception))] + public async Task DbInitAsync_NoDb_Migration_Error_Test() + { + #region Stubs + var current = (Version)null; + var mandatory = new Version(2, 3); + + var migrationPatterns = new Tuple[] + { + new Tuple(new Version(1,0), new Version(1,7)), + new Tuple(new Version(1,7), new Version(2,0)), + new Tuple(new Version(2,0), new Version(2,2)) + }; + #endregion + + #region Mocks + var dbInitializerDal = new Mock(MockBehavior.Strict); + + dbInitializerDal + .Setup(x => x.GetCurrentDbVersionAsync()) + .ReturnsAsync(current); + + dbInitializerDal + .Setup(x => x.GetMandatoryDbVersion()) + .Returns(mandatory); + + dbInitializerDal + .Setup(x => x.InitDbAsync()) + .ReturnsAsync(new Version(1, 0)); + + dbInitializerDal + .Setup(x => x.GetMigrationPatterns()) + .Returns(migrationPatterns); + + foreach (var m in migrationPatterns) + { + dbInitializerDal + .Setup(x => x.MigrateDbAsync( + It.Is(y => y == m.Item1), + It.Is(y => y == m.Item2) + )) + .ReturnsAsync(m.Item2); + } + #endregion + + var dbInitializer = new DatabaseInitializer(dbInitializerDal.Object); + try + { + await dbInitializer.InitAndMigrateDbAsync(); + } + finally + { + #region Validations + dbInitializerDal.VerifyAll(); + #endregion + } + } + } +} diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj index 3dd6984..aa7750b 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/BirdsiteLive.Pipeline.Tests.csproj @@ -18,4 +18,8 @@ + + + + diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 2bf5d74..d66c2f7 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -54,7 +55,8 @@ public async Task ProcessAsync_UserNotSync_Test() .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user1.Id), It.Is(y => y == tweets.Last().Id), - It.Is(y => y == tweets.Last().Id) + It.Is(y => y == tweets.Last().Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 6cbd008..5600e1c 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -3,9 +3,11 @@ using System.Threading; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Pipeline.Processors; +using BirdsiteLive.Pipeline.Tools; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -26,24 +28,32 @@ public async Task GetTwitterUsersAsync_Test() new SyncTwitterUser(), new SyncTwitterUser(), }; + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(3, buffer.Count); buffer.TryReceive(out var result); @@ -60,25 +70,37 @@ public async Task GetTwitterUsersAsync_Multi_Test() for (var i = 0; i < 30; i++) users.Add(new SyncTwitterUser()); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); - await Task.Delay(200); + await Task.Delay(300); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(15, buffer.Count); buffer.TryReceive(out var result); @@ -95,25 +117,37 @@ public async Task GetTwitterUsersAsync_Multi2_Test() for (var i = 0; i < 31; i++) users.Add(new SyncTwitterUser()); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .SetupSequence(x => x.GetAllTwitterUsersAsync()) + .SetupSequence(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(users.ToArray()) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) + .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]); - + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 2; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(200); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(11, buffer.Count); buffer.TryReceive(out var result); @@ -126,24 +160,33 @@ public async Task GetTwitterUsersAsync_NoUsers_Test() { #region Stubs var buffer = new BufferBlock(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.Delay(50); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -154,24 +197,33 @@ public async Task GetTwitterUsersAsync_Exception_Test() { #region Stubs var buffer = new BufferBlock(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync( + It.Is(y => y == maxUsers))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 10; var t = processor.GetTwitterUsersAsync(buffer, CancellationToken.None); await Task.WhenAny(t, Task.Delay(50)); #region Validations + maxUsersNumberProviderMock.VerifyAll(); twitterUserDalMock.VerifyAll(); Assert.AreEqual(0, buffer.Count); #endregion @@ -185,14 +237,22 @@ public async Task GetTwitterUsersAsync_Cancellation_Test() var buffer = new BufferBlock(); var canTokenS = new CancellationTokenSource(); canTokenS.Cancel(); + + var maxUsers = 1000; #endregion #region Mocks + var maxUsersNumberProviderMock = new Mock(MockBehavior.Strict); + maxUsersNumberProviderMock + .Setup(x => x.GetMaxUsersNumberAsync()) + .ReturnsAsync(maxUsers); + var twitterUserDalMock = new Mock(MockBehavior.Strict); + var loggerMock = new Mock>(); #endregion - var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new RetrieveTwitterUsersProcessor(twitterUserDalMock.Object, maxUsersNumberProviderMock.Object, loggerMock.Object); processor.WaitFactor = 1; await processor.GetTwitterUsersAsync(buffer, canTokenS.Token); } diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index d3880e6..b2a99b9 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; @@ -60,7 +61,8 @@ public async Task ProcessAsync_Test() .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet2.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -123,7 +125,8 @@ public async Task ProcessAsync_PartiallySynchronized_Test() .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion @@ -194,7 +197,8 @@ public async Task ProcessAsync_PartiallySynchronized_MultiUsers_Test() .Setup(x => x.UpdateTwitterUserAsync( It.Is(y => y == user.Id), It.Is(y => y == tweet3.Id), - It.Is(y => y == tweet2.Id) + It.Is(y => y == tweet2.Id), + It.IsAny() )) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs index 23c9fe3..367a642 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToInboxTaskTests.cs @@ -4,6 +4,7 @@ using System.Net.Http; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; @@ -54,6 +55,243 @@ public async Task ExecuteAsync_SingleTweet_Test() InboxRoute = inbox, FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .Returns(Task.CompletedTask); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_Reply_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = false + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + var statusServiceMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_ReplyThread_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = true + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .Returns(Task.CompletedTask); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_PublishReply_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = false + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + + var settings = new InstanceSettings + { + PublishReplies = true + }; #endregion #region Mocks @@ -83,7 +321,7 @@ public async Task ExecuteAsync_SingleTweet_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations @@ -126,6 +364,11 @@ public async Task ExecuteAsync_MultipleTweets_Test() InboxRoute = inbox, FollowingsSyncStatus = new Dictionary { { twitterUserId, 10 } } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; #endregion #region Mocks @@ -161,7 +404,7 @@ public async Task ExecuteAsync_MultipleTweets_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); #region Validations @@ -205,6 +448,11 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() InboxRoute = inbox, FollowingsSyncStatus = new Dictionary { { twitterUserId, 10 } } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; #endregion #region Mocks @@ -247,7 +495,7 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); try { @@ -262,5 +510,147 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() #endregion } } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_ParsingError_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Throws(new ArgumentException("Invalid pattern blabla at offset 9")); + + var followersDalMock = new Mock(MockBehavior.Strict); + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/user/inbox"; + var follower = new Follower + { + Id = 1, + Host = host, + InboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Throws(new ArgumentException()); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + + try + { + await task.ExecuteAsync(tweets.ToArray(), follower, twitterUser); + + } + finally + { + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs index 5ae9c7f..7ab06a2 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SubTasks/SendTweetsToSharedInboxTests.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Threading.Tasks; using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.Common.Settings; using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; @@ -72,6 +73,307 @@ public async Task ExecuteAsync_SingleTweet_Test() FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .Returns(Task.CompletedTask); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + } + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_Reply_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = false + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + var statusServiceMock = new Mock(MockBehavior.Strict); + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + } + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_ReplyThread_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = true + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + activityPubService + .Setup(x => x.PostNewNoteActivity( + It.Is(y => y.id == noteId), + It.Is(y => y == twitterHandle), + It.Is(y => y == tweetId.ToString()), + It.Is(y => y == host), + It.Is(y => y == inbox))) + .Returns(Task.CompletedTask); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Returns(note); + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + } + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_PublishReply_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + IsReply = true, + IsThread = false + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + + var settings = new InstanceSettings + { + PublishReplies = true + }; #endregion #region Mocks @@ -105,7 +407,7 @@ public async Task ExecuteAsync_SingleTweet_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations @@ -165,6 +467,11 @@ public async Task ExecuteAsync_MultipleTweets_Test() FollowingsSyncStatus = new Dictionary {{twitterUserId, 7}} } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; #endregion #region Mocks @@ -204,7 +511,7 @@ public async Task ExecuteAsync_MultipleTweets_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); #region Validations @@ -265,6 +572,11 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() FollowingsSyncStatus = new Dictionary {{twitterUserId, 7}} } }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; #endregion #region Mocks @@ -311,11 +623,191 @@ public async Task ExecuteAsync_MultipleTweets_Error_Test() var loggerMock = new Mock>(); #endregion - var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, loggerMock.Object); + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + + try + { + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + } + finally + { + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + } + + [TestMethod] + public async Task ExecuteAsync_SingleTweet_ParsingError_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var noteId = "noteId"; + var note = new Note() + { + id = noteId + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Throws(new ArgumentException("Invalid pattern blabla at offset 9")); + + var followersDalMock = new Mock(MockBehavior.Strict); + + foreach (var follower in followers) + { + followersDalMock + .Setup(x => x.UpdateFollowerAsync( + It.Is(y => y.Id == follower.Id && y.FollowingsSyncStatus[twitterUserId] == tweetId))) + .Returns(Task.CompletedTask); + } + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); + await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + + #region Validations + activityPubService.VerifyAll(); + statusServiceMock.VerifyAll(); + followersDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public async Task ExecuteAsync_SingleTweet_ArgumentException_Test() + { + #region Stubs + var tweetId = 10; + var tweets = new List + { + new ExtractedTweet + { + Id = tweetId, + } + }; + + var twitterHandle = "Test"; + var twitterUserId = 7; + var twitterUser = new SyncTwitterUser + { + Id = twitterUserId, + Acct = twitterHandle + }; + + var host = "domain.ext"; + var inbox = "/inbox"; + var followers = new List + { + new Follower + { + Id = 1, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 9 } } + }, + new Follower + { + Id = 2, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 8 } } + }, + new Follower + { + Id = 3, + Host = host, + SharedInboxRoute = inbox, + FollowingsSyncStatus = new Dictionary { { twitterUserId, 7 } } + } + }; + + var settings = new InstanceSettings + { + PublishReplies = false + }; + #endregion + + #region Mocks + var activityPubService = new Mock(MockBehavior.Strict); + + var statusServiceMock = new Mock(MockBehavior.Strict); + statusServiceMock + .Setup(x => x.GetStatus( + It.Is(y => y == twitterHandle), + It.Is(y => y.Id == tweetId))) + .Throws(new ArgumentException()); + + var followersDalMock = new Mock(MockBehavior.Strict); + + var loggerMock = new Mock>(); + #endregion + + var task = new SendTweetsToSharedInboxTask(activityPubService.Object, statusServiceMock.Object, followersDalMock.Object, settings, loggerMock.Object); try { await task.ExecuteAsync(tweets.ToArray(), twitterUser, host, followers.ToArray()); + } finally { diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs new file mode 100644 index 0000000..d48beb8 --- /dev/null +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Tools/MaxUsersNumberProviderTests.cs @@ -0,0 +1,79 @@ +using System.Threading.Tasks; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.Pipeline.Tools; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace BirdsiteLive.Pipeline.Tests.Tools +{ + [TestClass] + public class MaxUsersNumberProviderTests + { + [TestMethod] + public async Task GetMaxUsersNumberAsync_WarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(1000); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(250, result); + + result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task GetMaxUsersNumberAsync_NoWarmUp_Test() + { + #region Stubs + var settings = new InstanceSettings + { + MaxUsersCapacity = 1000 + }; + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.GetTwitterUsersCountAsync()) + .ReturnsAsync(249); + #endregion + + var provider = new MaxUsersNumberProvider(settings, twitterUserDalMock.Object); + + var result = await provider.GetMaxUsersNumberAsync(); + Assert.AreEqual(1000, result); + + #region Validations + twitterUserDalMock.VerifyAll(); + #endregion + } + } +} \ No newline at end of file