diff --git a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs index deb7e7f..c628a61 100644 --- a/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs +++ b/src/BirdsiteLive.ActivityPub/Models/ActivityDelete.cs @@ -4,6 +4,7 @@ namespace BirdsiteLive.ActivityPub.Models { public class ActivityDelete : Activity { + public string[] to { get; set; } [JsonProperty("object")] public object apObject { get; set; } } diff --git a/src/BirdsiteLive.ActivityPub/Models/Actor.cs b/src/BirdsiteLive.ActivityPub/Models/Actor.cs index 713ea89..ea4f8a3 100644 --- a/src/BirdsiteLive.ActivityPub/Models/Actor.cs +++ b/src/BirdsiteLive.ActivityPub/Models/Actor.cs @@ -17,6 +17,7 @@ public class Actor public string name { get; set; } public string summary { get; set; } public string url { get; set; } + public string movedTo { get; set; } public bool manuallyApprovesFollowers { get; set; } public string inbox { get; set; } public bool? discoverable { get; set; } = true; diff --git a/src/BirdsiteLive.Domain/ActivityPubService.cs b/src/BirdsiteLive.Domain/ActivityPubService.cs index c460a2d..cab0c58 100644 --- a/src/BirdsiteLive.Domain/ActivityPubService.cs +++ b/src/BirdsiteLive.Domain/ActivityPubService.cs @@ -16,10 +16,18 @@ namespace BirdsiteLive.Domain { public interface IActivityPubService { + Task GetUserIdAsync(string acct); Task GetUser(string objectId); Task PostDataAsync(T data, string targetHost, string actorUrl, string inbox = null); Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox); + Task DeleteUserAsync(string username, string targetHost, string targetInbox); + } + + public class WebFinger + { + public string subject { get; set; } + public string[] aliases { get; set; } } public class ActivityPubService : IActivityPubService @@ -39,6 +47,24 @@ public ActivityPubService(ICryptoService cryptoService, InstanceSettings instanc } #endregion + public async Task GetUserIdAsync(string acct) + { + var splittedAcct = acct.Trim('@').Split('@'); + + var url = $"https://{splittedAcct[1]}/.well-known/webfinger?resource=acct:{splittedAcct[0]}@{splittedAcct[1]}"; + + var httpClient = _httpClientFactory.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + var result = await httpClient.GetAsync(url); + + result.EnsureSuccessStatusCode(); + + var content = await result.Content.ReadAsStringAsync(); + + var actor = JsonConvert.DeserializeObject(content); + return actor.aliases.FirstOrDefault(); + } + public async Task GetUser(string objectId) { var httpClient = _httpClientFactory.CreateClient(); @@ -57,6 +83,31 @@ public async Task GetUser(string objectId) return actor; } + public async Task DeleteUserAsync(string username, string targetHost, string targetInbox) + { + try + { + var actor = UrlFactory.GetActorUrl(_instanceSettings.Domain, username); + + var deleteUser = new ActivityDelete + { + context = "https://www.w3.org/ns/activitystreams", + id = $"{actor}#delete", + type = "Delete", + actor = actor, + to = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + apObject = actor + }; + + await PostDataAsync(deleteUser, targetHost, actor, targetInbox); + } + catch (Exception e) + { + _logger.LogError(e, "Error deleting {Username} to {Host}{Inbox}", username, targetHost, targetInbox); + throw; + } + } + public async Task PostNewNoteActivity(Note note, string username, string noteId, string targetHost, string targetInbox) { try diff --git a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj index 8c601b4..7bc9873 100644 --- a/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj +++ b/src/BirdsiteLive.Domain/BirdsiteLive.Domain.csproj @@ -15,4 +15,8 @@ + + + + diff --git a/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs new file mode 100644 index 0000000..92b1444 --- /dev/null +++ b/src/BirdsiteLive.Domain/Enum/MigrationTypeEnum.cs @@ -0,0 +1,9 @@ +namespace BirdsiteLive.Domain.Enum +{ + public enum MigrationTypeEnum + { + Unknown = 0, + Migration = 1, + Deletion = 2 + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/MigrationService.cs b/src/BirdsiteLive.Domain/MigrationService.cs new file mode 100644 index 0000000..975272e --- /dev/null +++ b/src/BirdsiteLive.Domain/MigrationService.cs @@ -0,0 +1,281 @@ +using System; +using System.Linq; +using BirdsiteLive.Twitter; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using BirdsiteLive.ActivityPub; +using BirdsiteLive.ActivityPub.Models; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.ActivityPub.Converters; +using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Domain.Enum; + +namespace BirdsiteLive.Domain +{ + public class MigrationService + { + private readonly InstanceSettings _instanceSettings; + private readonly ITwitterTweetsService _twitterTweetsService; + private readonly IActivityPubService _activityPubService; + private readonly ITwitterUserDal _twitterUserDal; + private readonly IFollowersDal _followersDal; + + #region Ctor + public MigrationService(ITwitterTweetsService twitterTweetsService, IActivityPubService activityPubService, ITwitterUserDal twitterUserDal, IFollowersDal followersDal, InstanceSettings instanceSettings) + { + _twitterTweetsService = twitterTweetsService; + _activityPubService = activityPubService; + _twitterUserDal = twitterUserDal; + _followersDal = followersDal; + _instanceSettings = instanceSettings; + } + #endregion + + public string GetMigrationCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-MigrationCode|{hash.Substring(0, 10)}]]"; + } + + public string GetDeletionCode(string acct) + { + var hash = GetHashString(acct); + return $"[[BirdsiteLIVE-DeletionCode|{hash.Substring(0, 10)}]]"; + } + + public bool ValidateTweet(string acct, string tweetId, MigrationTypeEnum type) + { + string code; + if (type == MigrationTypeEnum.Migration) + code = GetMigrationCode(acct); + else if (type == MigrationTypeEnum.Deletion) + code = GetDeletionCode(acct); + else + throw new NotImplementedException(); + + var castedTweetId = ExtractedTweetId(tweetId); + var tweet = _twitterTweetsService.GetTweet(castedTweetId); + + if (tweet == null) + throw new Exception("Tweet not found"); + + if (tweet.CreatorName.Trim().ToLowerInvariant() != acct.Trim().ToLowerInvariant()) + throw new Exception($"Tweet not published by @{acct}"); + + if (!tweet.MessageContent.Contains(code)) + { + var message = "Tweet don't have migration code"; + if (type == MigrationTypeEnum.Deletion) + message = "Tweet don't have deletion code"; + + throw new Exception(message); + } + + return true; + } + + private long ExtractedTweetId(string tweetId) + { + if (string.IsNullOrWhiteSpace(tweetId)) + throw new ArgumentException("No provided Tweet ID"); + + long castedId; + if (long.TryParse(tweetId, out castedId)) + return castedId; + + var urlPart = tweetId.Split('/').LastOrDefault(); + if (long.TryParse(urlPart, out castedId)) + return castedId; + + throw new ArgumentException("Unvalid Tweet ID"); + } + + public async Task ValidateFediverseAcctAsync(string fediverseAcct) + { + if (string.IsNullOrWhiteSpace(fediverseAcct)) + throw new ArgumentException("Please provide Fediverse account"); + + if (!fediverseAcct.Contains('@') || !fediverseAcct.StartsWith("@") || fediverseAcct.Trim('@').Split('@').Length != 2) + throw new ArgumentException("Please provide valid Fediverse handle"); + + var objectId = await _activityPubService.GetUserIdAsync(fediverseAcct); + var user = await _activityPubService.GetUser(objectId); + + var result = new ValidatedFediverseUser + { + FediverseAcct = fediverseAcct, + ObjectId = objectId, + User = user, + IsValid = user != null + }; + + return result; + } + + public async Task MigrateAccountAsync(ValidatedFediverseUser validatedUser, string acct) + { + // Apply moved to + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1, validatedUser.ObjectId, validatedUser.FediverseAcct); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + } + + twitterAccount.MovedTo = validatedUser.User.id; + twitterAccount.MovedToAcct = validatedUser.FediverseAcct; + twitterAccount.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been disabled by its original owner.
It has been redirected to {validatedUser.FediverseAcct}.

"; + NotifyFollowers(acct, twitterAccount, message); + } + + private void NotifyFollowers(string acct, SyncTwitterUser twitterAccount, string message) + { + var t = Task.Run(async () => + { + var followers = await _followersDal.GetFollowersAsync(twitterAccount.Id); + foreach (var follower in followers) + { + try + { + var noteId = Guid.NewGuid().ToString(); + var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, acct); + var noteUrl = UrlFactory.GetNoteUrl(_instanceSettings.Domain, acct, noteId); + + //var to = validatedUser.ObjectId; + var to = follower.ActorId; + var cc = new string[0]; + + var note = new Note + { + id = noteUrl, + + published = DateTime.UtcNow.ToString("s") + "Z", + url = noteUrl, + attributedTo = actorUrl, + + to = new[] { to }, + cc = cc, + + content = message, + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = follower.ActorId, + name = $"@{follower.Acct}@{follower.Host}" + } + }, + }; + + if (!string.IsNullOrWhiteSpace(follower.SharedInboxRoute)) + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.SharedInboxRoute); + else + await _activityPubService.PostNewNoteActivity(note, acct, noteId, follower.Host, follower.InboxRoute); + } + catch (Exception e) + { + Console.WriteLine(e); + } + } + }); + } + + public async Task DeleteAccountAsync(string acct) + { + // Apply deleted state + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + if (twitterAccount == null) + { + await _twitterUserDal.CreateTwitterUserAsync(acct, -1); + twitterAccount = await _twitterUserDal.GetTwitterUserAsync(acct); + } + + twitterAccount.Deleted = true; + twitterAccount.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(twitterAccount); + + // Notify Followers + var message = $@"

[BSL MIRROR SERVICE NOTIFICATION]
This bot has been deleted by its original owner.

"; + NotifyFollowers(acct, twitterAccount, message); + + // Delete remote accounts + DeleteRemoteAccounts(acct); + } + + private void DeleteRemoteAccounts(string acct) + { + var t = Task.Run(async () => + { + var allUsers = await _followersDal.GetAllFollowersAsync(); + + var followersWtSharedInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .GroupBy(x => x.Host) + .ToList(); + foreach (var followerGroup in followersWtSharedInbox) + { + var host = followerGroup.First().Host; + var sharedInbox = followerGroup.First().SharedInboxRoute; + + var t1 = Task.Run(async () => + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + }); + } + + var followerWtInbox = allUsers + .Where(x => !string.IsNullOrWhiteSpace(x.SharedInboxRoute)) + .ToList(); + foreach (var followerGroup in followerWtInbox) + { + var host = followerGroup.Host; + var sharedInbox = followerGroup.InboxRoute; + + var t1 = Task.Run(async () => + { + await _activityPubService.DeleteUserAsync(acct, host, sharedInbox); + }); + } + }); + } + + public async Task TriggerRemoteMigrationAsync(string id, string tweetid, string handle) + { + //TODO + } + + public async Task TriggerRemoteDeleteAsync(string id, string tweetid) + { + //TODO + } + + private byte[] GetHash(string inputString) + { + using (HashAlgorithm algorithm = SHA256.Create()) + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + private string GetHashString(string inputString) + { + StringBuilder sb = new StringBuilder(); + foreach (byte b in GetHash(inputString)) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } + } + + public class ValidatedFediverseUser + { + public string FediverseAcct { get; set; } + public string ObjectId { get; set; } + public Actor User { get; set; } + public bool IsValid { get; set; } + } +} \ No newline at end of file diff --git a/src/BirdsiteLive.Domain/UserService.cs b/src/BirdsiteLive.Domain/UserService.cs index a080180..6f88543 100644 --- a/src/BirdsiteLive.Domain/UserService.cs +++ b/src/BirdsiteLive.Domain/UserService.cs @@ -11,6 +11,7 @@ using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; using BirdsiteLive.Cryptography; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain.BusinessUseCases; using BirdsiteLive.Domain.Repository; using BirdsiteLive.Domain.Statistics; @@ -24,7 +25,7 @@ namespace BirdsiteLive.Domain { public interface IUserService { - Actor GetUser(TwitterUser twitterUser); + Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser); Task FollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityFollow activity, string body); Task UndoFollowRequestedAsync(string signature, string method, string path, string queryString, Dictionary requestHeaders, ActivityUndoFollow activity, string body); @@ -64,7 +65,7 @@ public UserService(InstanceSettings instanceSettings, ICryptoService cryptoServi } #endregion - public Actor GetUser(TwitterUser twitterUser) + public Actor GetUser(TwitterUser twitterUser, SyncTwitterUser dbTwitterUser) { var actorUrl = UrlFactory.GetActorUrl(_instanceSettings.Domain, twitterUser.Acct); var acct = twitterUser.Acct.ToLowerInvariant(); @@ -87,9 +88,10 @@ public Actor GetUser(TwitterUser twitterUser) preferredUsername = acct, name = twitterUser.Name, inbox = $"{actorUrl}/inbox", - summary = description, + summary = "[UNOFFICIAL MIRROR: This is a view of Twitter using ActivityPub]

" + description, url = actorUrl, manuallyApprovesFollowers = twitterUser.Protected, + discoverable = false, publicKey = new PublicKey() { id = $"{actorUrl}#main-key", @@ -111,14 +113,27 @@ public Actor GetUser(TwitterUser twitterUser) new UserAttachment { type = "PropertyValue", - name = "Official", + name = "Official Account", value = $"https://twitter.com/{acct}" + }, + new UserAttachment + { + type = "PropertyValue", + name = "Disclaimer", + value = "This is an automatically created and managed mirror profile from Twitter. While it reflects exactly the content of the original account, it doesn't provide support for interactions and replies. It is an equivalent view from other 3rd party Twitter client apps and uses the same technical means to provide it." + }, + new UserAttachment + { + type = "PropertyValue", + name = "Take control of this account", + value = $"MANAGE" } }, endpoints = new EndPoints { sharedInbox = $"https://{_instanceSettings.Domain}/inbox" - } + }, + movedTo = dbTwitterUser?.MovedTo }; return user; } diff --git a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs index 91e3931..2f4d50e 100644 --- a/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs +++ b/src/BirdsiteLive.Moderation/Processors/TwitterAccountModerationProcessor.cs @@ -29,7 +29,7 @@ public async Task ProcessAsync(ModerationTypeEnum type) { if (type == ModerationTypeEnum.None) return; - var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(); + var twitterUsers = await _twitterUserDal.GetAllTwitterUsersAsync(false); foreach (var user in twitterUsers) { diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs index 58d35d0..321fbf0 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTweetsProcessor.cs @@ -49,12 +49,12 @@ public async Task ProcessAsync(UserWithDataToSync[] syncTw { var tweetId = tweets.Last().Id; var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, tweetId, tweetId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } else { var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, now, user.MovedTo, user.MovedToAcct, user.Deleted); } } diff --git a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs index 973b672..d9d0ffb 100644 --- a/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/RetrieveTwitterUsersProcessor.cs @@ -39,7 +39,7 @@ public async Task GetTwitterUsersAsync(BufferBlock twitterUse try { var maxUsersNumber = await _maxUsersNumberProvider.GetMaxUsersNumberAsync(); - var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber); + var users = await _twitterUserDal.GetAllTwitterUsersAsync(maxUsersNumber, false); var userCount = users.Any() ? users.Length : 1; var splitNumber = (int) Math.Ceiling(userCount / 15d); diff --git a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs index 1437255..1f94871 100644 --- a/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs +++ b/src/BirdsiteLive.Pipeline/Processors/SaveProgressionProcessor.cs @@ -3,6 +3,8 @@ using System.Threading; using System.Threading.Tasks; using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Contracts; using BirdsiteLive.Pipeline.Models; using Microsoft.Extensions.Logging; @@ -13,12 +15,14 @@ public class SaveProgressionProcessor : ISaveProgressionProcessor { private readonly ITwitterUserDal _twitterUserDal; private readonly ILogger _logger; + private readonly IRemoveTwitterAccountAction _removeTwitterAccountAction; #region Ctor - public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger logger) + public SaveProgressionProcessor(ITwitterUserDal twitterUserDal, ILogger logger, IRemoveTwitterAccountAction removeTwitterAccountAction) { _twitterUserDal = twitterUserDal; _logger = logger; + _removeTwitterAccountAction = removeTwitterAccountAction; } #endregion @@ -28,28 +32,23 @@ public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, Cancella { if (userWithTweetsToSync.Tweets.Length == 0) { - _logger.LogWarning("No tweets synchronized"); + _logger.LogInformation("No tweets synchronized"); + await UpdateUserSyncDateAsync(userWithTweetsToSync.User); return; } if(userWithTweetsToSync.Followers.Length == 0) { - _logger.LogWarning("No Followers found for {User}", userWithTweetsToSync.User.Acct); + _logger.LogInformation("No Followers found for {User}", userWithTweetsToSync.User.Acct); + await _removeTwitterAccountAction.ProcessAsync(userWithTweetsToSync.User); return; } var userId = userWithTweetsToSync.User.Id; var followingSyncStatuses = userWithTweetsToSync.Followers.Select(x => x.FollowingsSyncStatus[userId]).ToList(); - - if (followingSyncStatuses.Count == 0) - { - _logger.LogWarning("No Followers sync found for {User}, Id: {UserId}", userWithTweetsToSync.User.Acct, userId); - return; - } - var lastPostedTweet = userWithTweetsToSync.Tweets.Select(x => x.Id).Max(); var minimumSync = followingSyncStatuses.Min(); var now = DateTime.UtcNow; - await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now); + await _twitterUserDal.UpdateTwitterUserAsync(userId, lastPostedTweet, minimumSync, userWithTweetsToSync.User.FetchingErrorCount, now, userWithTweetsToSync.User.MovedTo, userWithTweetsToSync.User.MovedToAcct, userWithTweetsToSync.User.Deleted); } catch (Exception e) { @@ -57,5 +56,11 @@ public async Task ProcessAsync(UserWithDataToSync userWithTweetsToSync, Cancella throw; } } + + private async Task UpdateUserSyncDateAsync(SyncTwitterUser user) + { + user.LastSync = DateTime.UtcNow; + await _twitterUserDal.UpdateTwitterUserAsync(user); + } } } \ No newline at end of file diff --git a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs index 75ae645..e8e47fb 100644 --- a/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs +++ b/src/BirdsiteLive.Twitter/Extractors/TweetExtractor.cs @@ -28,7 +28,8 @@ public ExtractedTweet Extract(ITweet tweet) IsReply = tweet.InReplyToUserId != null, IsThread = tweet.InReplyToUserId != null && tweet.InReplyToUserId == tweet.CreatedBy.Id, IsRetweet = tweet.IsRetweet || tweet.QuotedStatusId != null, - RetweetUrl = ExtractRetweetUrl(tweet) + RetweetUrl = ExtractRetweetUrl(tweet), + CreatorName = tweet.CreatedBy.UserIdentifier.ScreenName }; return extractedTweet; diff --git a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs index f7f4e59..89a2e23 100644 --- a/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs +++ b/src/BirdsiteLive.Twitter/Models/ExtractedTweet.cs @@ -15,5 +15,6 @@ public class ExtractedTweet public bool IsThread { get; set; } public bool IsRetweet { get; set; } public string RetweetUrl { get; set; } + public string CreatorName { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/BirdsiteLive.csproj b/src/BirdsiteLive/BirdsiteLive.csproj index d4e7466..a81920d 100644 --- a/src/BirdsiteLive/BirdsiteLive.csproj +++ b/src/BirdsiteLive/BirdsiteLive.csproj @@ -4,7 +4,7 @@ netcoreapp3.1 d21486de-a812-47eb-a419-05682bb68856 Linux - 0.20.0 + 0.21.0 diff --git a/src/BirdsiteLive/Controllers/DebugingController.cs b/src/BirdsiteLive/Controllers/DebugingController.cs index 00accef..8f37f22 100644 --- a/src/BirdsiteLive/Controllers/DebugingController.cs +++ b/src/BirdsiteLive/Controllers/DebugingController.cs @@ -59,17 +59,22 @@ public async Task Follow() [HttpPost] public async Task PostNote() { - var username = "gra"; + var username = "twitter"; var actor = $"https://{_instanceSettings.Domain}/users/{username}"; - var targetHost = "mastodon.technology"; - var target = $"{targetHost}/users/testtest"; - var inbox = $"/users/testtest/inbox"; + var targetHost = "ioc.exchange"; + var target = $"https://{targetHost}/users/test"; + //var inbox = $"/users/testtest/inbox"; + var inbox = $"/inbox"; var noteGuid = Guid.NewGuid(); var noteId = $"https://{_instanceSettings.Domain}/users/{username}/statuses/{noteGuid}"; var noteUrl = $"https://{_instanceSettings.Domain}/@{username}/{noteGuid}"; var to = $"{actor}/followers"; + to = target; + + var cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }; + cc = new string[0]; var now = DateTime.UtcNow; var nowString = now.ToString("s") + "Z"; @@ -82,7 +87,7 @@ public async Task PostNote() actor = actor, published = nowString, to = new[] { to }, - //cc = new [] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = cc, apObject = new Note() { id = noteId, @@ -94,7 +99,8 @@ public async Task PostNote() // Unlisted to = new[] { to }, - cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, + cc = cc, + //cc = new[] { "https://www.w3.org/ns/activitystreams#Public" }, //// Public //to = new[] { "https://www.w3.org/ns/activitystreams#Public" }, @@ -102,8 +108,16 @@ public async Task PostNote() sensitive = false, content = "

TEST PUBLIC

", + //content = "

@test test

", attachment = new Attachment[0], - tag = new Tag[0] + tag = new Tag[]{ + new Tag() + { + type = "Mention", + href = target, + name = "@test@ioc.exchange" + } + }, } }; @@ -125,6 +139,17 @@ public async Task PostRejectFollow() await _userService.SendRejectFollowAsync(activityFollow, "mastodon.technology"); return View("Index"); } + + [HttpPost] + public async Task PostDeleteUser() + { + var userName = "twitter"; + var host = "ioc.exchange"; + var inbox = "/inbox"; + + await _activityPubService.DeleteUserAsync(userName, host, inbox); + return View("Index"); + } } #endif diff --git a/src/BirdsiteLive/Controllers/MigrationController.cs b/src/BirdsiteLive/Controllers/MigrationController.cs new file mode 100644 index 0000000..386644a --- /dev/null +++ b/src/BirdsiteLive/Controllers/MigrationController.cs @@ -0,0 +1,237 @@ +using System; +using Microsoft.AspNetCore.Mvc; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Npgsql.TypeHandlers; +using BirdsiteLive.Domain; +using BirdsiteLive.Domain.Enum; +using BirdsiteLive.DAL.Contracts; + +namespace BirdsiteLive.Controllers +{ + public class MigrationController : Controller + { + private readonly MigrationService _migrationService; + private readonly ITwitterUserDal _twitterUserDal; + + #region Ctor + public MigrationController(MigrationService migrationService, ITwitterUserDal twitterUserDal) + { + _migrationService = migrationService; + _twitterUserDal = twitterUserDal; + } + #endregion + + [HttpGet] + [Route("/migration/move/{id}")] + public IActionResult IndexMove(string id) + { + var migrationCode = _migrationService.GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View("Index", data); + } + + [HttpGet] + [Route("/migration/delete/{id}")] + public IActionResult IndexDelete(string id) + { + var migrationCode = _migrationService.GetDeletionCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode + }; + + return View("Delete", data); + } + + [HttpPost] + [Route("/migration/move/{id}")] + public async Task MigrateMove(string id, string tweetid, string handle) + { + var migrationCode = _migrationService.GetMigrationCode(id); + var data = new MigrationData() + { + Acct = id, + MigrationCode = migrationCode, + + IsAcctProvided = !string.IsNullOrWhiteSpace(handle), + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid, + FediverseAccount = handle + }; + ValidatedFediverseUser fediverseUserValidation = null; + + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be migrated"; + return View("Index", data); + } + if (twitterAccount != null && + (!string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct))) + { + data.ErrorMessage = "This account has been moved already, it can't be migrated again"; + return View("Index", data); + } + + // Start migration + try + { + fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + + data.IsAcctValid = fediverseUserValidation.IsValid; + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + if (data.IsAcctValid && data.IsTweetValid && fediverseUserValidation != null) + { + try + { + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + await _migrationService.TriggerRemoteMigrationAsync(id, tweetid, handle); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + + return View("Index", data); + } + + [HttpPost] + [Route("/migration/delete/{id}")] + public async Task MigrateDelete(string id, string tweetid) + { + var deletionCode = _migrationService.GetDeletionCode(id); + + var data = new MigrationData() + { + Acct = id, + MigrationCode = deletionCode, + + IsTweetProvided = !string.IsNullOrWhiteSpace(tweetid), + + TweetId = tweetid + }; + + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount != null && twitterAccount.Deleted) + { + data.ErrorMessage = "This account has been deleted, it can't be deleted again"; + return View("Delete", data); + } + + // Start deletion + try + { + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + data.IsTweetValid = isTweetValid; + } + catch (Exception e) + { + data.ErrorMessage = e.Message; + } + + if (data.IsTweetValid) + { + try + { + await _migrationService.DeleteAccountAsync(id); + await _migrationService.TriggerRemoteDeleteAsync(id, tweetid); + data.MigrationSuccess = true; + } + catch (Exception e) + { + Console.WriteLine(e); + data.ErrorMessage = e.Message; + } + } + + return View("Delete", data); + } + + [HttpPost] + [Route("/migration/move/{id}/{tweetid}/{handle}")] + public async Task RemoteMigrateMove(string id, string tweetid, string handle) + { + //Verify can be migrated + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted + || !string.IsNullOrWhiteSpace(twitterAccount.MovedTo) + || !string.IsNullOrWhiteSpace(twitterAccount.MovedToAcct)) + return Ok(); + + // Start migration + var fediverseUserValidation = await _migrationService.ValidateFediverseAcctAsync(handle); + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Migration); + + if (fediverseUserValidation.IsValid && isTweetValid) + { + await _migrationService.MigrateAccountAsync(fediverseUserValidation, id); + return Ok(); + } + + return StatusCode(400); + } + + [HttpPost] + [Route("/migration/delete/{id}/{tweetid}")] + public async Task RemoteMigrateDelete(string id, string tweetid) + { + //Verify can be deleted + var twitterAccount = await _twitterUserDal.GetTwitterUserAsync(id); + if (twitterAccount.Deleted) return Ok(); + + // Start deletion + var isTweetValid = _migrationService.ValidateTweet(id, tweetid, MigrationTypeEnum.Deletion); + + if (isTweetValid) + { + await _migrationService.DeleteAccountAsync(id); + return Ok(); + } + + return StatusCode(400); + } + } + + + + public class MigrationData + { + public string Acct { get; set; } + + public string FediverseAccount { get; set; } + public string TweetId { get; set; } + + public string MigrationCode { get; set; } + + public bool IsTweetProvided { get; set; } + public bool IsAcctProvided { get; set; } + + public bool IsTweetValid { get; set; } + public bool IsAcctValid { get; set; } + + public string ErrorMessage { get; set; } + public bool MigrationSuccess { get; set; } + } +} diff --git a/src/BirdsiteLive/Controllers/UsersController.cs b/src/BirdsiteLive/Controllers/UsersController.cs index 24a9eb5..e66dac5 100644 --- a/src/BirdsiteLive/Controllers/UsersController.cs +++ b/src/BirdsiteLive/Controllers/UsersController.cs @@ -11,6 +11,8 @@ using BirdsiteLive.ActivityPub.Models; using BirdsiteLive.Common.Regexes; using BirdsiteLive.Common.Settings; +using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Models; using BirdsiteLive.Domain; using BirdsiteLive.Models; using BirdsiteLive.Tools; @@ -28,13 +30,14 @@ public class UsersController : Controller { private readonly ITwitterUserService _twitterUserService; private readonly ITwitterTweetsService _twitterTweetService; + private readonly ITwitterUserDal _twitterUserDal; private readonly IUserService _userService; private readonly IStatusService _statusService; private readonly InstanceSettings _instanceSettings; private readonly ILogger _logger; #region Ctor - public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger) + public UsersController(ITwitterUserService twitterUserService, IUserService userService, IStatusService statusService, InstanceSettings instanceSettings, ITwitterTweetsService twitterTweetService, ILogger logger, ITwitterUserDal twitterUserDal) { _twitterUserService = twitterUserService; _userService = userService; @@ -42,6 +45,7 @@ public UsersController(ITwitterUserService twitterUserService, IUserService user _instanceSettings = instanceSettings; _twitterTweetService = twitterTweetService; _logger = logger; + _twitterUserDal = twitterUserDal; } #endregion @@ -56,11 +60,10 @@ public IActionResult Index() } return View("UserNotFound"); } - + [Route("/@{id}")] [Route("/users/{id}")] - [Route("/users/{id}/remote_follow")] - public IActionResult Index(string id) + public async Task Index(string id) { _logger.LogTrace("User Index: {Id}", id); @@ -102,6 +105,7 @@ public IActionResult Index(string id) } //var isSaturated = _twitterUserService.IsUserApiRateLimited(); + var dbUser = await _twitterUserDal.GetTwitterUserAsync(id); var acceptHeaders = Request.Headers["Accept"]; if (acceptHeaders.Any()) @@ -111,7 +115,8 @@ public IActionResult Index(string id) { if (isSaturated) return new ObjectResult("Too Many Requests") { StatusCode = 429 }; if (notFound) return NotFound(); - var apUser = _userService.GetUser(user); + if (dbUser != null && dbUser.Deleted) return new ObjectResult("Gone") { StatusCode = 410 }; + var apUser = _userService.GetUser(user, dbUser); var jsonApUser = JsonConvert.SerializeObject(apUser); return Content(jsonApUser, "application/activity+json; charset=utf-8"); } @@ -128,11 +133,21 @@ public IActionResult Index(string id) Url = user.Url, ProfileImageUrl = user.ProfileImageUrl, Protected = user.Protected, + + InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}", - InstanceHandle = $"@{user.Acct.ToLowerInvariant()}@{_instanceSettings.Domain}" + MovedTo = dbUser?.MovedTo, + MovedToAcct = dbUser?.MovedToAcct, + Deleted = dbUser?.Deleted ?? false, }; return View(displayableUser); } + + [Route("/users/{id}/remote_follow")] + public async Task IndexRemoteFollow(string id) + { + return Redirect($"/users/{id}"); + } [Route("/@{id}/{statusId}")] [Route("/users/{id}/statuses/{statusId}")] diff --git a/src/BirdsiteLive/Models/DisplayTwitterUser.cs b/src/BirdsiteLive/Models/DisplayTwitterUser.cs index 3a93875..0b17174 100644 --- a/src/BirdsiteLive/Models/DisplayTwitterUser.cs +++ b/src/BirdsiteLive/Models/DisplayTwitterUser.cs @@ -10,5 +10,9 @@ public class DisplayTwitterUser public bool Protected { get; set; } public string InstanceHandle { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } + public bool Deleted { get; set; } } } \ No newline at end of file diff --git a/src/BirdsiteLive/Views/Debuging/Index.cshtml b/src/BirdsiteLive/Views/Debuging/Index.cshtml index 5bcde75..e343cf2 100644 --- a/src/BirdsiteLive/Views/Debuging/Index.cshtml +++ b/src/BirdsiteLive/Views/Debuging/Index.cshtml @@ -23,4 +23,10 @@ + + +
+ + +
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Migration/Delete.cshtml b/src/BirdsiteLive/Views/Migration/Delete.cshtml new file mode 100644 index 0000000..9df9f62 --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Delete.cshtml @@ -0,0 +1,51 @@ +@model BirdsiteLive.Controllers.MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } + + @if (ViewData.Model.MigrationSuccess) + { + + } + +

Delete @@@ViewData.Model.Acct mirror

+ + @if (!ViewData.Model.IsTweetProvided) + { +

What is needed?

+ +

You'll need access to the Twitter account to provide proof of ownership.

+ +

What will deletion do?

+ +

+ Deletion will remove all followers, delete the account and will be blacklisted so that it can't be recreated.
+

+ } + +

Start the deletion!

+ +

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide deletion information:

+
+
+ + +
+ +
+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Migration/Index.cshtml b/src/BirdsiteLive/Views/Migration/Index.cshtml new file mode 100644 index 0000000..670a209 --- /dev/null +++ b/src/BirdsiteLive/Views/Migration/Index.cshtml @@ -0,0 +1,66 @@ +@model BirdsiteLive.Controllers.MigrationData +@{ + ViewData["Title"] = "Migration"; +} + +
+ @if (!string.IsNullOrWhiteSpace(ViewData.Model.ErrorMessage)) + { + + } + + @if (ViewData.Model.MigrationSuccess) + { + + } + +

Migrate @@@ViewData.Model.Acct mirror to my Fediverse account

+ + @if (!ViewData.Model.IsAcctProvided && !ViewData.Model.IsTweetProvided) + { +

What is needed?

+ +

You'll need a Fediverse account and access to the Twitter account to provide proof of ownership.

+ +

What will migration do?

+ +

+ Migration will notify followers of the migration of the mirror account to your fediverse account and will be disabled after that.
+

+ } + +

Start the migration!

+ +

Please copy and post this string in a public Tweet (the string must be untampered, but you can write anything you want before or after it):

+ + +
+ +

Provide migration information:

+
+ @*
+ + + We'll never share your email with anyone else. +
*@ +
+ + +
+
+ + +
+ +
+
+
+
+
+ I don't have a fediverse account and I'd like to delete this mirror. +
+
\ No newline at end of file diff --git a/src/BirdsiteLive/Views/Users/Index.cshtml b/src/BirdsiteLive/Views/Users/Index.cshtml index 945964a..2ff2ce3 100644 --- a/src/BirdsiteLive/Views/Users/Index.cshtml +++ b/src/BirdsiteLive/Views/Users/Index.cshtml @@ -37,6 +37,19 @@ This account is protected, BirdsiteLIVE cannot fetch their tweets and will not provide follow support until it is unprotected again. } + else if (ViewData.Model.Deleted) + { + + } + else if (!string.IsNullOrEmpty(ViewData.Model.MovedTo)) + { + + } else {
@@ -45,4 +58,8 @@
} + +
+ I'm the owner of this account and I would like to take control of this mirror. +
\ No newline at end of file diff --git a/src/BirdsiteLive/wwwroot/css/birdsite.css b/src/BirdsiteLive/wwwroot/css/birdsite.css index 5b6023c..159a50a 100644 --- a/src/BirdsiteLive/wwwroot/css/birdsite.css +++ b/src/BirdsiteLive/wwwroot/css/birdsite.css @@ -71,3 +71,18 @@ margin-left: 60px; /*font-weight: bold;*/ } + +.user-owner { + font-size: .8em; + padding-top: 20px; +} + +/** Migration **/ + +.migration__title { + font-size: 1.8em; +} + +.migration__subtitle { + font-size: 1.4em; +} \ No newline at end of file diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs index 2f9cb54..55b38f6 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/DbInitializerPostgresDal.cs @@ -23,7 +23,7 @@ internal class DbVersion public class DbInitializerPostgresDal : PostgresBase, IDbInitializerDal { private readonly PostgresTools _tools; - private readonly Version _currentVersion = new Version(2, 4); + private readonly Version _currentVersion = new Version(2, 5); private const string DbVersionType = "db-version"; #region Ctor @@ -135,7 +135,8 @@ public Tuple[] GetMigrationPatterns() new Tuple(new Version(2,0), new Version(2,1)), new Tuple(new Version(2,1), new Version(2,2)), new Tuple(new Version(2,2), new Version(2,3)), - new Tuple(new Version(2,3), new Version(2,4)) + new Tuple(new Version(2,3), new Version(2,4)), + new Tuple(new Version(2,4), new Version(2,5)) }; } @@ -172,6 +173,17 @@ public async Task MigrateDbAsync(Version from, Version to) var alterPostingError = $@"ALTER TABLE {_settings.FollowersTableName} ALTER COLUMN postingErrorCount TYPE INTEGER"; await _tools.ExecuteRequestAsync(alterPostingError); } + else if (from == new Version(2, 4) && to == new Version(2, 5)) + { + var addMovedTo = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedTo VARCHAR(2048)"; + await _tools.ExecuteRequestAsync(addMovedTo); + + var addMovedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD movedToAcct VARCHAR(305)"; + await _tools.ExecuteRequestAsync(addMovedToAcct); + + var addDeletedToAcct = $@"ALTER TABLE {_settings.TwitterUserTableName} ADD deleted BOOLEAN"; + await _tools.ExecuteRequestAsync(addDeletedToAcct); + } else { throw new NotImplementedException(); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs index 11214d4..d542a76 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL.Postgres/DataAccessLayers/TwitterUserPostgresDal.cs @@ -18,7 +18,7 @@ public TwitterUserPostgresDal(PostgresSettings settings) : base(settings) } #endregion - public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId) + public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, string movedToAcct = null) { acct = acct.ToLowerInvariant(); @@ -27,8 +27,15 @@ public async Task CreateTwitterUserAsync(string acct, long lastTweetPostedId) dbConnection.Open(); await dbConnection.ExecuteAsync( - $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId)", - new { acct, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = lastTweetPostedId }); + $"INSERT INTO {_settings.TwitterUserTableName} (acct,lastTweetPostedId,lastTweetSynchronizedForAllFollowersId, movedTo, movedToAcct) VALUES(@acct,@lastTweetPostedId,@lastTweetSynchronizedForAllFollowersId,@movedTo,@movedToAcct)", + new + { + acct, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId = lastTweetPostedId, + movedTo, + movedToAcct + }); } } @@ -62,7 +69,7 @@ public async Task GetTwitterUserAsync(int id) public async Task GetTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -75,7 +82,7 @@ public async Task GetTwitterUsersCountAsync() public async Task GetFailingTwitterUsersCountAsync() { - var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0"; + var query = $"SELECT COUNT(*) FROM {_settings.TwitterUserTableName} WHERE fetchingErrorCount > 0 AND (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; using (var dbConnection = Connection) { @@ -86,9 +93,10 @@ public async Task GetFailingTwitterUsersCountAsync() } } - public async Task GetAllTwitterUsersAsync(int maxNumber) + public async Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; + if (retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName} ORDER BY lastSync ASC NULLS FIRST LIMIT @maxNumber"; using (var dbConnection = Connection) { @@ -99,9 +107,10 @@ public async Task GetAllTwitterUsersAsync(int maxNumber) } } - public async Task GetAllTwitterUsersAsync() + public async Task GetAllTwitterUsersAsync(bool retrieveDisabledUser) { - var query = $"SELECT * FROM {_settings.TwitterUserTableName}"; + var query = $"SELECT * FROM {_settings.TwitterUserTableName} WHERE (movedTo = '') IS NOT FALSE AND deleted IS NOT TRUE"; + if(retrieveDisabledUser) query = $"SELECT * FROM {_settings.TwitterUserTableName}"; using (var dbConnection = Connection) { @@ -112,26 +121,36 @@ public async Task GetAllTwitterUsersAsync() } } - public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync) + public async Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted) { if(id == default) throw new ArgumentException("id"); if(lastTweetPostedId == default) throw new ArgumentException("lastTweetPostedId"); if(lastTweetSynchronizedForAllFollowersId == default) throw new ArgumentException("lastTweetSynchronizedForAllFollowersId"); if(lastSync == default) throw new ArgumentException("lastSync"); - var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync WHERE id = @id"; + var query = $"UPDATE {_settings.TwitterUserTableName} SET lastTweetPostedId = @lastTweetPostedId, lastTweetSynchronizedForAllFollowersId = @lastTweetSynchronizedForAllFollowersId, fetchingErrorCount = @fetchingErrorCount, lastSync = @lastSync, movedTo = @movedTo, movedToAcct = @movedToAcct, deleted = @deleted WHERE id = @id"; using (var dbConnection = Connection) { dbConnection.Open(); - await dbConnection.QueryAsync(query, new { id, lastTweetPostedId, lastTweetSynchronizedForAllFollowersId, fetchingErrorCount, lastSync = lastSync.ToUniversalTime() }); + await dbConnection.QueryAsync(query, new + { + id, + lastTweetPostedId, + lastTweetSynchronizedForAllFollowersId, + fetchingErrorCount, + lastSync = lastSync.ToUniversalTime(), + movedTo, + movedToAcct, + deleted + }); } } public async Task UpdateTwitterUserAsync(SyncTwitterUser user) { - await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync); + await UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, user.FetchingErrorCount, user.LastSync, user.MovedTo, user.MovedToAcct, user.Deleted); } public async Task DeleteTwitterUserAsync(string acct) diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs index ef2cc36..0c58881 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Contracts/ITwitterUserDal.cs @@ -6,12 +6,13 @@ namespace BirdsiteLive.DAL.Contracts { public interface ITwitterUserDal { - Task CreateTwitterUserAsync(string acct, long lastTweetPostedId); + Task CreateTwitterUserAsync(string acct, long lastTweetPostedId, string movedTo = null, + string movedToAcct = null); Task GetTwitterUserAsync(string acct); Task GetTwitterUserAsync(int id); - Task GetAllTwitterUsersAsync(int maxNumber); - Task GetAllTwitterUsersAsync(); - Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync); + Task GetAllTwitterUsersAsync(int maxNumber, bool retrieveDisabledUser); + Task GetAllTwitterUsersAsync(bool retrieveDisabledUser); + Task UpdateTwitterUserAsync(int id, long lastTweetPostedId, long lastTweetSynchronizedForAllFollowersId, int fetchingErrorCount, DateTime lastSync, string movedTo, string movedToAcct, bool deleted); Task UpdateTwitterUserAsync(SyncTwitterUser user); Task DeleteTwitterUserAsync(string acct); Task DeleteTwitterUserAsync(int id); diff --git a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs index 8b18ba1..5089afc 100644 --- a/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs +++ b/src/DataAccessLayers/BirdsiteLive.DAL/Models/SyncTwitterUser.cs @@ -12,6 +12,11 @@ public class SyncTwitterUser public DateTime LastSync { get; set; } - public int FetchingErrorCount { get; set; } //TODO: update DAL + public int FetchingErrorCount { get; set; } + + public string MovedTo { get; set; } + public string MovedToAcct { get; set; } + + public bool Deleted { get; set; } } } \ No newline at end of file diff --git a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs index c9bc746..936bb73 100644 --- a/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs +++ b/src/Tests/BirdsiteLive.DAL.Postgres.Tests/DataAccessLayers/TwitterUserPostgresDalTests.cs @@ -71,6 +71,28 @@ public async Task CreateAndGetUser_byId() Assert.AreEqual(result.Id, resultById.Id); } + [TestMethod] + public async Task CreateAndGetMigratedUser_byId() + { + var acct = "myid"; + var lastTweetId = 1548L; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId, movedTo, movedToAcct); + var result = await dal.GetTwitterUserAsync(acct); + var resultById = await dal.GetTwitterUserAsync(result.Id); + + Assert.AreEqual(acct, resultById.Acct); + Assert.AreEqual(lastTweetId, resultById.LastTweetPostedId); + Assert.AreEqual(lastTweetId, resultById.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(result.Id, resultById.Id); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + [TestMethod] public async Task CreateUpdateAndGetUser() { @@ -87,7 +109,66 @@ public async Task CreateUpdateAndGetUser() var updatedLastSyncId = 1550L; var now = DateTime.Now; var errors = 15; - await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now); + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, false); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); + } + + [TestMethod] + public async Task CreateUpdateAndGetMigratedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + var movedTo = "https://"; + var movedToAcct = "@account@instance"; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, movedTo, movedToAcct, false); + + result = await dal.GetTwitterUserAsync(acct); + + Assert.AreEqual(acct, result.Acct); + Assert.AreEqual(updatedLastTweetId, result.LastTweetPostedId); + Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); + Assert.AreEqual(errors, result.FetchingErrorCount); + Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(movedTo, result.MovedTo); + Assert.AreEqual(movedToAcct, result.MovedToAcct); + } + + [TestMethod] + public async Task CreateUpdateAndGetDeletedUser() + { + var acct = "myid"; + var lastTweetId = 1548L; + + var dal = new TwitterUserPostgresDal(_settings); + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var result = await dal.GetTwitterUserAsync(acct); + + var updatedLastTweetId = 1600L; + var updatedLastSyncId = 1550L; + var now = DateTime.Now; + var errors = 15; + await dal.UpdateTwitterUserAsync(result.Id, updatedLastTweetId, updatedLastSyncId, errors, now, null, null, true); result = await dal.GetTwitterUserAsync(acct); @@ -96,6 +177,9 @@ public async Task CreateUpdateAndGetUser() Assert.AreEqual(updatedLastSyncId, result.LastTweetSynchronizedForAllFollowersId); Assert.AreEqual(errors, result.FetchingErrorCount); Assert.IsTrue(Math.Abs((now.ToUniversalTime() - result.LastSync).Milliseconds) < 100); + Assert.AreEqual(null, result.MovedTo); + Assert.AreEqual(null, result.MovedToAcct); + Assert.AreEqual(true, result.Deleted); } [TestMethod] @@ -167,7 +251,7 @@ public async Task CreateUpdate3AndGetUser() public async Task Update_NoId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(default, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -175,7 +259,7 @@ public async Task Update_NoId() public async Task Update_NoLastTweetPostedId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, default, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -183,7 +267,7 @@ public async Task Update_NoLastTweetPostedId() public async Task Update_NoLastTweetSynchronizedForAllFollowersId() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow); + await dal.UpdateTwitterUserAsync(12, 9556, default, default, DateTime.UtcNow, null, null, false); } [TestMethod] @@ -191,7 +275,7 @@ public async Task Update_NoLastTweetSynchronizedForAllFollowersId() public async Task Update_NoLastSync() { var dal = new TwitterUserPostgresDal(_settings); - await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default); + await dal.UpdateTwitterUserAsync(12, 9556, 65, default, default, null, null, false); } [TestMethod] @@ -256,12 +340,79 @@ public async Task GetAllTwitterUsers_Top() await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(1000); + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); Assert.IsFalse(result[0].LastTweetPostedId == default); Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); + + foreach (var user in result) + { + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedTo)); + Assert.IsTrue(string.IsNullOrWhiteSpace(user.MovedToAcct)); + Assert.IsFalse(user.Deleted); + } + } + + [TestMethod] + public async Task GetAllTwitterUsers_Top_RetrieveDeleted() + { + var dal = new TwitterUserPostgresDal(_settings); + for (var i = 0; i < 1000; i++) + { + var acct = $"myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + for (int i = 0; i < 10; i++) + { + var acct = $"deleted-myid{i}"; + var lastTweetId = 148L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId); + var user = await dal.GetTwitterUserAsync(acct); + user.Deleted = true; + user.LastSync = DateTime.UtcNow; + await dal.UpdateTwitterUserAsync(user); + } + + var result = await dal.GetAllTwitterUsersAsync(1100, true); + Assert.AreEqual(1020, result.Length); + Assert.IsFalse(result[0].Id == default); + Assert.IsFalse(result[0].Acct == default); + Assert.IsFalse(result[0].LastTweetPostedId == default); + Assert.IsFalse(result[0].LastTweetSynchronizedForAllFollowersId == default); } [TestMethod] @@ -279,7 +430,7 @@ public async Task GetAllTwitterUsers_Top_NotInit() // Update accounts var now = DateTime.UtcNow; - var allUsers = await dal.GetAllTwitterUsersAsync(); + var allUsers = await dal.GetAllTwitterUsersAsync(false); foreach (var acc in allUsers) { var lastSync = now.AddDays(acc.LastTweetPostedId); @@ -290,7 +441,7 @@ public async Task GetAllTwitterUsers_Top_NotInit() // Create a not init account await dal.CreateTwitterUserAsync("not_init", -1); - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.IsTrue(result.Any(x => x.Acct == "myid0")); Assert.IsTrue(result.Any(x => x.Acct == "myid8")); @@ -313,15 +464,15 @@ public async Task GetAllTwitterUsers_Limited() await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var allUsers = await dal.GetAllTwitterUsersAsync(100); + var allUsers = await dal.GetAllTwitterUsersAsync(100, false); for (var i = 0; i < 20; i++) { var user = allUsers[i]; var date = i % 2 == 0 ? oldest : newest; - await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date); + await dal.UpdateTwitterUserAsync(user.Id, user.LastTweetPostedId, user.LastTweetSynchronizedForAllFollowersId, 0, date, null, null, false); } - var result = await dal.GetAllTwitterUsersAsync(10); + var result = await dal.GetAllTwitterUsersAsync(10, false); Assert.AreEqual(10, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -344,7 +495,15 @@ public async Task GetAllTwitterUsers() await dal.CreateTwitterUserAsync(acct, lastTweetId); } - var result = await dal.GetAllTwitterUsersAsync(); + for (int i = 0; i < 10; i++) + { + var acct = $"migrated-myid{i}"; + var lastTweetId = 1548L; + + await dal.CreateTwitterUserAsync(acct, lastTweetId, "https://url/account", "@user@domain"); + } + + var result = await dal.GetAllTwitterUsersAsync(false); Assert.AreEqual(1000, result.Length); Assert.IsFalse(result[0].Id == default); Assert.IsFalse(result[0].Acct == default); @@ -382,7 +541,7 @@ public async Task CountFailingTwitterUsers() if (i == 0 || i == 2 || i == 3) { var t = await dal.GetTwitterUserAsync(acct); - await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now); + await dal.UpdateTwitterUserAsync(t.Id ,1L,2L, 50+i*2, DateTime.Now, null, null, false); } } diff --git a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs index 0fb03ae..8f2f393 100644 --- a/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs +++ b/src/Tests/BirdsiteLive.Domain.Tests/BusinessUseCases/ProcessFollowUserTests.cs @@ -77,7 +77,9 @@ public async Task ExecuteAsync_UserDontExists_TwitterDontExists_Test() twitterUserDalMock .Setup(x => x.CreateTwitterUserAsync( It.Is(y => y == twitterName), - It.Is(y => y == -1))) + It.Is(y => y == -1), + It.Is(y => y == null), + It.Is(y => y == null))) .Returns(Task.CompletedTask); #endregion diff --git a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs index 21d1288..8473424 100644 --- a/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Moderation.Tests/Processors/TwitterAccountModerationProcessorTests.cs @@ -48,7 +48,7 @@ public async Task ProcessAsync_WhiteListing_WhiteListed() #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -87,7 +87,7 @@ public async Task ProcessAsync_WhiteListing_NotWhiteListed() #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -130,7 +130,7 @@ public async Task ProcessAsync_BlackListing_BlackListed() #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); @@ -173,7 +173,7 @@ public async Task ProcessAsync_BlackListing_NotBlackListed() #region Mocks var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock - .Setup(x => x.GetAllTwitterUsersAsync()) + .Setup(x => x.GetAllTwitterUsersAsync(It.Is(y => y == false))) .ReturnsAsync(allUsers.ToArray()); var moderationRepositoryMock = new Mock(MockBehavior.Strict); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs index 17a3aa2..f95ad82 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTweetsProcessorTests.cs @@ -64,7 +64,10 @@ public async Task ProcessAsync_UserNotSync_Test() It.Is(y => y == tweets.Last().Id), It.Is(y => y == tweets.Last().Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs index 4d0e465..daf0bfa 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/RetrieveTwitterUsersProcessorTests.cs @@ -40,7 +40,8 @@ public async Task GetTwitterUsersAsync_Test() var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users); var loggerMock = new Mock>(); @@ -83,7 +84,8 @@ public async Task GetTwitterUsersAsync_Multi_Test() var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -130,7 +132,8 @@ public async Task GetTwitterUsersAsync_Multi2_Test() var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .SetupSequence(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(users.ToArray()) .ReturnsAsync(new SyncTwitterUser[0]) .ReturnsAsync(new SyncTwitterUser[0]) @@ -178,7 +181,8 @@ public async Task GetTwitterUsersAsync_NoUsers_Test() var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .ReturnsAsync(new SyncTwitterUser[0]); var loggerMock = new Mock>(); @@ -215,7 +219,8 @@ public async Task GetTwitterUsersAsync_Exception_Test() var twitterUserDalMock = new Mock(MockBehavior.Strict); twitterUserDalMock .Setup(x => x.GetAllTwitterUsersAsync( - It.Is(y => y == maxUsers))) + It.Is(y => y == maxUsers), + It.Is(y => y == false))) .Returns(async () => await DelayFaultedTask(new Exception())); var loggerMock = new Mock>(); diff --git a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs index 4587071..d245713 100644 --- a/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs +++ b/src/Tests/BirdsiteLive.Pipeline.Tests/Processors/SaveProgressionProcessorTests.cs @@ -1,16 +1,16 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using BirdsiteLive.DAL.Contracts; +using BirdsiteLive.DAL.Contracts; using BirdsiteLive.DAL.Models; +using BirdsiteLive.Moderation.Actions; using BirdsiteLive.Pipeline.Models; using BirdsiteLive.Pipeline.Processors; using BirdsiteLive.Twitter.Models; -using Castle.DynamicProxy.Contributors; using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; namespace BirdsiteLive.Pipeline.Tests.Processors { @@ -66,17 +66,93 @@ public async Task ProcessAsync_Test() It.Is(y => y == tweet2.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public async Task ProcessAsync_Exception_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = new[] + { + tweet1, + tweet2 + }, + Followers = new[] + { + follower1 + }, + User = user + }; + + var loggerMock = new Mock>(); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y == user.Id), + It.Is(y => y == tweet2.Id), + It.Is(y => y == tweet2.Id), + It.Is(y => y == 0), + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) + )) + .Throws(new ArgumentException()); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } @@ -132,19 +208,25 @@ public async Task ProcessAsync_PartiallySynchronized_Test() It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) )) .Returns(Task.CompletedTask); var loggerMock = new Mock>(); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } @@ -208,20 +290,130 @@ public async Task ProcessAsync_PartiallySynchronized_MultiUsers_Test() It.Is(y => y == tweet3.Id), It.Is(y => y == tweet2.Id), It.Is(y => y == 0), - It.IsAny() + It.IsAny(), + It.Is(y => y == null), + It.Is(y => y == null), + It.Is(y => y == false) + )) + .Returns(Task.CompletedTask); + + var loggerMock = new Mock>(); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_NoTweets_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1, + LastTweetPostedId = 42, + LastSync = DateTime.UtcNow.AddDays(-3) + }; + var follower1 = new Follower + { + FollowingsSyncStatus = new Dictionary + { + {1, 37} + } + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = Array.Empty(), + Followers = new[] + { + follower1 + }, + User = user + }; + + var loggerMock = new Mock>(); + #endregion + + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + twitterUserDalMock + .Setup(x => x.UpdateTwitterUserAsync( + It.Is(y => y.LastTweetPostedId == 42 + && y.LastSync > DateTime.UtcNow.AddDays(-1)) )) .Returns(Task.CompletedTask); + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); + await processor.ProcessAsync(usersWithTweets, CancellationToken.None); + + #region Validations + twitterUserDalMock.VerifyAll(); + loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); + #endregion + } + + [TestMethod] + public async Task ProcessAsync_NoFollower_Test() + { + #region Stubs + var user = new SyncTwitterUser + { + Id = 1 + }; + var tweet1 = new ExtractedTweet + { + Id = 36 + }; + var tweet2 = new ExtractedTweet + { + Id = 37 + }; + + var usersWithTweets = new UserWithDataToSync + { + Tweets = new[] + { + tweet1, + tweet2 + }, + Followers = Array.Empty(), + User = user + }; + var loggerMock = new Mock>(); #endregion - var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object); + #region Mocks + var twitterUserDalMock = new Mock(MockBehavior.Strict); + + var removeTwitterAccountActionMock = new Mock(MockBehavior.Strict); + removeTwitterAccountActionMock + .Setup(x => x.ProcessAsync(It.Is(y => y.Id == user.Id))) + .Returns(Task.CompletedTask); + #endregion + + var processor = new SaveProgressionProcessor(twitterUserDalMock.Object, loggerMock.Object, removeTwitterAccountActionMock.Object); await processor.ProcessAsync(usersWithTweets, CancellationToken.None); #region Validations twitterUserDalMock.VerifyAll(); loggerMock.VerifyAll(); + removeTwitterAccountActionMock.VerifyAll(); #endregion } + } } \ No newline at end of file