Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show more comprehensive background processing progress notifications #25832

Merged
merged 7 commits into from
Dec 18, 2023
Merged

Show more comprehensive background processing progress notifications #25832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7462a9f
Add helper method to handle progress notifications for background jobs
peppy Dec 18, 2023
e7d1cf7
Add progress notifications for background tasks which don't already h…
peppy Dec 18, 2023
bfa90e9
Also populate `ObjectCount`s when running a full beatmap process
peppy Dec 18, 2023
5ab3815
Add logging of progress every so often
peppy Dec 18, 2023
e3251b4
Fix progress notifications queueing up infinite text changes when not…
peppy Dec 18, 2023
03ac2c3
Remove some excessive logging
peppy Dec 18, 2023
9aaaa12
Don't show progress notifications when there are too few items to be …
peppy Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 100 additions & 18 deletions osu.Game/BackgroundDataStoreProcessor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ protected override void LoadComplete()

checkForOutdatedStarRatings();
processBeatmapSetsWithMissingMetrics();
// Note that the previous method will also update these on a fresh run.
processBeatmapsWithMissingObjectCounts();
processScoresWithMissingStatistics();
convertLegacyTotalScoreToStandardised();
Expand Down Expand Up @@ -144,12 +145,24 @@ private void processBeatmapSetsWithMissingMetrics()
}
});

if (beatmapSetIds.Count == 0)
return;

Logger.Log($"Found {beatmapSetIds.Count} beatmap sets which require reprocessing.");

int i = 0;
// Technically this is doing more than just star ratings, but easier for the end user to understand.
var notification = showProgressNotification(beatmapSetIds.Count, "Reprocessing star rating for beatmaps", "beatmaps' star ratings have been updated");

int processedCount = 0;
int failedCount = 0;

foreach (var id in beatmapSetIds)
{
if (notification?.State == ProgressNotificationState.Cancelled)
break;

updateNotificationProgress(notification, processedCount, beatmapSetIds.Count);

sleepIfRequired();

realmAccess.Run(r =>
Expand All @@ -160,16 +173,19 @@ private void processBeatmapSetsWithMissingMetrics()
{
try
{
Logger.Log($"Background processing {set} ({++i} / {beatmapSetIds.Count})");
beatmapUpdater.Process(set);
++processedCount;
}
catch (Exception e)
{
Logger.Log($"Background processing failed on {set}: {e}");
++failedCount;
}
}
});
}

completeNotification(notification, processedCount, beatmapSetIds.Count, failedCount);
}

private void processBeatmapsWithMissingObjectCounts()
Expand All @@ -184,12 +200,23 @@ private void processBeatmapsWithMissingObjectCounts()
beatmapIds.Add(b.ID);
});

Logger.Log($"Found {beatmapIds.Count} beatmaps which require reprocessing.");
if (beatmapIds.Count == 0)
return;

Logger.Log($"Found {beatmapIds.Count} beatmaps which require statistics population.");

int i = 0;
var notification = showProgressNotification(beatmapIds.Count, "Populating missing statistics for beatmaps", "beatmaps have been populated with missing statistics");

int processedCount = 0;
int failedCount = 0;

foreach (var id in beatmapIds)
{
if (notification?.State == ProgressNotificationState.Cancelled)
break;

updateNotificationProgress(notification, processedCount, beatmapIds.Count);

sleepIfRequired();

realmAccess.Run(r =>
Expand All @@ -200,16 +227,19 @@ private void processBeatmapsWithMissingObjectCounts()
{
try
{
Logger.Log($"Background processing {beatmap} ({++i} / {beatmapIds.Count})");
beatmapUpdater.ProcessObjectCounts(beatmap);
++processedCount;
}
catch (Exception e)
{
Logger.Log($"Background processing failed on {beatmap}: {e}");
++failedCount;
}
}
});
}

completeNotification(notification, processedCount, beatmapIds.Count, failedCount);
}

private void processScoresWithMissingStatistics()
Expand All @@ -231,10 +261,23 @@ private void processScoresWithMissingStatistics()
}
});

Logger.Log($"Found {scoreIds.Count} scores which require reprocessing.");
if (scoreIds.Count == 0)
return;

Logger.Log($"Found {scoreIds.Count} scores which require statistics population.");

var notification = showProgressNotification(scoreIds.Count, "Populating missing statistics for scores", "scores have been populated with missing statistics");

int processedCount = 0;
int failedCount = 0;

foreach (var id in scoreIds)
{
if (notification?.State == ProgressNotificationState.Cancelled)
break;

updateNotificationProgress(notification, processedCount, scoreIds.Count);

sleepIfRequired();

try
Expand All @@ -250,7 +293,7 @@ private void processScoresWithMissingStatistics()
r.Find<ScoreInfo>(id)!.MaximumStatisticsJson = JsonConvert.SerializeObject(score.MaximumStatistics);
});

Logger.Log($"Populated maximum statistics for score {id}");
++processedCount;
}
catch (ObjectDisposedException)
{
Expand All @@ -260,8 +303,11 @@ private void processScoresWithMissingStatistics()
{
Logger.Log(@$"Failed to populate maximum statistics for {id}: {e}");
realmAccess.Write(r => r.Find<ScoreInfo>(id)!.BackgroundReprocessingFailed = true);
++failedCount;
}
}

completeNotification(notification, processedCount, scoreIds.Count, failedCount);
}

private void convertLegacyTotalScoreToStandardised()
Expand All @@ -279,20 +325,17 @@ private void convertLegacyTotalScoreToStandardised()
if (scoreIds.Count == 0)
return;

ProgressNotification notification = new ProgressNotification { State = ProgressNotificationState.Active };

notificationOverlay?.Post(notification);
var notification = showProgressNotification(scoreIds.Count, "Upgrading scores to new scoring algorithm", "scores have been upgraded to the new scoring algorithm");

int processedCount = 0;
int failedCount = 0;

foreach (var id in scoreIds)
{
if (notification.State == ProgressNotificationState.Cancelled)
if (notification?.State == ProgressNotificationState.Cancelled)
break;

notification.Text = $"Upgrading scores to new scoring algorithm ({processedCount} of {scoreIds.Count})";
notification.Progress = (float)processedCount / scoreIds.Count;
updateNotificationProgress(notification, processedCount, scoreIds.Count);

sleepIfRequired();

Expand All @@ -310,7 +353,6 @@ private void convertLegacyTotalScoreToStandardised()
s.TotalScoreVersion = LegacyScoreEncoder.LATEST_VERSION;
});

Logger.Log($"Converted total score for score {id}");
++processedCount;
}
catch (ObjectDisposedException)
Expand All @@ -325,24 +367,64 @@ private void convertLegacyTotalScoreToStandardised()
}
}

if (processedCount == scoreIds.Count)
completeNotification(notification, processedCount, scoreIds.Count, failedCount);
}

private void updateNotificationProgress(ProgressNotification? notification, int processedCount, int totalCount)
{
if (notification == null)
return;

notification.Text = notification.Text.ToString().Split('(').First().TrimEnd() + $" ({processedCount} of {totalCount})";
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a bit of a crime but i'm not gonna split hairs for now as none of this is correctly localisable anyways (no pluralisation support)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

split hairs 😎

yeah i dunno, it's isolated and can be refactored if/when we manage to localise this.

notification.Progress = (float)processedCount / totalCount;

if (processedCount % 100 == 0)
Logger.Log(notification.Text.ToString());
}

private void completeNotification(ProgressNotification? notification, int processedCount, int totalCount, int? failedCount = null)
{
if (notification == null)
return;

if (processedCount == totalCount)
{
notification.CompletionText = $"{processedCount} score(s) have been upgraded to the new scoring algorithm";
notification.CompletionText = $"{processedCount} {notification.CompletionText}";
notification.Progress = 1;
notification.State = ProgressNotificationState.Completed;
}
else
{
notification.Text = $"{processedCount} of {scoreIds.Count} score(s) have been upgraded to the new scoring algorithm.";
notification.Text = $"{processedCount} of {totalCount} {notification.CompletionText}";

// We may have arrived here due to user cancellation or completion with failures.
if (failedCount > 0)
notification.Text += $" Check logs for issues with {failedCount} failed upgrades.";
notification.Text += $" Check logs for issues with {failedCount} failed items.";

notification.State = ProgressNotificationState.Cancelled;
}
}

private ProgressNotification? showProgressNotification(int totalCount, string running, string completed)
{
if (notificationOverlay == null)
return null;

if (totalCount < 10)
return null;

ProgressNotification notification = new ProgressNotification
{
Text = running,
CompletionText = completed,
State = ProgressNotificationState.Active
};

notificationOverlay?.Post(notification);

return notification;
}

private void sleepIfRequired()
{
while (localUserPlayInfo?.IsPlaying.Value == true)
Expand Down
2 changes: 2 additions & 0 deletions osu.Game/Beatmaps/BeatmapUpdater.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ public void Process(BeatmapSetInfo beatmapSet, MetadataLookupScope lookupScope =
beatmap.StarRating = calculator.Calculate().StarRating;
beatmap.Length = working.Beatmap.CalculatePlayableLength();
beatmap.BPM = 60000 / working.Beatmap.GetMostCommonBeatLength();
beatmap.EndTimeObjectCount = working.Beatmap.HitObjects.Count(h => h is IHasDuration);
beatmap.TotalObjectCount = working.Beatmap.HitObjects.Count;
}

// And invalidate again afterwards as re-fetching the most up-to-date database metadata will be required.
Expand Down
2 changes: 1 addition & 1 deletion osu.Game/Overlays/Notifications/ProgressNotification.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public override LocalisableString Text
set
{
text = value;
Schedule(() => textDrawable.Text = text);
Scheduler.AddOnce(t => textDrawable.Text = t, text);
}
}

Expand Down
Loading