Skip to content

Commit

Permalink
CDMS-200 captures ched types of movements & adds to analytics. One of… (
Browse files Browse the repository at this point in the history
#36)

* CDMS-200 captures ched types of movements & adds to analytics. One of the tests is failing so needs investigation

* CDMS-200 finalising country filtering

* Adds test generation unit test that instantiates and validates all scenario generators

* Refactors to reuse chedtype & country filtering

* Removing comments

* Refactoring analytics to re-use link status analysis

* Refactored Link status out of analytics to be calculated at write time and stored on movement

* Fixed tests

* Fixes tests
  • Loading branch information
craigedmunds authored Jan 2, 2025
1 parent 504ff2d commit a80c298
Show file tree
Hide file tree
Showing 48 changed files with 702 additions and 175 deletions.
3 changes: 3 additions & 0 deletions Btms.Analytics.Tests/Fixtures/MultiItemDataTestFixture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public MultiItemDataTestFixture(IMessageSink messageSink)
app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<CrNoMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();

app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<CrNoMatchNoDecisionScenarioGenerator>(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();

app.Services.GeneratorPushToConsumers(_logger, app.Services.CreateScenarioConfig<ChedASimpleMatchScenarioGenerator>(10, 3, arrivalDateRange: 0))
.GetAwaiter().GetResult();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
using Xunit.Abstractions;

using Btms.Analytics.Tests.Fixtures;
using Btms.Model.Ipaffs;

namespace Btms.Analytics.Tests;

[Collection(nameof(BasicSampleDataTestCollection))]
public class ImportNotificationsByVersionTests(
public class ImportNotificationsByMaxVersionTests(
BasicSampleDataTestFixture basicSampleDataTestFixture,
ITestOutputHelper testOutputHelper)
{
Expand All @@ -30,13 +31,11 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(8);
result.Values.Keys.Order().Should().Equal(
"CHEDA Linked", "CHEDA Not Linked", "CHEDD Linked", "CHEDD Not Linked", "CHEDP Linked", "CHEDP Not Linked", "CHEDPP Linked", "CHEDPP Not Linked");
result.Values.Count.Should().Be(1);

}

Expand All @@ -45,10 +44,34 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
.ByMaxVersion(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(8);
result.Values.Count.Should().Be(0);
}

[Fact]
public async Task WhenCalledWithChedType_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(1);
}

[Fact]
public async Task WhenCalledWithCountry_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetImportNotificationsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(1);
}
}
24 changes: 12 additions & 12 deletions Btms.Analytics.Tests/MovementsByCreatedDateTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()

testOutputHelper.WriteLine(result.ToJsonString());

result.Count.Should().Be(2);
result.Count.Should().Be(3);

result[0].Name.Should().Be("Linked");
result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
result[0].Periods.Count.Should().Be(48);
result[1].Name.Should().Be("Linked");
result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
result[1].Periods.Count.Should().Be(48);

result[1].Name.Should().Be("Not Linked");
result[2].Name.Should().Be("Not Linked");
}

[Fact]
Expand All @@ -44,9 +44,9 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati

testOutputHelper.WriteLine(result.ToJsonString());

result.Count.Should().Be(2);
result.Count.Should().Be(3);

result.Select(r => r.Name).Should().Equal("Linked", "Not Linked");
result.Select(r => r.Name).Should().Equal("Investigate", "Linked", "Not Linked");

result.Should().AllSatisfy(r =>
{
Expand All @@ -69,12 +69,12 @@ public async Task WhenCalledLastMonth_ReturnExpectedAggregation()

testOutputHelper.WriteLine(result.ToJsonString());

result.Count.Should().Be(2);
result.Count.Should().Be(3);

result[0].Name.Should().Be("Linked");
result[0].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
result[0].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);
result[1].Name.Should().Be("Linked");
result[1].Periods[0].Period.Should().BeOnOrBefore(DateTime.Today);
result[1].Periods.Count.Should().Be(DateTime.Today.DaysSinceMonthAgo() + 1);

result[1].Name.Should().Be("Not Linked");
result[2].Name.Should().Be("Not Linked");
}
}
4 changes: 2 additions & 2 deletions Btms.Analytics.Tests/MovementsByItemsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()

testOutputHelper.WriteLine("{0} aggregated items found", result.Count);

result.Count.Should().Be(2);
result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked");
result.Count.Should().Be(3);
result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked");

result.Should().AllSatisfy(r =>
{
Expand Down
78 changes: 78 additions & 0 deletions Btms.Analytics.Tests/MovementsByMaxVersionTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
using Btms.Common.Extensions;
using FluentAssertions;
using Xunit;
using Xunit.Abstractions;

using Btms.Analytics.Tests.Fixtures;
using Btms.Model.Ipaffs;

namespace Btms.Analytics.Tests;

[Collection(nameof(BasicSampleDataTestCollection))]
public class MovementsByMaxVersionTests(
BasicSampleDataTestFixture basicSampleDataTestFixture,
ITestOutputHelper testOutputHelper)
{
// [Fact]
// public async Task WhenCalledLastWeek_ReturnExpectedAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.Today.WeekAgo(), DateTime.Today.Tomorrow()));
//
// testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }
//
// [Fact]
// public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour()));
//
// testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }
//
// [Fact]
// public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregation()
// {
// testOutputHelper.WriteLine("Querying for aggregated data");
// var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
// .ByStatus(DateTime.MaxValue.AddDays(-1), DateTime.MaxValue));
//
// testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");
//
// result.Values.Count.Should().Be(2);
// result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
// }

[Fact]
public async Task WhenCalledWithChedType_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), chedTypes: [ImportNotificationTypeEnum.Cveda]));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(2);
}

[Fact]
public async Task WhenCalledWithCountry_ReturnsResults()
{
testOutputHelper.WriteLine("Querying for aggregated data");
var result = (await basicSampleDataTestFixture.GetMovementsAggregationService(testOutputHelper)
.ByMaxVersion(DateTime.Now.NextHour().AddDays(-2), DateTime.Now.NextHour(), country: "AL"));

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(1);
}
}
12 changes: 6 additions & 6 deletions Btms.Analytics.Tests/MovementsByStatusTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()

testOutputHelper.WriteLine("{0} aggregated items found", result.Values.Count);

result.Values.Count.Should().Be(2);
result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
result.Values.Count.Should().Be(3);
result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
}

[Fact]
Expand All @@ -34,8 +34,8 @@ public async Task WhenCalledLast48Hours_ReturnExpectedAggregation()

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(2);
result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
result.Values.Count.Should().Be(3);
result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
}

[Fact]
Expand All @@ -47,7 +47,7 @@ public async Task WhenCalledWithTimePeriodYieldingNoResults_ReturnEmptyAggregati

testOutputHelper.WriteLine($"{result.Values.Count} aggregated items found");

result.Values.Count.Should().Be(2);
result.Values.Keys.Order().Should().Equal("Linked", "Not Linked");
result.Values.Count.Should().Be(3);
result.Values.Keys.Order().Should().Equal("Investigate", "Linked", "Not Linked");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public async Task WhenCalledLastWeek_ReturnExpectedAggregation()

testOutputHelper.WriteLine("{0} aggregated items found", result.Count);

result.Count().Should().Be(2);
result.Select(r => r.Name).Order().Should().Equal("Linked", "Not Linked");
result.Count().Should().Be(3);
result.Select(r => r.Name).Order().Should().Equal("Investigate", "Linked", "Not Linked");

result.Should().AllSatisfy(d =>
{
Expand Down
2 changes: 2 additions & 0 deletions Btms.Analytics/DatasetDimensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Btms.Model.Auditing;
using Btms.Model.Ipaffs;

namespace Btms.Analytics;

Expand All @@ -19,6 +20,7 @@ public class ExceptionResult : IDimensionResult
public required DateTime Updated { get; set; }

public required int ItemCount { get; set; }
public required ImportNotificationTypeEnum[] ChedTypes { get; set; }
public required int MaxEntryVersion { get; set; }
public required int MaxDecisionNumber { get; set; }
public required int LinkedCheds { get; set; }
Expand Down
9 changes: 7 additions & 2 deletions Btms.Analytics/Extensions/AnalyticsHelpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ public static class AnalyticsHelpers
{
internal static DateTime AggregateDateCreator(this BsonValue b) => b["dateToUse"].BsonType != BsonType.Null ? b["dateToUse"].ToUniversalTime() : DateTime.MinValue;

internal static string GetLinkedName(string linked, string type)
{
return $"{type} {linked}";
}

internal static string GetLinkedName(bool linked, string type)
{
return $"{type} {GetLinkedName(linked)}";
Expand All @@ -46,8 +51,8 @@ public static string[] GetImportNotificationSegments()
.ToArray();
}

public static string[] GetMovementSegments()
public static string[] GetMovementStatusSegments()
{
return ["Linked", "Not Linked"];
return ["Linked", "Not Linked", "Investigate"];
}
}
42 changes: 42 additions & 0 deletions Btms.Analytics/Extensions/MovementExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
using Btms.Model;
using Btms.Model.Ipaffs;

namespace Btms.Analytics.Extensions;

public static class MovementExtensions
{
public static IQueryable<Movement> WhereFilteredByCreatedDateAndParams(this IQueryable<Movement> source, DateTime from, DateTime to,
ImportNotificationTypeEnum[]? chedTypes = null, string? country = null)
{
return source
.Where(m => (m.CreatedSource >= from && m.CreatedSource < to)
&& (country == null || m.DispatchCountryCode == country)
&& (chedTypes == null || !chedTypes!.Any() ||
!m.AlvsDecisionStatus!.Context!.ChedTypes!.Any() ||
m.AlvsDecisionStatus!.Context!.ChedTypes!.Any(c => chedTypes!.Contains(c))));

}

public class MovementWithLinkStatus
{
public required Movement Movement;
public required DateTime CreatedSource;
public required string Description;
}

public static IQueryable<MovementWithLinkStatus> SelectLinkStatus(this IQueryable<Movement> source)
{
var m = source
.Select(m => new MovementWithLinkStatus() {
Movement = m,
CreatedSource = m.CreatedSource!.Value,
Description =
m.Status.LinkStatus
// m.Relationships.Notifications.Data.Count > 0 ? "Linked" :
// m.AlvsDecisionStatus!.Context!.AlvsCheckStatus!.AnyMatch ? "Investigate" :
// "Not Linked"
});

return m;
}
}
4 changes: 3 additions & 1 deletion Btms.Analytics/IImportNotificationsAggregationService.cs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using Btms.Model.Ipaffs;

namespace Btms.Analytics;

public interface IImportNotificationsAggregationService
Expand All @@ -6,5 +8,5 @@ public interface IImportNotificationsAggregationService
public Task<MultiSeriesDatetimeDataset> ByArrival(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByCommodityCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
}
11 changes: 6 additions & 5 deletions Btms.Analytics/IMovementsAggregationService.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Btms.Model.Auditing;
using Btms.Model.Ipaffs;

namespace Btms.Analytics;

Expand All @@ -7,15 +8,15 @@ public interface IMovementsAggregationService
public Task<MultiSeriesDatetimeDataset> ByCreated(DateTime from, DateTime to, AggregationPeriod aggregateBy = AggregationPeriod.Day);
public Task<SingleSeriesDataset> ByStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByItemCount(DateTime from, DateTime to);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SummarisedDataset<SingleSeriesDataset, StringBucketDimensionResult>> ByDecision(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
// public Task<TabularDataset<ByNameDimensionResult>> ByDecisionAndLinkStatus(DateTime from, DateTime to);
public Task<MultiSeriesDataset> ByUniqueDocumentReferenceCount(DateTime from, DateTime to);
public Task<SingleSeriesDataset> UniqueDocumentReferenceByMovementCount(DateTime from, DateTime to);
// public Task<MultiSeriesDataset> ByCheck(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<EntityDataset<AuditHistory>?> GetHistory(string movementId);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, string[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxVersion(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ByMaxDecisionNumber(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<List<ExceptionResult>> GetExceptions(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);
public Task<SingleSeriesDataset> ExceptionSummary(DateTime from, DateTime to, ImportNotificationTypeEnum[]? chedTypes = null, string? country = null);

}
Loading

0 comments on commit a80c298

Please sign in to comment.