Do not duplicate registration of data derived from the same file

YaSuenag · Dec 20, 2023 · 84662a4 · 84662a4
1 parent 046e5ec
commit 84662a4
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 2 deletions.
diff --git a/src/CarbonAware.LocationSources/src/LocationSource.cs b/src/CarbonAware.LocationSources/src/LocationSource.cs
@@ -57,6 +57,7 @@ private async Task LoadLocationJsonFileAsync()
     {
         var options = new JsonSerializerOptions(JsonSerializerDefaults.Web);
         var keyCounter = new Dictionary<string, int>(); // used to keep track of key dups
+        var processedFiles = new HashSet<string>(); // used to keep name of processed files
 
         var sourceFiles = !_configuration.LocationSourceFiles.Any() ? DiscoverFiles() : _configuration.LocationSourceFiles;
         foreach (var source in sourceFiles)
@@ -68,8 +69,9 @@ private async Task LoadLocationJsonFileAsync()
                 var geoInstance = namedGeoMap[locationKey];
                 geoInstance.AssertValid();
                 var key = BuildKey(source, locationKey);
-                AddToLocationMap(key, geoInstance, source.DataFileLocation, keyCounter);
+                AddToLocationMap(key, geoInstance, source.DataFileLocation, keyCounter, processedFiles);
             }
+            processedFiles.Add(source.DataFileLocation);
         }
     }
 
@@ -112,7 +114,7 @@ private IEnumerable<LocationSourceFile> DiscoverFiles()
         return files.Select(x => x.Substring(pathCombined.Length + 1)).Select(n => new LocationSourceFile { DataFileLocation = n });
     }
 
-    private void AddToLocationMap(string key, NamedGeoposition data, string sourceFile, Dictionary<string, int> keyCounter)
+    private void AddToLocationMap(string key, NamedGeoposition data, string sourceFile, Dictionary<string, int> keyCounter, HashSet<string> processedFiles)
     {
         var loc = (Location) data;
 
@@ -121,6 +123,11 @@ private void AddToLocationMap(string key, NamedGeoposition data, string sourceFi
             keyCounter.Add(key, 0);
             return;
         }
+        if (processedFiles.Contains(sourceFile))
+        {
+            // We can ignore because same location from same file is already added.
+            return;
+        }
         // Generate new key using keyCounter counter
         _logger.LogWarning("Location key {key} from {sourceFile} already exists. Creating new key.", key, sourceFile);
         var counter = keyCounter[key];

diff --git a/src/CarbonAware.LocationSources/test/LocationSourceTest.cs b/src/CarbonAware.LocationSources/test/LocationSourceTest.cs
@@ -120,6 +120,46 @@ public async Task GeopositionLocation_ValidLocation_With_MultiConfiguration()
         AssertLocationsEqual(Constants.LocationWestUs, westResult);
     }
 
+    [Test]
+    public async Task GeopositionLocation_ValidLocation_With_SameMultiConfiguration()
+    {
+        var configuration = new LocationDataSourcesConfiguration();
+        configuration.LocationSourceFiles.Add(new LocationSourceFile
+        {
+            Prefix = "prefix",
+            Delimiter = "-",
+            DataFileLocation = _goodFile
+        });
+        configuration.LocationSourceFiles.Add(new LocationSourceFile
+        {
+            Prefix = "prefix",
+            Delimiter = "-",
+            DataFileLocation = _goodFile
+        });
+        var options = new Mock<IOptionsMonitor<LocationDataSourcesConfiguration>>();
+        options.Setup(o => o.CurrentValue).Returns(() => configuration);
+        var logger = Mock.Of<ILogger<LocationSource>>();
+        var locationSource = new LocationSource(logger, options.Object);
+
+        Location inputLocation = new Location
+        {
+            Name = "prefix-test-eastus"
+        };
+
+        var result = await locationSource.ToGeopositionLocationAsync(inputLocation);
+        AssertLocationsEqual(Constants.LocationEastUs, result);
+
+        inputLocation = new Location
+        {
+            Name = "prefix_test-eastus_1" // generated key when the key is duplicated
+        };
+
+        Assert.ThrowsAsync<ArgumentException>(async () =>
+        {
+            await locationSource.ToGeopositionLocationAsync(inputLocation);
+        });
+    }
+
     [TestCase("prefix-test-eastus", TestName = "ValidLocation_CaseInsensitive: exact match case")]
     [TestCase("PrEfIx-test-eastus", TestName = "ValidLocation_CaseInsensitive: prefix case insensitive")]
     [TestCase("prefix-teST-EAstus", TestName = "ValidLocation_CaseInsensitive: location name case insensitive")]