diff --git a/Converter/MainWindow.xaml.cs b/Converter/MainWindow.xaml.cs index 85e9169..574fd07 100644 --- a/Converter/MainWindow.xaml.cs +++ b/Converter/MainWindow.xaml.cs @@ -1,5 +1,6 @@ using Converter.Model.SQLite; using Converter.Service; +using MongoDB.Bson; using System; using System.Collections.Generic; using System.Diagnostics; @@ -124,17 +125,29 @@ private void ConversionLogic() { //Processing Texts var totalTexts = Total = _serviceMongo.TextsCount(); Log($"Processing: Texts #{totalTexts}"); + int textBatchAmount = 390; + List textParseList = new List(); for (int i = 0; i < totalTexts; i++) { - var txt = _serviceMongo.GetTextAt(i, _serviceSQLite); - var test = _serviceSQLite.Texts.Local; - _serviceSQLite.AddAsync(txt); + if (textParseList.Count == 0) + { + textParseList = _serviceMongo.GetTexts(i, textBatchAmount); + } + + var raw = textParseList[0]; + textParseList.RemoveAt(0); + + var txt = _serviceMongo.ParseText(raw, _serviceSQLite); + //var test = _serviceSQLite.Texts.Local; + _serviceSQLite.Add(txt); - if (i % 260 == 0) + if (i % textBatchAmount == 0) { _serviceSQLite.SaveChanges(); _serviceSQLite.DisposeAsync(); _serviceSQLite = new SefariaSQLiteConversionContext(new Microsoft.EntityFrameworkCore.DbContextOptions { }); + //break;//For Testing Links + } Complete = i; //Log($"Processing: index {i} / total {totalTexts}"); @@ -145,34 +158,46 @@ private void ConversionLogic() { _serviceSQLite.DisposeAsync(); _serviceSQLite = new SefariaSQLiteConversionContext(new Microsoft.EntityFrameworkCore.DbContextOptions { }); - + const int linkBatchAmount = 52000; var totalLinks = Total = _serviceMongo.LinksCount(); - var linksList = _serviceMongo.GetLinks(); + List linksList = new List(); Log($"Processing: Links #{totalLinks}"); - for (int i = 0; i < linksList.Count; i++) + for (int i = 0; i < totalLinks; i++) { - var processing = linksList[i]; - if (processing != null) + if (linksList.Count == 0) + { + linksList = _serviceMongo.GetLinks(i, linkBatchAmount); + } + + if (linksList.Count > 0) { - var link = _serviceMongo.ParseLink(processing, _serviceSQLite); - if (link != null) _serviceSQLite.AddAsync(link); + var processing = linksList[0]; + linksList.RemoveAt(0); + + //var processing = linksList[i]; + if (processing != null) + { + var link = _serviceMongo.ParseLink(processing, _serviceSQLite); + if (link != null) _serviceSQLite.Add(link); + } + //bool hasNew = false; + //if (link.LinkGroup.Id == 0) { + // hasNew = true; + //} + //foreach (var item in link.LinkGroup.LinkedLanguages) + //{ + // if (item.Id == 0) { + // hasNew = true; + // } + //} + + if (i % linkBatchAmount == 0) + { + _serviceSQLite.SaveChanges(); + _serviceSQLite.DisposeAsync(); + _serviceSQLite = new SefariaSQLiteConversionContext(new Microsoft.EntityFrameworkCore.DbContextOptions { }); + } } - //bool hasNew = false; - //if (link.LinkGroup.Id == 0) { - // hasNew = true; - //} - //foreach (var item in link.LinkGroup.LinkedLanguages) - //{ - // if (item.Id == 0) { - // hasNew = true; - // } - //} - - //if (i%520==0) { - // _serviceSQLite.SaveChanges(); - // _serviceSQLite.DisposeAsync(); - // _serviceSQLite = new SefariaSQLiteConversionContext(new Microsoft.EntityFrameworkCore.DbContextOptions { }); - //} Complete = i; } _serviceSQLite.SaveChanges(); diff --git a/Converter/Model/SQLite/Chapter.cs b/Converter/Model/SQLite/Chapter.cs index d7cb11e..cf16d0b 100644 --- a/Converter/Model/SQLite/Chapter.cs +++ b/Converter/Model/SQLite/Chapter.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations.Schema; +using System.IO; using System.Text; namespace Converter.Model.SQLite @@ -16,6 +17,7 @@ public class Chapter public Chapter ParentChapter { get; set; } public ICollection Children { get; set; } public int Index { get; set; } + public string Path { get; set; } public string Text { get; set; } //TODO: Consider Adding a ChapterClonedText to reduce duplication... diff --git a/Converter/Model/SQLite/Link.cs b/Converter/Model/SQLite/Link.cs index a2c2d7e..be012b0 100644 --- a/Converter/Model/SQLite/Link.cs +++ b/Converter/Model/SQLite/Link.cs @@ -11,10 +11,11 @@ public class LinkItem [Key] public int Id { get; set; } [ForeignKey("LinkGroup")] - public int LinkGroupId { get; set; } + public int? LinkGroupId { get; set; } public LinkGroup LinkGroup { get; set; } public string PrimaryLocation { get; set; } public string SecondaryLocation { get; set; } + public string DebugInfo { get; set; } } public class LinkGroup { diff --git a/Converter/Model/SQLite/Text.cs b/Converter/Model/SQLite/Text.cs index a41c624..9fc3cb8 100644 --- a/Converter/Model/SQLite/Text.cs +++ b/Converter/Model/SQLite/Text.cs @@ -32,5 +32,7 @@ public class Text public int? ChapterId { get; set; } public Chapter Chapter { get; set; } + public int ChapterCount { get; set; } + } } diff --git a/Converter/SefariaMongoDBService.cs b/Converter/SefariaMongoDBService.cs index a466853..a7e03ef 100644 --- a/Converter/SefariaMongoDBService.cs +++ b/Converter/SefariaMongoDBService.cs @@ -99,11 +99,21 @@ public long TextsCount() return _texts.CountDocuments(new BsonDocument()); } - public Text GetTextAt(int index, SefariaSQLiteConversionContext targetContext) { + public List GetTexts(int startLocation, int amount) { + var result = new List(); + if (startLocation < TextsCount()) { + result = _texts.Find(_ => true).Skip(startLocation).Limit(amount).ToList(); + } + + return result; + } + + public Text ParseText(BsonDocument value, SefariaSQLiteConversionContext targetContext) { Text text = new Text(); LabelGroup versionTitleLG = new LabelGroup(); versionTitleLG.Labels = new List