Various Tweaks and Enhancements

- Added Parsing of Links for the Gemara
- Added Index for Chapter on TopicTextId and Path
- Fixed Id not setting correctly
- Added Chapter TextTopicId

Converter/MainWindow.xaml.cs

@@ -138,11 +138,20 @@ private void ConversionLogic() {
                 textParseList.RemoveAt(0);
 
                 var txt = _serviceMongo.ParseText(raw, _serviceSQLite);
-               //var test = _serviceSQLite.Texts.Local;
-                _serviceSQLite.Add(txt);
+                //var test = _serviceSQLite.Texts.Local;
+
+                _serviceSQLite.Add(_serviceSQLite.Texts,txt);
+                //_serviceSQLite.SaveChanges();
+                foreach (var chapter in _serviceMongo.ListChapters(txt.Chapter))
+                {
+                    chapter.TopicText = txt;
+                    _serviceSQLite.Add(_serviceSQLite.Chapters, chapter);
+                }
 
                 if (i % textBatchAmount == 0)
                 {
+                    //_serviceSQLite.SaveChanges();
+                    //_serviceSQLite.ChangeTracker.DetectChanges();
                     _serviceSQLite.SaveChanges();
                     _serviceSQLite.DisposeAsync();
                     _serviceSQLite = new SefariaSQLiteConversionContext(new Microsoft.EntityFrameworkCore.DbContextOptions<SefariaSQLiteConversionContext> { });

Converter/Model/SQLite/Chapter.cs

@@ -11,6 +11,10 @@ public class Chapter
     {
         [Key]
         public int Id { get; set; }
+        [ForeignKey("Text")]
+        public int TopicTextId { get; set; }
+        public virtual Text TopicText { get; set; }
+
         public bool HasChild { get; set; }
         [ForeignKey("Chapter")]
         public int? ParentChapterId { get; set; }

Converter/Model/SQLite/Text.cs

@@ -8,6 +8,10 @@ namespace Converter.Model.SQLite
 {
     public class Text
     {
+        public Text() {
+            Id = ++IdCounter;
+        }
+        static public int IdCounter = 0;
         [Key]
         public int Id { get; set; }
 

Converter/SefariaMongoDBService.cs

@@ -164,7 +164,7 @@ public Text ParseText(BsonDocument value, SefariaSQLiteConversionContext targetC
                         versionTitleLG.Labels.Add(new Label { LanguageId = (int)LanguageTypes.Hebrew, Text = element.Value.AsString });
                         break;
                     case "chapter":
-                        text.Chapter = GenerateChapterTree(element.Value);
+                        text.Chapter = GenerateChapterTree(text,element.Value);
                         break;
                     default:
                         break;
@@ -176,7 +176,17 @@ public Text ParseText(BsonDocument value, SefariaSQLiteConversionContext targetC
 
             return text;
         }
+        public List<Chapter> ListChapters(Chapter c) {
+            List<Chapter> result = new List<Chapter>();
+            result.Add(c);
+            if(c.Children != null)
+            foreach (var item in c.Children)
+            {
+                result.AddRange(ListChapters(item));
+            }
 
+            return result;
+        }
         private int CountChapters(Chapter c) {
             int count = 0;
             if (c != null) {
@@ -192,23 +202,23 @@ private int CountChapters(Chapter c) {
             return count;
         }
 
-        private Chapter GenerateChapterTree(BsonValue value, Chapter parent = null, int index = 1)
+        private Chapter GenerateChapterTree(Text txt, BsonValue value, Chapter parent = null, int index = 1)
         {
-            Chapter instance = new Chapter { Index = index };
+            Chapter instance = new Chapter { Index = index, TopicText = txt, TopicTextId = txt.Id };
             if (parent != null) {
                 instance.ParentChapter = parent;
             }
 
             //To recreate fast Lookup Path
-            instance.Path = (parent!=null? parent.Index.ToString()+":":"")+instance.Index.ToString();
+            instance.Path = (parent!=null && parent.ParentChapter != null? parent.Path+":":"")+instance.Index.ToString();
 
             switch (value.BsonType) {
                 case BsonType.Array:
                     var array = value.AsBsonArray;
                     instance.Children = new List<Chapter>();
                     for (int i = 0; i < array.Count; i++)
                     {
-                        instance.Children.Add(GenerateChapterTree(array[i], instance, i+1));
+                        instance.Children.Add(GenerateChapterTree(txt, array[i], instance, i+1));
                         instance.HasChild = true;
                     }
                     break;
@@ -218,7 +228,7 @@ private Chapter GenerateChapterTree(BsonValue value, Chapter parent = null, int
                     for (int i = 0; i < document.Elements.Count();i++)
                     {
                         var element = document.GetElement(i);
-                        var child = GenerateChapterTree(element.Value, instance, i+1);
+                        var child = GenerateChapterTree(txt, element.Value, instance, i+1);
                         child.Text = element.Name;
                         instance.Children.Add(child);
                     }
@@ -302,14 +312,41 @@ public LinkItem ParseLink(BsonDocument value, SefariaSQLiteConversionContext tar
                     return null;
                 }
 
+                string parseLocation(string location) {
+                    var gemaraIndicators = new string[] { "a", "b" };
+                    for (int i = 0; i < gemaraIndicators.Length; i++)
+                    {
+                        var indicator = gemaraIndicators[i];
+                        var foundIndex = location.IndexOf(indicator, 0, StringComparison.OrdinalIgnoreCase);
+                        if (foundIndex >= 0) {
+                            var parts = location.Split(":");
+                            for (int j = 0; j < parts.Length; j++)
+                            {
+                                if (parts[j].Contains(indicator)) {
+                                    parts[j] = parts[j].Replace(indicator, "");
+                                    var value = int.Parse(parts[j]);
+                                    value = (value - 1) * 2 + (i + 1);
+                                    parts[j] = value.ToString();
+                                    break;
+                                }
+                            }
+
+                            return string.Join(":", parts);
+                        }
+                    }
+
+                    return location;
+                }
+
                 var primaryTopicSeperator = PrimaryTopic.LastIndexOf(' ');
                 string primaryTopicName = PrimaryTopic.Substring(0, primaryTopicSeperator);
-                string primaryTopicLocation = PrimaryTopic.Substring(primaryTopicSeperator + 1);
+                string primaryTopicLocation = parseLocation(PrimaryTopic.Substring(primaryTopicSeperator + 1));
+
                 int primaryTopicId = targetContext.Topics.Where(t => t.Name == primaryTopicName).Select(t => t.Id).FirstOrDefault();
 
                 var secondaryTopicSeperator = SecondaryTopic.LastIndexOf(' ');
                 string secondaryTopicName = SecondaryTopic.Substring(0, secondaryTopicSeperator);
-                string secondaryTopicLocation = SecondaryTopic.Substring(secondaryTopicSeperator + 1);
+                string secondaryTopicLocation = parseLocation(SecondaryTopic.Substring(secondaryTopicSeperator + 1));
                 int secondaryTopicId = targetContext.Topics.Where(t => t.Name == secondaryTopicName).Select(t => t.Id).FirstOrDefault();
 
                 link.PrimaryLocation = primaryTopicLocation;

Converter/SefariaSQLiteConversionContext.cs

@@ -17,6 +17,7 @@ public SefariaSQLiteConversionContext(DbContextOptions<SefariaSQLiteConversionCo
         public DbSet<LabelGroup> LabelGroups { get; set; }
         public DbSet<Label> Labels { get; set; }
         public DbSet<Text> Texts { get; set; }
+        public DbSet<Chapter> Chapters { get; set; }
         public DbSet<Topic> Topics { get; set; }
         public DbSet<LinkItem> Links { get; set; }
         public DbSet<LinkGroup> LinkGroups { get; set; }
@@ -31,6 +32,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
             base.OnModelCreating(modelBuilder);
 
             //modelBuilder.Entity<Text>().HasKey(t=>t.Id);
+            modelBuilder.Entity<Chapter>().HasIndex(c => c.TopicTextId);
+            modelBuilder.Entity<Chapter>().HasIndex(c => c.Path);
+
         }
 
         private Dictionary<Type, object> _trackedList = new Dictionary<Type, object>();