Add mismatches cache to improve performance on ultra-long reads

packages/core/BaseFeatureWidget/SequenceFeatureDetails/model.ts

@@ -12,22 +12,54 @@ function localStorageGetNumber(key: string, defaultVal: number) {
   return +(localStorageGetItem(key) ?? defaultVal)
 }
 
+function localStorageGetBoolean(key: string, defaultVal: boolean) {
+  return Boolean(
+    JSON.parse(localStorageGetItem(key) || JSON.stringify(defaultVal)),
+  )
+}
+
+function localStorageSetNumber(key: string, value: number) {
+  localStorageSetItem(key, JSON.stringify(value))
+}
+
+function localStorageSetBoolean(key: string, value: boolean) {
+  localStorageSetItem(key, JSON.stringify(value))
+}
+
+const p = 'sequenceFeatureDetails'
+
 export function SequenceFeatureDetailsF() {
   return types
     .model('SequenceFeatureDetails')
     .volatile(() => ({
+      /**
+       * #volatile
+       */
       showCoordinatesSetting:
-        localStorageGetItem('sequenceFeatureDetails-showCoordinatesSetting') ||
-        'none',
-      intronBp: localStorageGetNumber('sequenceFeatureDetails-intronBp', 10),
-      upDownBp: localStorageGetNumber('sequenceFeatureDetails-upDownBp', 100),
-      upperCaseCDS: Boolean(
-        JSON.parse(
-          localStorageGetItem('sequenceFeatureDetails-upperCaseCDS') || 'true',
-        ),
-      ),
+        localStorageGetItem(`${p}-showCoordinatesSetting`) || 'none',
+      /**
+       * #volatile
+       */
+      intronBp: localStorageGetNumber(`${p}-intronBp`, 10),
+      /**
+       * #volatile
+       */
+      upDownBp: localStorageGetNumber(`${p}-upDownBp`, 100),
+      /**
+       * #volatile
+       */
+      upperCaseCDS: localStorageGetBoolean(`${p}-upperCaseCDS`, true),
+      /**
+       * #volatile
+       */
       charactersPerRow: 100,
+      /**
+       * #volatile
+       */
       feature: undefined as SimpleFeatureSerialized | undefined,
+      /**
+       * #volatile
+       */
       mode: '',
     }))
     .actions(self => ({
@@ -110,20 +142,11 @@ export function SequenceFeatureDetailsF() {
         addDisposer(
           self,
           autorun(() => {
+            localStorageSetNumber(`${p}-upDownBp`, self.upDownBp)
+            localStorageSetNumber(`${p}-intronBp`, self.intronBp)
+            localStorageSetBoolean(`${p}-upperCaseCDS`, self.upperCaseCDS)
             localStorageSetItem(
-              'sequenceFeatureDetails-upDownBp',
-              JSON.stringify(self.upDownBp),
-            )
-            localStorageSetItem(
-              'sequenceFeatureDetails-intronBp',
-              JSON.stringify(self.intronBp),
-            )
-            localStorageSetItem(
-              'sequenceFeatureDetails-upperCaseCDS',
-              JSON.stringify(self.upperCaseCDS),
-            )
-            localStorageSetItem(
-              'sequenceFeatureDetails-showCoordinatesSetting',
+              `${p}-showCoordinatesSetting`,
               self.showCoordinatesSetting,
             )
           }),

packages/core/util/index.ts

@@ -609,24 +609,29 @@ export function cartesianToPolar(x: number, y: number) {
   const theta = Math.atan(y / x)
   return [rho, theta] as [number, number]
 }
+interface MinimalRegion {
+  start: number
+  end: number
+  reversed?: boolean
+}
 
 export function featureSpanPx(
   feature: Feature,
-  region: { start: number; end: number; reversed?: boolean },
+  region: MinimalRegion,
   bpPerPx: number,
-): [number, number] {
+) {
   return bpSpanPx(feature.get('start'), feature.get('end'), region, bpPerPx)
 }
 
 export function bpSpanPx(
   leftBp: number,
   rightBp: number,
-  region: { start: number; end: number; reversed?: boolean },
+  region: MinimalRegion,
   bpPerPx: number,
-): [number, number] {
+) {
   const start = bpToPx(leftBp, region, bpPerPx)
   const end = bpToPx(rightBp, region, bpPerPx)
-  return region.reversed ? [end, start] : [start, end]
+  return region.reversed ? ([end, start] as const) : ([start, end] as const)
 }
 
 // do an array map of an iterable
@@ -646,8 +651,7 @@ export function iterMap<T, U>(
 
 /**
  * Returns the index of the last element in the array where predicate is true,
- * and -1 otherwise.
- * Based on https://stackoverflow.com/a/53187807
+ * and -1 otherwise. Based on https://stackoverflow.com/a/53187807
  *
  * @param array - The source array to search in
  *
@@ -660,7 +664,7 @@ export function iterMap<T, U>(
 export function findLastIndex<T>(
   array: T[],
   predicate: (value: T, index: number, obj: T[]) => boolean,
-): number {
+) {
   let l = array.length
   while (l--) {
     if (predicate(array[l]!, l, array)) {
@@ -673,7 +677,7 @@ export function findLastIndex<T>(
 export function findLast<T>(
   array: T[],
   predicate: (value: T, index: number, obj: T[]) => boolean,
-): T | undefined {
+) {
   let l = array.length
   while (l--) {
     if (predicate(array[l]!, l, array)) {

plugins/alignments/src/BamAdapter/BamAdapter.ts

@@ -1,4 +1,7 @@
 import { BamFile } from '@gmod/bam'
+import { toArray } from 'rxjs/operators'
+import { firstValueFrom } from 'rxjs'
+// jbrowse
 import {
   BaseFeatureDataAdapter,
   BaseOptions,
@@ -7,8 +10,7 @@ import { Region } from '@jbrowse/core/util/types'
 import { bytesForRegions, updateStatus, Feature } from '@jbrowse/core/util'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
-import { toArray } from 'rxjs/operators'
-import { firstValueFrom } from 'rxjs'
+import QuickLRU from '@jbrowse/core/util/QuickLRU'
 
 // locals
 import BamSlightlyLazyFeature from './BamSlightlyLazyFeature'
@@ -23,6 +25,9 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
   private samHeader?: Header
 
   private setupP?: Promise<Header>
+
+  private featureCache = new QuickLRU<string, Feature>({ maxSize: 5000 })
+
   private configureP?: Promise<{
     bam: BamFile
     sequenceAdapter?: BaseFeatureDataAdapter
@@ -215,7 +220,17 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
             continue
           }
 
-          observer.next(new BamSlightlyLazyFeature(record, this, ref))
+          // retrieve a feature from our feature cache if it is available, the
+          // features in the cache have pre-computed mismatches objects that
+          // can be re-used across blocks
+          const ret = this.featureCache.get(`${record.id}`)
+          if (!ret) {
+            const elt = new BamSlightlyLazyFeature(record, this, ref)
+            this.featureCache.set(`${record.id}`, elt)
+            observer.next(elt)
+          } else {
+            observer.next(ret)
+          }
         }
         observer.complete()
       })

plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts

@@ -7,6 +7,7 @@ import { BamRecord } from '@gmod/bam'
 // locals
 import { getMismatches } from '../MismatchParser'
 import BamAdapter from './BamAdapter'
+import { cacheGetter } from '../shared/util'
 
 export default class BamSlightlyLazyFeature implements Feature {
   // uses parameter properties to automatically create fields on the class
@@ -30,11 +31,15 @@ export default class BamSlightlyLazyFeature implements Feature {
     )
   }
 
+  get qual() {
+    return this.record.qual?.join(' ')
+  }
+
   get(field: string): any {
     return field === 'mismatches'
       ? this.mismatches
       : field === 'qual'
-        ? this.record.qual?.join(' ')
+        ? this.qual
         : this.fields[field]
   }
 
@@ -74,27 +79,12 @@ export default class BamSlightlyLazyFeature implements Feature {
   }
 
   toJSON(): SimpleFeatureSerialized {
-    return this.fields
-  }
-}
-
-function cacheGetter<T>(ctor: { prototype: T }, prop: keyof T): void {
-  const desc = Object.getOwnPropertyDescriptor(ctor.prototype, prop)
-  if (!desc) {
-    throw new Error('t1')
-  }
-
-  const getter = desc.get
-  if (!getter) {
-    throw new Error('t2')
+    return {
+      ...this.fields,
+      qual: this.qual,
+    }
   }
-  Object.defineProperty(ctor.prototype, prop, {
-    get() {
-      const ret = getter.call(this)
-      Object.defineProperty(this, prop, { value: ret })
-      return ret
-    },
-  })
 }
 
 cacheGetter(BamSlightlyLazyFeature, 'fields')
+cacheGetter(BamSlightlyLazyFeature, 'mismatches')

plugins/alignments/src/BamAdapter/__snapshots__/BamAdapter.test.ts.snap

@@ -11,6 +11,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "TACACTGGTTCGGAGACGGTTCGTGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATGAGCGGCCGCTGACCGGCGGCACGACTAATATAGTGCAAGA",
@@ -39,6 +40,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "ACACTGGTTCGGAGACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCGGCCCCTGTCCGGCGGCACGAATAATATAGTGCAAGAA",
@@ -67,6 +69,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "CTGGTTCGGAGACGGTTCATGACGACCGCGCTATATGTCGGCATCTGCGTCGCATGAGCGGCCGCTGTCCGGCGGCTCGAATAATATAGTGCAAGAAAAA",
@@ -95,6 +98,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "AGACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACT",
@@ -123,6 +127,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "GACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCCGCCGCTGTCCGACGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTA",
@@ -151,6 +156,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "TTCATGACGAGCGCGCTATATGACGGCATCTGCGCCGCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTT",
@@ -179,6 +185,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "TCATGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATCAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTA",
@@ -207,6 +214,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "AGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTATATATGAT",
@@ -235,6 +243,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "CCCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCTAAGACTACGGTTATATATGATGGAACGGCCCTCACAGCATTCTCACA",
@@ -263,6 +272,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = `
     "next_ref": undefined,
     "next_segment_position": undefined,
     "pair_orientation": undefined,
+    "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17",
     "refName": "ctgA",
     "score": 37,
     "seq": "CATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTATATATGATGGAACGGCCCTCACAGCATTGTAACAGG",

plugins/alignments/src/CramAdapter/CramAdapter.ts

@@ -1,4 +1,7 @@
 import { CraiIndex, IndexedCramFile, CramRecord } from '@gmod/cram'
+import { toArray } from 'rxjs/operators'
+import { firstValueFrom } from 'rxjs'
+// jbrowse
 import {
   BaseFeatureDataAdapter,
   BaseOptions,
@@ -8,8 +11,7 @@ import type { Region, Feature } from '@jbrowse/core/util'
 import { checkAbortSignal, updateStatus, toLocale } from '@jbrowse/core/util'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
-import { toArray } from 'rxjs/operators'
-import { firstValueFrom } from 'rxjs'
+import QuickLRU from '@jbrowse/core/util/QuickLRU'
 
 // locals
 import CramSlightlyLazyFeature from './CramSlightlyLazyFeature'
@@ -35,6 +37,8 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
     sequenceAdapter: BaseSequenceAdapter
   }>
 
+  private featureCache = new QuickLRU<string, Feature>({ maxSize: 5000 })
+
   // maps a refname to an id
   private seqIdToRefName: string[] | undefined
 
@@ -266,7 +270,17 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
           if (readName && record.readName !== readName) {
             continue
           }
-          observer.next(this.cramRecordToFeature(record))
+          // retrieve a feature from our feature cache if it is available, the
+          // features in the cache have pre-computed mismatches objects that
+          // can be re-used across blocks
+          const ret = this.featureCache.get(`${record.uniqueId}`)
+          if (!ret) {
+            const elt = this.cramRecordToFeature(record)
+            this.featureCache.set(`${record.uniqueId}`, elt)
+            observer.next(elt)
+          } else {
+            observer.next(ret)
+          }
         }
 
         observer.complete()