From 6d89c72187a92303d001036339bfd8a4aaeea830 Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 12 Nov 2024 22:11:06 -0500 Subject: [PATCH 1/2] Feature cache --- .../alignments/src/BamAdapter/BamAdapter.ts | 21 ++++++++- .../src/BamAdapter/BamSlightlyLazyFeature.ts | 1 + yarn.lock | 46 +++++++++---------- 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/plugins/alignments/src/BamAdapter/BamAdapter.ts b/plugins/alignments/src/BamAdapter/BamAdapter.ts index 8a6df1ee87..9d7ca34451 100644 --- a/plugins/alignments/src/BamAdapter/BamAdapter.ts +++ b/plugins/alignments/src/BamAdapter/BamAdapter.ts @@ -9,6 +9,7 @@ import { openLocation } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' import { toArray } from 'rxjs/operators' import { firstValueFrom } from 'rxjs' +import QuickLRU from '@jbrowse/core/util/QuickLRU' // locals import BamSlightlyLazyFeature from './BamSlightlyLazyFeature' @@ -23,6 +24,11 @@ export default class BamAdapter extends BaseFeatureDataAdapter { private samHeader?: Header private setupP?: Promise
+ + private featureMap = new QuickLRU({ + maxSize: 5000, + }) + private configureP?: Promise<{ bam: BamFile sequenceAdapter?: BaseFeatureDataAdapter @@ -184,6 +190,8 @@ export default class BamAdapter extends BaseFeatureDataAdapter { readName, } = filterBy || {} + let retrievedFromCache = 0 + let notRetrievedFromCache = 0 for (const record of records) { let ref: string | undefined if (!record.tags.MD) { @@ -215,8 +223,19 @@ export default class BamAdapter extends BaseFeatureDataAdapter { continue } - observer.next(new BamSlightlyLazyFeature(record, this, ref)) + const ret = this.featureMap.get(`${record.id}`) + + if (!ret) { + const elt = new BamSlightlyLazyFeature(record, this, ref) + this.featureMap.set(`${record.id}`, elt) + observer.next(elt) + notRetrievedFromCache++ + } else { + retrievedFromCache++ + observer.next(ret) + } } + // console.log({ retrievedFromCache, notRetrievedFromCache }) observer.complete() }) }, signal) diff --git a/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts b/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts index 7924f54a6a..ae3d8ecc0d 100644 --- a/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts +++ b/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts @@ -98,3 +98,4 @@ function cacheGetter(ctor: { prototype: T }, prop: keyof T): void { } cacheGetter(BamSlightlyLazyFeature, 'fields') +cacheGetter(BamSlightlyLazyFeature, 'mismatches') diff --git a/yarn.lock b/yarn.lock index 0308390caa..e214204ba1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -89,9 +89,9 @@ tslib "^2.6.2" "@aws-sdk/client-cloudfront@^3.687.0": - version "3.689.0" - resolved "https://registry.yarnpkg.com/@aws-sdk/client-cloudfront/-/client-cloudfront-3.689.0.tgz#3b64f47e7ee2627a8904b57a9cb7636af76ea712" - integrity sha512-0GaRHkkjoc06bx0OZKcJgDZoljP6z9+F0g+t094WovpxVj4pg9TJ8rPGCtQ9/ZQqzRCMvWS9CXYkZpfmo+Mbug== + version "3.690.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-cloudfront/-/client-cloudfront-3.690.0.tgz#fec87087365a223bb62c139f7508a61bd0cba77b" + integrity sha512-3gI6Ky3zYDkGG5gQdBRZjSup2sOpsfPDiWKe38jYmA7rdIV4Ec0An/DaBtWJ5eZukJREX5rvwbNjQryeyHdUxA== dependencies: "@aws-crypto/sha256-browser" "5.2.0" "@aws-crypto/sha256-js" "5.2.0" @@ -3465,9 +3465,9 @@ wrap-ansi "^7.0.0" "@oclif/core@^4", "@oclif/core@^4.0.31": - version "4.0.31" - resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.0.31.tgz#3f7ac806f27de6a87a7ee7caab8826687ce50412" - integrity sha512-7oyIZv/C1TP+fPc2tSzVPYqG1zU+nel1QvJxjAWyVhud0J8B5SpKZnryedxs3nlSVPJ6K1MT31C9esupCBYgZw== + version "4.0.32" + resolved "https://registry.yarnpkg.com/@oclif/core/-/core-4.0.32.tgz#0e8078c53b079549d685798893b9f9534ca69bf6" + integrity sha512-O3jfIAhqaJxXI2dzF81PLTMhKpFFA0Nyz8kfBnc9WYDJnvdmXK0fVAOSpwpi2mHTow/9FXxY6Kww8+Kbe7/sag== dependencies: ansi-escapes "^4.3.2" ansis "^3.3.2" @@ -3686,9 +3686,9 @@ integrity sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A== "@r2wc/core@^1.0.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@r2wc/core/-/core-1.1.0.tgz#93c16cd5bf5bc6d9a52b23536aed4e6a4d2a6ef2" - integrity sha512-pEgtPXhfgg8mv/MooU83cb5sXC2aQOXPLm9UX7E7Oz/OXmrnP5r8hD/nJL1empWxC4wo1YeBXvrFu8fXsMgGZQ== + version "1.2.0" + resolved "https://registry.yarnpkg.com/@r2wc/core/-/core-1.2.0.tgz#a86cd2f732f499a4de8fa23b037053323f3749ed" + integrity sha512-vAfiuS5KywtV54SRzc4maEHcpdgeUyJzln+ATpNCOkO+ArIuOkTXd92b5YauVAd0A8B2rV/y9OeVW19vb73bUQ== "@r2wc/react-to-web-component@^2.0.3": version "2.0.3" @@ -6740,9 +6740,9 @@ ci-info@^3.2.0: integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== ci-info@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.0.0.tgz#65466f8b280fc019b9f50a5388115d17a63a44f2" - integrity sha512-TdHqgGf9odd8SXNuxtUBVx8Nv+qZOejE6qyqiy5NtbYYQOeFa6zmHkxlPzmaLxWWHsU6nJmB7AETdVPi+2NBUg== + version "4.1.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.1.0.tgz#92319d2fa29d2620180ea5afed31f589bc98cf83" + integrity sha512-HutrvTNsF48wnxkzERIXOe5/mlcfFcbfCmwcg6CJnizbSue78AbDt+1cgl26zwn61WFxhcPykPfZrbqjGmBb4A== cjs-module-lexer@^1.0.0, cjs-module-lexer@^1.2.3: version "1.4.1" @@ -8183,9 +8183,9 @@ electron-publish@25.1.7: mime "^2.5.2" electron-to-chromium@^1.5.41: - version "1.5.56" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.56.tgz#3213f369efc3a41091c3b2c05bc0f406108ac1df" - integrity sha512-7lXb9dAvimCFdvUMTyucD4mnIndt/xhRKFAlky0CyFogdnNmdPQNoHI23msF/2V4mpTxMzgMdjK4+YRlFlRQZw== + version "1.5.57" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.57.tgz#cb43af8784166bca24565b3418bf5f775a6b1c86" + integrity sha512-xS65H/tqgOwUBa5UmOuNSLuslDo7zho0y/lgQw35pnrqiZh7UOWHCeL/Bt6noJATbA6tpQJGCifsFsIRZj1Fqg== electron-updater@^6.1.1: version "6.3.9" @@ -8331,9 +8331,9 @@ error-stack-parser@^2.0.6: stackframe "^1.3.4" es-abstract@^1.17.5, es-abstract@^1.22.1, es-abstract@^1.22.3, es-abstract@^1.23.0, es-abstract@^1.23.1, es-abstract@^1.23.2, es-abstract@^1.23.3: - version "1.23.3" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.23.3.tgz#8f0c5a35cd215312573c5a27c87dfd6c881a0aa0" - integrity sha512-e+HfNH61Bj1X9/jLc5v1owaLYuHdeHHSQlkhCBiTK8rBvKaULl/beGMxwrMXjpYrv4pz22BlY570vVePA2ho4A== + version "1.23.4" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.23.4.tgz#f006871f484d6a78229d2343557f2597f8333ed4" + integrity sha512-HR1gxH5OaiN7XH7uiWH0RLw0RcFySiSoW1ctxmD1ahTw3uGBtkmm/ng0tDU1OtYx5OK6EOL5Y6O21cDflG3Jcg== dependencies: array-buffer-byte-length "^1.0.1" arraybuffer.prototype.slice "^1.0.3" @@ -8350,7 +8350,7 @@ es-abstract@^1.17.5, es-abstract@^1.22.1, es-abstract@^1.22.3, es-abstract@^1.23 function.prototype.name "^1.1.6" get-intrinsic "^1.2.4" get-symbol-description "^1.0.2" - globalthis "^1.0.3" + globalthis "^1.0.4" gopd "^1.0.1" has-property-descriptors "^1.0.2" has-proto "^1.0.3" @@ -8366,10 +8366,10 @@ es-abstract@^1.17.5, es-abstract@^1.22.1, es-abstract@^1.22.3, es-abstract@^1.23 is-string "^1.0.7" is-typed-array "^1.1.13" is-weakref "^1.0.2" - object-inspect "^1.13.1" + object-inspect "^1.13.3" object-keys "^1.1.1" object.assign "^4.1.5" - regexp.prototype.flags "^1.5.2" + regexp.prototype.flags "^1.5.3" safe-array-concat "^1.1.2" safe-regex-test "^1.0.3" string.prototype.trim "^1.2.9" @@ -12704,7 +12704,7 @@ object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.1: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== -object-inspect@^1.13.1: +object-inspect@^1.13.1, object-inspect@^1.13.3: version "1.13.3" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.3.tgz#f14c183de51130243d6d18ae149375ff50ea488a" integrity sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA== @@ -14163,7 +14163,7 @@ regexp-tree@^0.1.27: resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd" integrity sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA== -regexp.prototype.flags@^1.5.2: +regexp.prototype.flags@^1.5.2, regexp.prototype.flags@^1.5.3: version "1.5.3" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.3.tgz#b3ae40b1d2499b8350ab2c3fe6ef3845d3a96f42" integrity sha512-vqlC04+RQoFalODCbCumG2xIOvapzVMHwsyIGM/SIE8fRhFFsXeH8/QQ+s0T0kDAhKc4k30s73/0ydkHQz6HlQ== From 7eff4bf20ba729bd127d58734483fc4a1760bc8e Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 12 Nov 2024 22:25:17 -0500 Subject: [PATCH 2/2] Misc --- .../SequenceFeatureDetails/model.ts | 67 +++++++++++++------ packages/core/util/index.ts | 22 +++--- .../alignments/src/BamAdapter/BamAdapter.ts | 22 +++--- .../src/BamAdapter/BamSlightlyLazyFeature.ts | 31 +++------ .../__snapshots__/BamAdapter.test.ts.snap | 10 +++ .../alignments/src/CramAdapter/CramAdapter.ts | 20 +++++- .../CramAdapter/CramSlightlyLazyFeature.ts | 21 +----- .../src/PileupRenderer/makeImageData.ts | 4 -- .../src/PileupRenderer/renderAlignment.ts | 21 ++++-- .../PileupRenderer/renderAlignmentShape.ts | 6 +- .../src/PileupRenderer/renderMismatches.ts | 3 +- .../src/PileupRenderer/renderSoftClipping.ts | 5 +- .../generateCoverageBins.ts | 8 +-- plugins/alignments/src/shared/util.ts | 12 ++++ test_data/config_demo.json | 4 ++ 15 files changed, 146 insertions(+), 110 deletions(-) diff --git a/packages/core/BaseFeatureWidget/SequenceFeatureDetails/model.ts b/packages/core/BaseFeatureWidget/SequenceFeatureDetails/model.ts index ec2b38006c..da603b7797 100644 --- a/packages/core/BaseFeatureWidget/SequenceFeatureDetails/model.ts +++ b/packages/core/BaseFeatureWidget/SequenceFeatureDetails/model.ts @@ -12,22 +12,54 @@ function localStorageGetNumber(key: string, defaultVal: number) { return +(localStorageGetItem(key) ?? defaultVal) } +function localStorageGetBoolean(key: string, defaultVal: boolean) { + return Boolean( + JSON.parse(localStorageGetItem(key) || JSON.stringify(defaultVal)), + ) +} + +function localStorageSetNumber(key: string, value: number) { + localStorageSetItem(key, JSON.stringify(value)) +} + +function localStorageSetBoolean(key: string, value: boolean) { + localStorageSetItem(key, JSON.stringify(value)) +} + +const p = 'sequenceFeatureDetails' + export function SequenceFeatureDetailsF() { return types .model('SequenceFeatureDetails') .volatile(() => ({ + /** + * #volatile + */ showCoordinatesSetting: - localStorageGetItem('sequenceFeatureDetails-showCoordinatesSetting') || - 'none', - intronBp: localStorageGetNumber('sequenceFeatureDetails-intronBp', 10), - upDownBp: localStorageGetNumber('sequenceFeatureDetails-upDownBp', 100), - upperCaseCDS: Boolean( - JSON.parse( - localStorageGetItem('sequenceFeatureDetails-upperCaseCDS') || 'true', - ), - ), + localStorageGetItem(`${p}-showCoordinatesSetting`) || 'none', + /** + * #volatile + */ + intronBp: localStorageGetNumber(`${p}-intronBp`, 10), + /** + * #volatile + */ + upDownBp: localStorageGetNumber(`${p}-upDownBp`, 100), + /** + * #volatile + */ + upperCaseCDS: localStorageGetBoolean(`${p}-upperCaseCDS`, true), + /** + * #volatile + */ charactersPerRow: 100, + /** + * #volatile + */ feature: undefined as SimpleFeatureSerialized | undefined, + /** + * #volatile + */ mode: '', })) .actions(self => ({ @@ -110,20 +142,11 @@ export function SequenceFeatureDetailsF() { addDisposer( self, autorun(() => { + localStorageSetNumber(`${p}-upDownBp`, self.upDownBp) + localStorageSetNumber(`${p}-intronBp`, self.intronBp) + localStorageSetBoolean(`${p}-upperCaseCDS`, self.upperCaseCDS) localStorageSetItem( - 'sequenceFeatureDetails-upDownBp', - JSON.stringify(self.upDownBp), - ) - localStorageSetItem( - 'sequenceFeatureDetails-intronBp', - JSON.stringify(self.intronBp), - ) - localStorageSetItem( - 'sequenceFeatureDetails-upperCaseCDS', - JSON.stringify(self.upperCaseCDS), - ) - localStorageSetItem( - 'sequenceFeatureDetails-showCoordinatesSetting', + `${p}-showCoordinatesSetting`, self.showCoordinatesSetting, ) }), diff --git a/packages/core/util/index.ts b/packages/core/util/index.ts index d7617d8b76..36fb0c1d7f 100644 --- a/packages/core/util/index.ts +++ b/packages/core/util/index.ts @@ -609,24 +609,29 @@ export function cartesianToPolar(x: number, y: number) { const theta = Math.atan(y / x) return [rho, theta] as [number, number] } +interface MinimalRegion { + start: number + end: number + reversed?: boolean +} export function featureSpanPx( feature: Feature, - region: { start: number; end: number; reversed?: boolean }, + region: MinimalRegion, bpPerPx: number, -): [number, number] { +) { return bpSpanPx(feature.get('start'), feature.get('end'), region, bpPerPx) } export function bpSpanPx( leftBp: number, rightBp: number, - region: { start: number; end: number; reversed?: boolean }, + region: MinimalRegion, bpPerPx: number, -): [number, number] { +) { const start = bpToPx(leftBp, region, bpPerPx) const end = bpToPx(rightBp, region, bpPerPx) - return region.reversed ? [end, start] : [start, end] + return region.reversed ? ([end, start] as const) : ([start, end] as const) } // do an array map of an iterable @@ -646,8 +651,7 @@ export function iterMap( /** * Returns the index of the last element in the array where predicate is true, - * and -1 otherwise. - * Based on https://stackoverflow.com/a/53187807 + * and -1 otherwise. Based on https://stackoverflow.com/a/53187807 * * @param array - The source array to search in * @@ -660,7 +664,7 @@ export function iterMap( export function findLastIndex( array: T[], predicate: (value: T, index: number, obj: T[]) => boolean, -): number { +) { let l = array.length while (l--) { if (predicate(array[l]!, l, array)) { @@ -673,7 +677,7 @@ export function findLastIndex( export function findLast( array: T[], predicate: (value: T, index: number, obj: T[]) => boolean, -): T | undefined { +) { let l = array.length while (l--) { if (predicate(array[l]!, l, array)) { diff --git a/plugins/alignments/src/BamAdapter/BamAdapter.ts b/plugins/alignments/src/BamAdapter/BamAdapter.ts index 9d7ca34451..25f28141e4 100644 --- a/plugins/alignments/src/BamAdapter/BamAdapter.ts +++ b/plugins/alignments/src/BamAdapter/BamAdapter.ts @@ -1,4 +1,7 @@ import { BamFile } from '@gmod/bam' +import { toArray } from 'rxjs/operators' +import { firstValueFrom } from 'rxjs' +// jbrowse import { BaseFeatureDataAdapter, BaseOptions, @@ -7,8 +10,6 @@ import { Region } from '@jbrowse/core/util/types' import { bytesForRegions, updateStatus, Feature } from '@jbrowse/core/util' import { openLocation } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' -import { toArray } from 'rxjs/operators' -import { firstValueFrom } from 'rxjs' import QuickLRU from '@jbrowse/core/util/QuickLRU' // locals @@ -25,9 +26,7 @@ export default class BamAdapter extends BaseFeatureDataAdapter { private setupP?: Promise
- private featureMap = new QuickLRU({ - maxSize: 5000, - }) + private featureCache = new QuickLRU({ maxSize: 5000 }) private configureP?: Promise<{ bam: BamFile @@ -190,8 +189,6 @@ export default class BamAdapter extends BaseFeatureDataAdapter { readName, } = filterBy || {} - let retrievedFromCache = 0 - let notRetrievedFromCache = 0 for (const record of records) { let ref: string | undefined if (!record.tags.MD) { @@ -223,19 +220,18 @@ export default class BamAdapter extends BaseFeatureDataAdapter { continue } - const ret = this.featureMap.get(`${record.id}`) - + // retrieve a feature from our feature cache if it is available, the + // features in the cache have pre-computed mismatches objects that + // can be re-used across blocks + const ret = this.featureCache.get(`${record.id}`) if (!ret) { const elt = new BamSlightlyLazyFeature(record, this, ref) - this.featureMap.set(`${record.id}`, elt) + this.featureCache.set(`${record.id}`, elt) observer.next(elt) - notRetrievedFromCache++ } else { - retrievedFromCache++ observer.next(ret) } } - // console.log({ retrievedFromCache, notRetrievedFromCache }) observer.complete() }) }, signal) diff --git a/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts b/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts index ae3d8ecc0d..c85a6ccaa2 100644 --- a/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts +++ b/plugins/alignments/src/BamAdapter/BamSlightlyLazyFeature.ts @@ -7,6 +7,7 @@ import { BamRecord } from '@gmod/bam' // locals import { getMismatches } from '../MismatchParser' import BamAdapter from './BamAdapter' +import { cacheGetter } from '../shared/util' export default class BamSlightlyLazyFeature implements Feature { // uses parameter properties to automatically create fields on the class @@ -30,11 +31,15 @@ export default class BamSlightlyLazyFeature implements Feature { ) } + get qual() { + return this.record.qual?.join(' ') + } + get(field: string): any { return field === 'mismatches' ? this.mismatches : field === 'qual' - ? this.record.qual?.join(' ') + ? this.qual : this.fields[field] } @@ -74,27 +79,11 @@ export default class BamSlightlyLazyFeature implements Feature { } toJSON(): SimpleFeatureSerialized { - return this.fields - } -} - -function cacheGetter(ctor: { prototype: T }, prop: keyof T): void { - const desc = Object.getOwnPropertyDescriptor(ctor.prototype, prop) - if (!desc) { - throw new Error('t1') - } - - const getter = desc.get - if (!getter) { - throw new Error('t2') + return { + ...this.fields, + qual: this.qual, + } } - Object.defineProperty(ctor.prototype, prop, { - get() { - const ret = getter.call(this) - Object.defineProperty(this, prop, { value: ret }) - return ret - }, - }) } cacheGetter(BamSlightlyLazyFeature, 'fields') diff --git a/plugins/alignments/src/BamAdapter/__snapshots__/BamAdapter.test.ts.snap b/plugins/alignments/src/BamAdapter/__snapshots__/BamAdapter.test.ts.snap index c320e57051..9e4f0466cf 100644 --- a/plugins/alignments/src/BamAdapter/__snapshots__/BamAdapter.test.ts.snap +++ b/plugins/alignments/src/BamAdapter/__snapshots__/BamAdapter.test.ts.snap @@ -11,6 +11,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "TACACTGGTTCGGAGACGGTTCGTGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATGAGCGGCCGCTGACCGGCGGCACGACTAATATAGTGCAAGA", @@ -39,6 +40,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "ACACTGGTTCGGAGACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCGGCCCCTGTCCGGCGGCACGAATAATATAGTGCAAGAA", @@ -67,6 +69,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "CTGGTTCGGAGACGGTTCATGACGACCGCGCTATATGTCGGCATCTGCGTCGCATGAGCGGCCGCTGTCCGGCGGCTCGAATAATATAGTGCAAGAAAAA", @@ -95,6 +98,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "AGACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACT", @@ -123,6 +127,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "GACGGTTCATGACGAGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCCGCCGCTGTCCGACGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTA", @@ -151,6 +156,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "TTCATGACGAGCGCGCTATATGACGGCATCTGCGCCGCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTT", @@ -179,6 +185,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "TCATGACGAGCGCGCTATATGTCGGCATCTGCGCCGCATCAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTA", @@ -207,6 +214,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "AGCGCGCTATATGTCGGCATCTGCGCCCCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTATATATGAT", @@ -235,6 +243,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "CCCATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCTAAGACTACGGTTATATATGATGGAACGGCCCTCACAGCATTCTCACA", @@ -263,6 +272,7 @@ exports[`adapter can fetch features from volvox.bam 1`] = ` "next_ref": undefined, "next_segment_position": undefined, "pair_orientation": undefined, + "qual": "17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17 17", "refName": "ctgA", "score": 37, "seq": "CATGAGCGGCCGCTGTCCGGCGGCACGAATAATATAGTGCAAGAAAAACCGAAGACTACGGTTATATATGATGGAACGGCCCTCACAGCATTGTAACAGG", diff --git a/plugins/alignments/src/CramAdapter/CramAdapter.ts b/plugins/alignments/src/CramAdapter/CramAdapter.ts index 8bc8debcf4..a4f9397c1b 100644 --- a/plugins/alignments/src/CramAdapter/CramAdapter.ts +++ b/plugins/alignments/src/CramAdapter/CramAdapter.ts @@ -1,4 +1,7 @@ import { CraiIndex, IndexedCramFile, CramRecord } from '@gmod/cram' +import { toArray } from 'rxjs/operators' +import { firstValueFrom } from 'rxjs' +// jbrowse import { BaseFeatureDataAdapter, BaseOptions, @@ -8,8 +11,7 @@ import type { Region, Feature } from '@jbrowse/core/util' import { checkAbortSignal, updateStatus, toLocale } from '@jbrowse/core/util' import { openLocation } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' -import { toArray } from 'rxjs/operators' -import { firstValueFrom } from 'rxjs' +import QuickLRU from '@jbrowse/core/util/QuickLRU' // locals import CramSlightlyLazyFeature from './CramSlightlyLazyFeature' @@ -35,6 +37,8 @@ export default class CramAdapter extends BaseFeatureDataAdapter { sequenceAdapter: BaseSequenceAdapter }> + private featureCache = new QuickLRU({ maxSize: 5000 }) + // maps a refname to an id private seqIdToRefName: string[] | undefined @@ -266,7 +270,17 @@ export default class CramAdapter extends BaseFeatureDataAdapter { if (readName && record.readName !== readName) { continue } - observer.next(this.cramRecordToFeature(record)) + // retrieve a feature from our feature cache if it is available, the + // features in the cache have pre-computed mismatches objects that + // can be re-used across blocks + const ret = this.featureCache.get(`${record.uniqueId}`) + if (!ret) { + const elt = this.cramRecordToFeature(record) + this.featureCache.set(`${record.uniqueId}`, elt) + observer.next(elt) + } else { + observer.next(ret) + } } observer.complete() diff --git a/plugins/alignments/src/CramAdapter/CramSlightlyLazyFeature.ts b/plugins/alignments/src/CramAdapter/CramSlightlyLazyFeature.ts index 29146e3b0f..354ff92b3f 100644 --- a/plugins/alignments/src/CramAdapter/CramSlightlyLazyFeature.ts +++ b/plugins/alignments/src/CramAdapter/CramSlightlyLazyFeature.ts @@ -6,6 +6,7 @@ import CramAdapter from './CramAdapter' import { readFeaturesToCIGAR, readFeaturesToMismatches } from './util' import { parseCigar } from '../MismatchParser' import { mdToMismatches } from '../MismatchParser/mdToMismatches' +import { cacheGetter } from '../shared/util' export default class CramSlightlyLazyFeature implements Feature { // uses parameter properties to automatically create fields on the class @@ -173,22 +174,6 @@ export default class CramSlightlyLazyFeature implements Feature { } } -function cacheGetter(ctor: { prototype: T }, prop: keyof T): void { - const desc = Object.getOwnPropertyDescriptor(ctor.prototype, prop) - if (!desc) { - throw new Error('t1') - } - - const getter = desc.get - if (!getter) { - throw new Error('t2') - } - Object.defineProperty(ctor.prototype, prop, { - get() { - const ret = getter.call(this) - Object.defineProperty(this, prop, { value: ret }) - return ret - }, - }) -} cacheGetter(CramSlightlyLazyFeature, 'fields') +cacheGetter(CramSlightlyLazyFeature, 'CIGAR') +cacheGetter(CramSlightlyLazyFeature, 'mismatches') diff --git a/plugins/alignments/src/PileupRenderer/makeImageData.ts b/plugins/alignments/src/PileupRenderer/makeImageData.ts index 4bffdbe2a6..dddf5d5258 100644 --- a/plugins/alignments/src/PileupRenderer/makeImageData.ts +++ b/plugins/alignments/src/PileupRenderer/makeImageData.ts @@ -12,7 +12,6 @@ import { import { renderAlignment } from './renderAlignment' import { renderMismatches } from './renderMismatches' import { renderSoftClipping } from './renderSoftClipping' -import { parseCigar } from '../MismatchParser' export type RenderArgsWithColor = RenderArgsDeserializedWithFeaturesAndLayout @@ -50,7 +49,6 @@ export function makeImageData({ const drawSNPsMuted = shouldDrawSNPsMuted(colorBy?.type) const drawIndels = shouldDrawIndels() for (const feat of layoutRecords) { - const cigarOps = parseCigar(feat.feature.get('CIGAR')) renderAlignment({ ctx, feat, @@ -61,7 +59,6 @@ export function makeImageData({ charWidth, charHeight, canvasWidth, - cigarOps, }) renderMismatches({ ctx, @@ -87,7 +84,6 @@ export function makeImageData({ config, theme, canvasWidth, - cigarOps, }) } } diff --git a/plugins/alignments/src/PileupRenderer/renderAlignment.ts b/plugins/alignments/src/PileupRenderer/renderAlignment.ts index 33fceff9cf..0795f10a11 100644 --- a/plugins/alignments/src/PileupRenderer/renderAlignment.ts +++ b/plugins/alignments/src/PileupRenderer/renderAlignment.ts @@ -7,6 +7,7 @@ import { renderPerBaseLettering } from './renderPerBaseLettering' import { renderModifications } from './renderModifications' import { RenderArgsWithColor } from './makeImageData' import { renderMethylation } from './renderMethylation' +import { parseCigar } from '../MismatchParser' export function renderAlignment({ ctx, @@ -18,7 +19,6 @@ export function renderAlignment({ charHeight, defaultColor, canvasWidth, - cigarOps, }: { ctx: CanvasRenderingContext2D feat: LayoutFeature @@ -29,7 +29,6 @@ export function renderAlignment({ charHeight: number defaultColor: boolean canvasWidth: number - cigarOps: string[] }) { const { config, bpPerPx, regions, colorBy, colorTagMap = {} } = renderArgs const { tag = '', type: colorType = '' } = colorBy || {} @@ -45,12 +44,13 @@ export function renderAlignment({ colorTagMap, }) - renderAlignmentShape({ cigarOps, ctx, feat, renderArgs }) + renderAlignmentShape({ ctx, feat, renderArgs }) // second pass for color types that render per-base things that go over the // existing drawing switch (colorType) { - case 'perBaseQuality': + case 'perBaseQuality': { + const cigarOps = parseCigar(feature.get('CIGAR')) renderPerBaseQuality({ ctx, feat, @@ -60,8 +60,10 @@ export function renderAlignment({ cigarOps, }) break + } - case 'perBaseLettering': + case 'perBaseLettering': { + const cigarOps = parseCigar(feature.get('CIGAR')) renderPerBaseLettering({ ctx, feat, @@ -75,8 +77,10 @@ export function renderAlignment({ cigarOps, }) break + } - case 'modifications': + case 'modifications': { + const cigarOps = parseCigar(feature.get('CIGAR')) renderModifications({ ctx, feat, @@ -87,8 +91,10 @@ export function renderAlignment({ cigarOps, }) break + } - case 'methylation': + case 'methylation': { + const cigarOps = parseCigar(feature.get('CIGAR')) renderMethylation({ ctx, feat, @@ -99,5 +105,6 @@ export function renderAlignment({ cigarOps, }) break + } } } diff --git a/plugins/alignments/src/PileupRenderer/renderAlignmentShape.ts b/plugins/alignments/src/PileupRenderer/renderAlignmentShape.ts index 70dce244d0..5ffe92103b 100644 --- a/plugins/alignments/src/PileupRenderer/renderAlignmentShape.ts +++ b/plugins/alignments/src/PileupRenderer/renderAlignmentShape.ts @@ -1,17 +1,16 @@ import { bpSpanPx } from '@jbrowse/core/util' import { RenderArgsDeserialized } from './PileupRenderer' import { LayoutFeature } from './util' +import { parseCigar } from '../MismatchParser' export function renderAlignmentShape({ ctx, feat, renderArgs, - cigarOps, }: { ctx: CanvasRenderingContext2D feat: LayoutFeature renderArgs: RenderArgsDeserialized - cigarOps: string[] }) { const { regions, bpPerPx } = renderArgs const { heightPx, topPx, feature } = feat @@ -22,7 +21,8 @@ export function renderAlignmentShape({ const flip = region.reversed ? -1 : 1 const strand = feature.get('strand') * flip const renderChevrons = bpPerPx < 10 && heightPx > 5 - if (CIGAR) { + if (CIGAR.includes('N')) { + const cigarOps = parseCigar(CIGAR) if (strand === 1) { let drawLen = 0 let drawStart = s diff --git a/plugins/alignments/src/PileupRenderer/renderMismatches.ts b/plugins/alignments/src/PileupRenderer/renderMismatches.ts index 046c4221b1..673dc2d027 100644 --- a/plugins/alignments/src/PileupRenderer/renderMismatches.ts +++ b/plugins/alignments/src/PileupRenderer/renderMismatches.ts @@ -57,7 +57,7 @@ export function renderMismatches({ const mlen = mismatch.length const mbase = mismatch.base const [leftPx, rightPx] = bpSpanPx(mstart, mstart + mlen, region, bpPerPx) - const widthPx = Math.max(minSubfeatureWidth, Math.abs(leftPx - rightPx)) + const widthPx = Math.max(minSubfeatureWidth, rightPx - leftPx) if (mismatch.type === 'mismatch') { if (!drawSNPsMuted) { const baseColor = colorForBase[mismatch.base] || '#888' @@ -113,7 +113,6 @@ export function renderMismatches({ ctx.fillText(txt, (leftPx + rightPx) / 2 - rwidth / 2, topPx + heightPx) } } else if (mismatch.type === 'insertion' && drawIndels) { - ctx.fillStyle = 'purple' const pos = leftPx + extraHorizontallyFlippedOffset const len = +mismatch.base || mismatch.length const insW = Math.max(0, Math.min(1.2, 1 / bpPerPx)) diff --git a/plugins/alignments/src/PileupRenderer/renderSoftClipping.ts b/plugins/alignments/src/PileupRenderer/renderSoftClipping.ts index dbae53d0b3..dd462cc291 100644 --- a/plugins/alignments/src/PileupRenderer/renderSoftClipping.ts +++ b/plugins/alignments/src/PileupRenderer/renderSoftClipping.ts @@ -9,6 +9,7 @@ import { Theme } from '@mui/material' import { RenderArgsDeserializedWithFeaturesAndLayout } from './PileupRenderer' import { fillRect, getCharWidthHeight, LayoutFeature } from './util' import { Mismatch } from '../shared/types' +import { parseCigar } from '../MismatchParser' export function renderSoftClipping({ ctx, @@ -18,7 +19,6 @@ export function renderSoftClipping({ theme, colorForBase, canvasWidth, - cigarOps, }: { ctx: CanvasRenderingContext2D feat: LayoutFeature @@ -27,7 +27,6 @@ export function renderSoftClipping({ colorForBase: Record theme: Theme canvasWidth: number - cigarOps: string[] }) { const { feature, topPx, heightPx } = feat const { regions, bpPerPx } = renderArgs @@ -45,6 +44,8 @@ export function renderSoftClipping({ const heightLim = charHeight - 2 let seqOffset = 0 let refOffset = 0 + const CIGAR = feature.get('CIGAR') + const cigarOps = parseCigar(CIGAR) for (let i = 0; i < cigarOps.length; i += 2) { const op = cigarOps[i + 1]! const len = +cigarOps[i]! diff --git a/plugins/alignments/src/SNPCoverageAdapter/generateCoverageBins.ts b/plugins/alignments/src/SNPCoverageAdapter/generateCoverageBins.ts index 50e71717b4..a55eed4dc0 100644 --- a/plugins/alignments/src/SNPCoverageAdapter/generateCoverageBins.ts +++ b/plugins/alignments/src/SNPCoverageAdapter/generateCoverageBins.ts @@ -118,12 +118,6 @@ function processSNPs({ const bin = bins[epos]! const { base, type } = mismatch const interbase = isInterbase(type) - if (!interbase) { - // bin.ref.entryDepth++ - // bin.ref[fstrand]++ - } else { - inc(bin, fstrand, 'noncov', type) - } if (type === 'deletion' || type === 'skip') { inc(bin, fstrand, 'delskips', type) @@ -132,6 +126,8 @@ function processSNPs({ inc(bin, fstrand, 'snps', base) bin.ref.entryDepth-- bin.ref[fstrand]-- + } else { + inc(bin, fstrand, 'noncov', type) } } } diff --git a/plugins/alignments/src/shared/util.ts b/plugins/alignments/src/shared/util.ts index 2e784e5aff..27311ff264 100644 --- a/plugins/alignments/src/shared/util.ts +++ b/plugins/alignments/src/shared/util.ts @@ -22,3 +22,15 @@ export const defaultFilterFlags = { flagInclude: 0, flagExclude: 1540, } + +export function cacheGetter(ctor: { prototype: T }, prop: keyof T): void { + const desc = Object.getOwnPropertyDescriptor(ctor.prototype, prop)! + const getter = desc.get! + Object.defineProperty(ctor.prototype, prop, { + get() { + const ret = getter.call(this) + Object.defineProperty(this, prop, { value: ret }) + return ret + }, + }) +} diff --git a/test_data/config_demo.json b/test_data/config_demo.json index 482e9a6b98..74b67ba87d 100644 --- a/test_data/config_demo.json +++ b/test_data/config_demo.json @@ -4233,6 +4233,7 @@ { "type": "AlignmentsTrack", "name": "Paired-end stranded RNA-seq (BAM,XS)", + "category": ["RNA-seq"], "metadata": { "source": "From sample files listed at https://rseqc.sourceforge.net/" }, @@ -4524,6 +4525,7 @@ "type": "AlignmentsTrack", "trackId": "Pairend_StrandSpecific_51mer_Human_hg19.chr1.cram", "name": "Paired-end stranded RNA-seq (CRAM,XS)", + "category": ["RNA-seq"], "metadata": { "source": "From sample files listed at https://rseqc.sourceforge.net/" }, @@ -4559,6 +4561,7 @@ "type": "AlignmentsTrack", "trackId": "Pairend_StrandSpecific_51mer_Human_hg19.chr1.TS", "name": "Paired-end stranded RNA-seq (CRAM,TS)", + "category": ["RNA-seq"], "adapter": { "type": "CramAdapter", "cramLocation": { @@ -4591,6 +4594,7 @@ "type": "AlignmentsTrack", "trackId": "Pairend_StrandSpecific_51mer_Human_hg19.chr1.TS.bam", "name": "Paired-end stranded RNA-seq (BAM,TS)", + "category": ["RNA-seq"], "adapter": { "type": "BamAdapter", "bamLocation": {