Skip to content

Commit

Permalink
Webservice (#1842)
Browse files Browse the repository at this point in the history 
* Add support for webservice that serves annotation and variant file slices for a genomic range.  Fixes #1666
  • Loading branch information
@jrobinso
jrobinso authored Jul 10, 2024
1 parent d12940e commit bc65cdd
Show file tree
Hide file tree
Showing 7 changed files with 736 additions and 211 deletions.
94 changes: 94 additions & 0 deletions dev/annotation/webservice.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>igv.js</title>
</head>

<body>

<p>

<h1>Webservice example</h1>

<p>
This example illustrates use of a web service for loading bed and vcf tracks. Tracks are defined as follows

<pre>
{
sourceType: "service",
format: "bed",
name: "Bed features",
url: "http://localhost:8080/basic_feature_3_columns.bed?chr=$CHR&start=$STARTend=$END",
headerURL: "http://localhost:8080/basic_feature_3_columns.bed?class=header",
seqnamesURL: "http://localhost:8080/basic_feature_3_columns.bed?class=seqnames"
},
{
sourceType: "service",
format: "vcf",
name: "VCF variants",
url: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?chr=$CHR&start=$STARTend=$END",
headerURL: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?class=header",
seqnamesURL: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?class=seqnames"
}
</pre>

Properties
<ul>
<li><b>sourceType: "service"</b> - indicates the data source is a web service, as opposed to a file</li>
<li><b>url</b> - A url template for the features. The symbols $CHR, $START, and $END are substituted for genomic coordinates at runtime. </li>
<li><b>headerURL</b> - A url to fetch the header portion of the file. It is required for VCF files, optional for BED and GFF formats. </li>
<li><b>seqnamesURL</b> - A url to featch a comma delimited list of sequence names for the resource. Optional, but useful to support chromosome
aliasing (e.g. 1 == chr1). It should return a comma delimited list of sequence names for the track file. If omitted the sequence names
must match the names as defined by the genomic reference. </li>
</ul>

</p>

<p>To run this example first start the node webservice "featureService.js" in "test/service"</p>

<div id="igvDiv" style="margin-top: 50px; border:1px solid lightgray"></div>

<script type="module">

import igv from "../../js/index.js"

var options =
{
genome: "hg19",
locus: "chr1",
tracks:
[
{
sourceType: "service",
format: "bed",
name: "Bed features",
url: "http://localhost:8080/basic_feature_3_columns.bed?chr=$CHR&start=$STARTend=$END",
headerURL: "http://localhost:8080/basic_feature_3_columns.bed?class=header",
seqnamesURL: "http://localhost:8080/basic_feature_3_columns.bed?class=seqnames"
},
{
sourceType: "service",
format: "vcf",
name: "VCF variants",
url: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?chr=$CHR&start=$STARTend=$END",
headerURL: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?class=header",
seqnamesURL: "http://localhost:8080/SKBR3_Sniffles_variants_tra.vcf?class=seqnames"

}
]
}

var igvDiv = document.getElementById("igvDiv")

igv.createBrowser(igvDiv, options)
.then(function (browser) {
console.log("Created IGV browser")
})


</script>

</body>

</html>
207 changes: 127 additions & 80 deletions js/feature/featureFileReader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ class FeatureFileReader {
} else if (this.dataURI) {
this.indexed = false
return this.loadFeaturesFromDataURI()
} else if ("service" === this.config.sourceType) {
return this.loadFeaturesFromService(chr, start, end)
} else {
this.indexed = false
return this.loadFeaturesNoIndex()
Expand All @@ -123,62 +125,76 @@ class FeatureFileReader {
if (this.dataURI) {
await this.loadFeaturesFromDataURI(this.dataURI)
return this.header
} else {

if (this.config.indexURL) {
const index = await this.getIndex()
if (!index) {
// Note - it should be impossible to get here
throw new Error("Unable to load index: " + this.config.indexURL)
}
this.sequenceNames = new Set(index.sequenceNames)
} else if (this.config.indexURL) {
const index = await this.getIndex()
if (!index) {
// Note - it should be impossible to get here
throw new Error("Unable to load index: " + this.config.indexURL)
}
this.sequenceNames = new Set(index.sequenceNames)

let dataWrapper
if (index.tabix) {
this._blockLoader = new BGZBlockLoader(this.config)
dataWrapper = new BGZLineReader(this.config)
} else {
// Tribble
const maxSize = Object.values(index.chrIndex)
.flatMap(chr => chr.blocks)
.map(block => block.max)
.reduce((previous, current) =>
Math.min(previous, current), Number.MAX_SAFE_INTEGER)

const options = buildOptions(this.config, {bgz: index.tabix, range: {start: 0, size: maxSize}})
const data = await igvxhr.loadString(this.config.url, options)
dataWrapper = getDataWrapper(data)
}
let dataWrapper
if (index.tabix) {
this._blockLoader = new BGZBlockLoader(this.config)
dataWrapper = new BGZLineReader(this.config)
} else {
// Tribble
const maxSize = Object.values(index.chrIndex)
.flatMap(chr => chr.blocks)
.map(block => block.max)
.reduce((previous, current) =>
Math.min(previous, current), Number.MAX_SAFE_INTEGER)

const options = buildOptions(this.config, {bgz: index.tabix, range: {start: 0, size: maxSize}})
const data = await igvxhr.loadString(this.config.url, options)
dataWrapper = getDataWrapper(data)
}

this.header = await this.parser.parseHeader(dataWrapper)
return this.header

} else if ("service" === this.config.sourceType) {
if (this.config.seqnamesURL) {
// Side effect, a bit ugly
const options = buildOptions(this.config, {})
const seqnameString = await igvxhr.loadString(this.config.seqnamesURL, options)
if (seqnameString) {
this.sequenceNames = new Set(seqnameString.split(",").map(sn => sn.trim()).filter(sn => sn))
}
}
if (this.config.headerURL) {
const options = buildOptions(this.config, {})
const data = await igvxhr.loadString(this.config.headerURL, options)
const dataWrapper = getDataWrapper(data)
this.header = await this.parser.parseHeader(dataWrapper) // Cache header, might be needed to parse features
return this.header
}

} else {
// If this is a non-indexed file we will load all features in advance
const options = buildOptions(this.config)
let data = await igvxhr.loadByteArray(this.config.url, options)

// If the data size is < max string length decode entire string with TextDecoder. This is much faster
// than decoding by line
if(data.length < MAX_STRING_LENGTH) {
data = new TextDecoder().decode(data)
}
} else {
// If this is a non-indexed file we will load all features in advance
const options = buildOptions(this.config)
let data = await igvxhr.loadByteArray(this.config.url, options)

// If the data size is < max string length decode entire string with TextDecoder. This is much faster
// than decoding by line
if (data.length < MAX_STRING_LENGTH) {
data = new TextDecoder().decode(data)
}

let dataWrapper = getDataWrapper(data)
this.header = await this.parser.parseHeader(dataWrapper)
let dataWrapper = getDataWrapper(data)
this.header = await this.parser.parseHeader(dataWrapper)

// Reset data wrapper and parse features
dataWrapper = getDataWrapper(data)
this.features = await this.parser.parseFeatures(dataWrapper) // cache features
// Reset data wrapper and parse features
dataWrapper = getDataWrapper(data)
this.features = await this.parser.parseFeatures(dataWrapper) // cache features

// Extract chromosome names
this.sequenceNames = new Set()
for (let f of this.features) this.sequenceNames.add(f.chr)
// Extract chromosome names
this.sequenceNames = new Set()
for (let f of this.features) this.sequenceNames.add(f.chr)

return this.header
}
return this.header
}

}


Expand Down Expand Up @@ -217,7 +233,8 @@ class FeatureFileReader {
this.header = await this.parser.parseHeader(dataWrapper)
}
const dataWrapper = getDataWrapper(data)
const features = await this.parser.parseFeatures(dataWrapper) // <= PARSING DONE HERE
const features = []
await this._parse(features, dataWrapper) // <= PARSING DONE HERE
return features
}
}
Expand Down Expand Up @@ -256,46 +273,75 @@ class FeatureFileReader {

const slicedData = chunk.minv.offset ? inflated.slice(chunk.minv.offset) : inflated
const dataWrapper = getDataWrapper(slicedData)
let slicedFeatures = await parser.parseFeatures(dataWrapper)
await this._parse(allFeatures, dataWrapper, chr, end, start)

}
allFeatures.sort(function (a, b) {
return a.start - b.start
})

// Filter psuedo-features (e.g. created mates for VCF SV records)
slicedFeatures = slicedFeatures.filter(f => f._f === undefined)
return allFeatures
}
}

// Filter features not in requested range.
let inInterval = false
for (let i = 0; i < slicedFeatures.length; i++) {
const f = slicedFeatures[i]
async loadFeaturesFromService(chr, start, end) {

if (f.chr !== chr) {
if (allFeatures.length === 0) {
continue //adjacent chr to the left
} else {
break //adjacent chr to the right
}
}
if (f.start > end) {
allFeatures.push(f) // First feature beyond interval
break
let url
if (typeof this.config.url === 'function') {
url = this.config.url({chr, start, end})
} else {
url = this.config.url
.replace("$CHR", chr)
.replace("$START", start)
.replace("$END", end)
}
const options = buildOptions(this.config) // Adds oauth token, if any
const data = await igvxhr.loadString(url, options)
const dataWrapper = getDataWrapper(data)
const features = []
await this._parse(features, dataWrapper) // <= PARSING DONE HERE
return features

}

async _parse(allFeatures, dataWrapper, chr, end, start) {

let features = await this.parser.parseFeatures(dataWrapper)

// Filter psuedo-features (e.g. created mates for VCF SV records) TODO why?
//slicedFeatures = slicedFeatures.filter(f => f._f === undefined)

// Filter features not in requested range.
if (undefined === chr) {
for (let f of features) allFeatures.push(f) // Don't use spread operator !!! slicedFeatures might be very large
} else {
let inInterval = false
for (let i = 0; i < features.length; i++) {
const f = features[i]
if (f.chr !== chr) {
if (allFeatures.length === 0) {
continue //adjacent chr to the left
} else {
break //adjacent chr to the right
}
if (f.end >= start && f.start <= end) {
if (!inInterval) {
inInterval = true
if (i > 0) {
allFeatures.push(slicedFeatures[i - 1])
} else {
// TODO -- get block before this one for first feature;
}
}
if (f.start > end) {
allFeatures.push(f) // First feature beyond interval
break
}
if (f.end >= start && f.start <= end) {
// All this to grab first feature before start of interval. Needed for some track renderers, like line plot
if (!inInterval) {
inInterval = true
if (i > 0) {
allFeatures.push(features[i - 1])
} else {
// TODO -- get block before this one for first feature;
}
allFeatures.push(f)
}
allFeatures.push(f)
}

}
allFeatures.sort(function (a, b) {
return a.start - b.start
})

return allFeatures
}
}

Expand Down Expand Up @@ -332,8 +378,9 @@ class FeatureFileReader {
}

dataWrapper = getDataWrapper(plain)
this.features = await this.parser.parseFeatures(dataWrapper)
return this.features
const features = []
await this._parse().parseFeatures(features, dataWrapper)
return features
}
}

Expand Down
9 changes: 6 additions & 3 deletions js/feature/textFeatureSource.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class TextFeatureSource extends BaseFeatureSource {
} else if (config.sourceType === 'custom') {
this.reader = new CustomServiceReader(config.source)
this.queryable = false !== config.source.queryable
} else if ('service' === config.sourceType) {
this.reader = new FeatureFileReader(config, genome)
this.queryable = true
} else {
// File of some type (i.e. not a webservice)
this.reader = new FeatureFileReader(config, genome)
Expand Down Expand Up @@ -229,7 +232,7 @@ class TextFeatureSource extends BaseFeatureSource {
if (!this.featureMap) {
this.featureMap = new Map()
}
const searchableFields = config.searchableFields || ["name", "transcript_id", "gene_id", "gene_name", "id" ]
const searchableFields = config.searchableFields || ["name", "transcript_id", "gene_id", "gene_name", "id"]
for (let feature of featureList) {
for (let field of searchableFields) {
let key
Expand All @@ -239,9 +242,9 @@ class TextFeatureSource extends BaseFeatureSource {
if (key) {
key = key.replaceAll(' ', '+').toUpperCase()
// If feature is already present keep largest one
if(this.featureMap.has(key)) {
if (this.featureMap.has(key)) {
const f2 = this.featureMap.get(key)
if(feature.end - feature.start < f2.end - f2.start) {
if (feature.end - feature.start < f2.end - f2.start) {
continue
}
}
Expand Down
Loading

0 comments on commit bc65cdd

Please sign in to comment.