From c4165a222b3546098814a1b4abbc47f0ea878cfa Mon Sep 17 00:00:00 2001 From: Alex Tran Date: Thu, 3 Mar 2022 17:08:46 -0800 Subject: [PATCH 1/2] WebXRImageTracking feature and engine changes to support native image loading and tracking in BabylonNative --- src/Engines/ICanvas.ts | 5 +++ src/Engines/Native/nativeInterfaces.ts | 2 +- src/Engines/engine.ts | 27 +++++++++++++ src/Engines/nativeEngine.ts | 29 ++++++++++++++ src/XR/features/WebXRImageTracking.ts | 53 +++++++++----------------- src/XR/native/nativeXRFrame.ts | 2 + 6 files changed, 83 insertions(+), 35 deletions(-) diff --git a/src/Engines/ICanvas.ts b/src/Engines/ICanvas.ts index 239eedd14b2..38fc837fb57 100644 --- a/src/Engines/ICanvas.ts +++ b/src/Engines/ICanvas.ts @@ -37,6 +37,11 @@ export interface IImage { */ onload: ((this: GlobalEventHandlers, ev: Event) => any) | null; + /** + * Error callback. + */ + onerror: ((this: GlobalEventHandlers, ev: Event) => any) | null; + /** * Image source. */ diff --git a/src/Engines/Native/nativeInterfaces.ts b/src/Engines/Native/nativeInterfaces.ts index 3b95ef04203..315eb574f4f 100644 --- a/src/Engines/Native/nativeInterfaces.ts +++ b/src/Engines/Native/nativeInterfaces.ts @@ -35,7 +35,7 @@ export interface INativeEngine { copyTexture(desination: Nullable, source: Nullable): void; deleteTexture(texture: Nullable): void; - createImageBitmap(data: ArrayBufferView): ImageBitmap; + createImageBitmap(data: ArrayBufferView | IImage): ImageBitmap; resizeImageBitmap(image: ImageBitmap, bufferWidth: number, bufferHeight: number): Uint8Array; createFrameBuffer(texture: WebGLTexture, width: number, height: number, format: number, generateStencilBuffer: boolean, generateDepthBuffer: boolean, generateMips: boolean): WebGLFramebuffer; diff --git a/src/Engines/engine.ts b/src/Engines/engine.ts index 66d4d3971e4..ec65814e449 100755 --- a/src/Engines/engine.ts +++ b/src/Engines/engine.ts @@ -291,6 +291,33 @@ export class Engine extends ThinEngine { return EngineStore.LastCreatedScene; } + /** @hidden */ + /** + * Engine abstraction for loading and creating an image bitmap from a given source string. + * @param imageSource source to load the image from. + * @param options An object that sets options for the image's extraction. + * @returns ImageBitmap. + */ + public createImageBitmapFromSource(imageSource: string, options?: ImageBitmapOptions): Promise { + const promise = new Promise((resolve, reject) => { + const image = new Image(); + image.onload = () => { + image.decode().then(() => { + this.createImageBitmap(image, options).then((imageBitmap) => { + resolve(imageBitmap); + }); + }); + }; + image.onerror = () => { + reject(`Error loading image ${image.src}`); + }; + + image.src = imageSource; + }); + + return promise; + } + /** * Engine abstraction for createImageBitmap * @param image source for image diff --git a/src/Engines/nativeEngine.ts b/src/Engines/nativeEngine.ts index f21216e4609..c754a031175 100644 --- a/src/Engines/nativeEngine.ts +++ b/src/Engines/nativeEngine.ts @@ -2260,6 +2260,35 @@ export class NativeEngine extends Engine { } } + /** @hidden */ + /** + * Engine abstraction for loading and creating an image bitmap from a given source string. + * @param imageSource source to load the image from. + * @param options An object that sets options for the image's extraction. + * @returns ImageBitmap + */ + public createImageBitmapFromSource(imageSource: string, options?: ImageBitmapOptions): Promise { + const promise = new Promise((resolve, reject) => { + const image = this.createCanvasImage(); + image.onload = () => { + const imageBitmap = this._engine.createImageBitmap(image); + if (imageBitmap) { + resolve(imageBitmap); + return; + } else { + reject (`Error loading image ${image.src}`); + } + }; + image.onerror = () => { + reject(`Error loading image ${image.src}`); + }; + + image.src = imageSource; + }); + + return promise; + } + /** * Engine abstraction for createImageBitmap * @param image source for image diff --git a/src/XR/features/WebXRImageTracking.ts b/src/XR/features/WebXRImageTracking.ts index 9d972e2ca8f..0fc41e97c5d 100644 --- a/src/XR/features/WebXRImageTracking.ts +++ b/src/XR/features/WebXRImageTracking.ts @@ -3,7 +3,6 @@ import { WebXRSessionManager } from "../webXRSessionManager"; import { Observable } from "../../Misc/observable"; import { WebXRAbstractFeature } from "./WebXRAbstractFeature"; import { Matrix } from "../../Maths/math.vector"; -import { Tools } from "../../Misc/tools"; import { Nullable } from "../../types"; declare const XRImageTrackingResult: XRImageTrackingResult; @@ -91,6 +90,7 @@ export class WebXRImageTracking extends WebXRAbstractFeature { */ public onTrackedImageUpdatedObservable: Observable = new Observable(); + private _trackableScoresReceived: boolean = false; private _trackedImages: IWebXRTrackedImage[] = []; private _originalTrackingRequest: XRTrackedImageInit[]; @@ -109,17 +109,6 @@ export class WebXRImageTracking extends WebXRAbstractFeature { ) { super(_xrSessionManager); this.xrNativeFeatureName = "image-tracking"; - if (this.options.images.length === 0) { - // no images provided?... return. - return; - } - if (this._xrSessionManager.session) { - this._init(); - } else { - this._xrSessionManager.onXRSessionInit.addOnce(() => { - this._init(); - }); - } } /** @@ -184,24 +173,7 @@ export class WebXRImageTracking extends WebXRAbstractFeature { } const promises = this.options.images.map((image) => { if (typeof image.src === "string") { - const p = new Promise((resolve, reject) => { - if (typeof image.src === "string") { - const img = new Image(); - img.src = image.src; - img.onload = () => { - img.decode().then(() => { - this._xrSessionManager.scene.getEngine().createImageBitmap(img).then((imageBitmap) => { - resolve(imageBitmap); - }); - }); - }; - img.onerror = () => { - Tools.Error(`Error loading image ${image.src}`); - reject(`Error loading image ${image.src}`); - }; - } - }); - return p; + return this._xrSessionManager.scene.getEngine().createImageBitmapFromSource(image.src); } else { return Promise.resolve(image.src); // resolve is probably unneeded } @@ -225,6 +197,17 @@ export class WebXRImageTracking extends WebXRAbstractFeature { if (!_xrFrame.getImageTrackingResults) { return; } + + // Image tracking scores may be generated a few frames after the XR Session initializes. + // If we haven't received scores yet, then check scores first then bail out if necessary. + if (!this._trackableScoresReceived) { + this._checkScores(); + + if (!this._trackableScoresReceived) { + return; + } + } + const imageTrackedResults = _xrFrame.getImageTrackingResults(); for (const result of imageTrackedResults) { let changed = false; @@ -267,12 +250,12 @@ export class WebXRImageTracking extends WebXRAbstractFeature { } } - private async _init() { - if (!this._xrSessionManager.session.getTrackedImageScores) { + private _checkScores(): void { + if (!this._xrSessionManager.session.getTrackedImageScores || this._trackableScoresReceived) { return; } - // - const imageScores = await this._xrSessionManager.session.getTrackedImageScores(); + + const imageScores = this._xrSessionManager.session.getTrackedImageScores(); // check the scores for all for (let idx = 0; idx < imageScores.length; ++idx) { if (imageScores[idx] == "untrackable") { @@ -289,6 +272,8 @@ export class WebXRImageTracking extends WebXRAbstractFeature { this.onTrackableImageFoundObservable.notifyObservers(imageObject); } } + + this._trackableScoresReceived ||= imageScores.length > 0; } } diff --git a/src/XR/native/nativeXRFrame.ts b/src/XR/native/nativeXRFrame.ts index a45d93ba0de..f3fb589172f 100644 --- a/src/XR/native/nativeXRFrame.ts +++ b/src/XR/native/nativeXRFrame.ts @@ -75,6 +75,8 @@ export class NativeXRFrame implements XRFrame { public get featurePointCloud(): number[] | undefined { return this._nativeImpl.featurePointCloud; } + + public readonly getImageTrackingResults = this._nativeImpl.getImageTrackingResults!.bind(this._nativeImpl); } RegisterNativeTypeAsync("NativeXRFrame", NativeXRFrame); \ No newline at end of file From 236017e5e4c4539067346ac5b85ba3752577b299 Mon Sep 17 00:00:00 2001 From: Alex Tran Date: Fri, 4 Mar 2022 13:56:22 -0800 Subject: [PATCH 2/2] Add error handling --- src/XR/features/WebXRImageTracking.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/XR/features/WebXRImageTracking.ts b/src/XR/features/WebXRImageTracking.ts index 0fc41e97c5d..1eb8fc323a5 100644 --- a/src/XR/features/WebXRImageTracking.ts +++ b/src/XR/features/WebXRImageTracking.ts @@ -4,6 +4,7 @@ import { Observable } from "../../Misc/observable"; import { WebXRAbstractFeature } from "./WebXRAbstractFeature"; import { Matrix } from "../../Maths/math.vector"; import { Nullable } from "../../types"; +import { Tools } from "../../Misc/tools"; declare const XRImageTrackingResult: XRImageTrackingResult; @@ -179,18 +180,24 @@ export class WebXRImageTracking extends WebXRAbstractFeature { } }); - const images = await Promise.all(promises); + try + { + const images = await Promise.all(promises); + + this._originalTrackingRequest = images.map((image, idx) => { + return { + image, + widthInMeters: this.options.images[idx].estimatedRealWorldWidth, + }; + }); - this._originalTrackingRequest = images.map((image, idx) => { return { - image, - widthInMeters: this.options.images[idx].estimatedRealWorldWidth, + trackedImages: this._originalTrackingRequest, }; - }); - - return { - trackedImages: this._originalTrackingRequest, - }; + } catch (ex) { + Tools.Error("Error loading images for tracking, WebXRImageTracking disabled for this session."); + return {}; + } } protected _onXRFrame(_xrFrame: XRFrame) {