Merge pull request #12097 from Alex-MSFT/webxr-marker-tracking-native

WebXRImageTracking and NativeEngine changes to support image tracking in BabylonNative

src/Engines/ICanvas.ts

@@ -37,6 +37,11 @@ export interface IImage {
      */
     onload: ((this: GlobalEventHandlers, ev: Event) => any) | null;
 
+    /**
+     * Error callback.
+     */
+    onerror: ((this: GlobalEventHandlers, ev: Event) => any) | null;
+
     /**
      * Image source.
      */

src/Engines/Native/nativeInterfaces.ts

@@ -35,7 +35,7 @@ export interface INativeEngine {
     copyTexture(desination: Nullable<WebGLTexture>, source: Nullable<WebGLTexture>): void;
     deleteTexture(texture: Nullable<WebGLTexture>): void;
 
-    createImageBitmap(data: ArrayBufferView): ImageBitmap;
+    createImageBitmap(data: ArrayBufferView | IImage): ImageBitmap;
     resizeImageBitmap(image: ImageBitmap, bufferWidth: number, bufferHeight: number): Uint8Array;
 
     createFrameBuffer(texture: WebGLTexture, width: number, height: number, format: number, generateStencilBuffer: boolean, generateDepthBuffer: boolean, generateMips: boolean): WebGLFramebuffer;

src/Engines/engine.ts

@@ -290,6 +290,33 @@ export class Engine extends ThinEngine {
         return EngineStore.LastCreatedScene;
     }
 
+    /** @hidden */
+    /**
+     * Engine abstraction for loading and creating an image bitmap from a given source string.
+     * @param imageSource source to load the image from.
+     * @param options An object that sets options for the image's extraction.
+     * @returns ImageBitmap.
+     */
+    public createImageBitmapFromSource(imageSource: string, options?: ImageBitmapOptions): Promise<ImageBitmap> {
+        const promise = new Promise<ImageBitmap>((resolve, reject) => {
+            const image = new Image();
+            image.onload = () => {
+                image.decode().then(() => {
+                    this.createImageBitmap(image, options).then((imageBitmap) => {
+                        resolve(imageBitmap);
+                    });
+                });
+            };
+            image.onerror = () => {
+                reject(`Error loading image ${image.src}`);
+            };
+
+            image.src = imageSource;
+        });
+
+        return promise;
+    }
+
     /**
      * Engine abstraction for createImageBitmap
      * @param image source for image

src/Engines/nativeEngine.ts

@@ -2260,6 +2260,35 @@ export class NativeEngine extends Engine {
         }
     }
 
+    /** @hidden */
+    /**
+     * Engine abstraction for loading and creating an image bitmap from a given source string.
+     * @param imageSource source to load the image from.
+     * @param options An object that sets options for the image's extraction.
+     * @returns ImageBitmap
+     */
+     public createImageBitmapFromSource(imageSource: string, options?: ImageBitmapOptions): Promise<ImageBitmap> {
+        const promise = new Promise<ImageBitmap>((resolve, reject) => {
+            const image = this.createCanvasImage();
+            image.onload = () => {
+                const imageBitmap = this._engine.createImageBitmap(image);
+                if (imageBitmap) {
+                    resolve(imageBitmap);
+                    return;
+                } else {
+                    reject (`Error loading image ${image.src}`);
+                }
+            };
+            image.onerror = () => {
+                reject(`Error loading image ${image.src}`);
+            };
+
+            image.src = imageSource;
+        });
+
+        return promise;
+    }
+
     /**
      * Engine abstraction for createImageBitmap
      * @param image source for image

src/XR/features/WebXRImageTracking.ts

@@ -3,8 +3,8 @@ import { WebXRSessionManager } from "../webXRSessionManager";
 import { Observable } from "../../Misc/observable";
 import { WebXRAbstractFeature } from "./WebXRAbstractFeature";
 import { Matrix } from "../../Maths/math.vector";
-import { Tools } from "../../Misc/tools";
 import { Nullable } from "../../types";
+import { Tools } from "../../Misc/tools";
 
 declare const XRImageTrackingResult: XRImageTrackingResult;
 
@@ -91,6 +91,7 @@ export class WebXRImageTracking extends WebXRAbstractFeature {
      */
     public onTrackedImageUpdatedObservable: Observable<IWebXRTrackedImage> = new Observable();
 
+    private _trackableScoresReceived: boolean = false;
     private _trackedImages: IWebXRTrackedImage[] = [];
 
     private _originalTrackingRequest: XRTrackedImageInit[];
@@ -109,17 +110,6 @@ export class WebXRImageTracking extends WebXRAbstractFeature {
     ) {
         super(_xrSessionManager);
         this.xrNativeFeatureName = "image-tracking";
-        if (this.options.images.length === 0) {
-            // no images provided?... return.
-            return;
-        }
-        if (this._xrSessionManager.session) {
-            this._init();
-        } else {
-            this._xrSessionManager.onXRSessionInit.addOnce(() => {
-                this._init();
-            });
-        }
     }
 
     /**
@@ -184,47 +174,47 @@ export class WebXRImageTracking extends WebXRAbstractFeature {
         }
         const promises = this.options.images.map((image) => {
             if (typeof image.src === "string") {
-                const p = new Promise<ImageBitmap>((resolve, reject) => {
-                    if (typeof image.src === "string") {
-                        const img = new Image();
-                        img.src = image.src;
-                        img.onload = () => {
-                            img.decode().then(() => {
-                                this._xrSessionManager.scene.getEngine().createImageBitmap(img).then((imageBitmap) => {
-                                    resolve(imageBitmap);
-                                });
-                            });
-                        };
-                        img.onerror = () => {
-                            Tools.Error(`Error loading image ${image.src}`);
-                            reject(`Error loading image ${image.src}`);
-                        };
-                    }
-                });
-                return p;
+                return this._xrSessionManager.scene.getEngine().createImageBitmapFromSource(image.src);
             } else {
                 return Promise.resolve(image.src); // resolve is probably unneeded
             }
         });
 
-        const images = await Promise.all(promises);
+        try
+        {
+            const images = await Promise.all(promises);
+
+            this._originalTrackingRequest = images.map((image, idx) => {
+                return {
+                    image,
+                    widthInMeters: this.options.images[idx].estimatedRealWorldWidth,
+                };
+            });
 
-        this._originalTrackingRequest = images.map((image, idx) => {
             return {
-                image,
-                widthInMeters: this.options.images[idx].estimatedRealWorldWidth,
+                trackedImages: this._originalTrackingRequest,
             };
-        });
-
-        return {
-            trackedImages: this._originalTrackingRequest,
-        };
+        } catch (ex) {
+            Tools.Error("Error loading images for tracking, WebXRImageTracking disabled for this session.");
+            return {};
+        }
     }
 
     protected _onXRFrame(_xrFrame: XRFrame) {
         if (!_xrFrame.getImageTrackingResults) {
             return;
         }
+
+        // Image tracking scores may be generated a few frames after the XR Session initializes.
+        // If we haven't received scores yet, then check scores first then bail out if necessary.
+        if (!this._trackableScoresReceived) {
+            this._checkScores();
+
+            if (!this._trackableScoresReceived) {
+                return;
+            }
+        }
+
         const imageTrackedResults = _xrFrame.getImageTrackingResults();
         for (const result of imageTrackedResults) {
             let changed = false;
@@ -267,12 +257,12 @@ export class WebXRImageTracking extends WebXRAbstractFeature {
         }
     }
 
-    private async _init() {
-        if (!this._xrSessionManager.session.getTrackedImageScores) {
+    private _checkScores(): void {
+        if (!this._xrSessionManager.session.getTrackedImageScores || this._trackableScoresReceived) {
             return;
         }
-        //
-        const imageScores = await this._xrSessionManager.session.getTrackedImageScores();
+
+        const imageScores = this._xrSessionManager.session.getTrackedImageScores();
         // check the scores for all
         for (let idx = 0; idx < imageScores.length; ++idx) {
             if (imageScores[idx] == "untrackable") {
@@ -289,6 +279,8 @@ export class WebXRImageTracking extends WebXRAbstractFeature {
                 this.onTrackableImageFoundObservable.notifyObservers(imageObject);
             }
         }
+
+        this._trackableScoresReceived ||= imageScores.length > 0;
     }
 }
 

src/XR/native/nativeXRFrame.ts

@@ -75,6 +75,8 @@ export class NativeXRFrame implements XRFrame {
     public get featurePointCloud(): number[] | undefined {
         return this._nativeImpl.featurePointCloud;
     }
+
+    public readonly getImageTrackingResults = this._nativeImpl.getImageTrackingResults!.bind(this._nativeImpl);
 }
 
 RegisterNativeTypeAsync("NativeXRFrame", NativeXRFrame);