diff --git a/externs.js b/externs.js index 8da42a7128b..9e88c44c78f 100644 --- a/externs.js +++ b/externs.js @@ -16,6 +16,7 @@ var WebAssembly = {}; var XRWebGLLayer = {}; var XRRay = {}; var XRHand = {}; +var XRImageTrackingResult = {}; var DOMPoint = {}; // extras requires this diff --git a/src/index.js b/src/index.js index 1a60527ff8c..d9a3f31591a 100644 --- a/src/index.js +++ b/src/index.js @@ -275,6 +275,8 @@ export { XrLightEstimation } from './xr/xr-light-estimation.js'; export { XrManager } from './xr/xr-manager.js'; export { XrHitTest } from './xr/xr-hit-test.js'; export { XrHitTestSource } from './xr/xr-hit-test-source.js'; +export { XrImageTracking } from './xr/xr-image-tracking.js'; +export { XrTrackedImage } from './xr/xr-tracked-image.js'; export { XrDomOverlay } from './xr/xr-dom-overlay.js'; // BACKWARDS COMPATIBILITY diff --git a/src/xr/xr-image-tracking.js b/src/xr/xr-image-tracking.js new file mode 100644 index 00000000000..42a3dc55010 --- /dev/null +++ b/src/xr/xr-image-tracking.js @@ -0,0 +1,162 @@ +import { EventHandler } from '../core/event-handler.js'; +import { XrTrackedImage } from './xr-tracked-image.js'; + +/** + * @class + * @name pc.XrImageTracking + * @classdesc Image Tracking provides the ability to track real world images by provided image samples and their estimate sizes. + * @description Image Tracking provides the ability to track real world images by provided image samples and their estimate sizes. + * @param {pc.XrManager} manager - WebXR Manager. + * @property {boolean} supported True if Image Tracking is supported. + * @property {boolean} available True if Image Tracking is available. This property will be false if no images were provided for the AR session or there was an error processing the provided images. + * @property {pc.XrTrackedImage[]} images List of {@link pc.XrTrackedImage} that contain tracking information. + */ +function XrImageTracking(manager) { + EventHandler.call(this); + + this._manager = manager; + this._supported = !! window.XRImageTrackingResult; + this._available = false; + + this._images = []; + + if (this._supported) { + this._manager.on('start', this._onSessionStart, this); + this._manager.on('end', this._onSessionEnd, this); + } +} +XrImageTracking.prototype = Object.create(EventHandler.prototype); +XrImageTracking.prototype.constructor = XrImageTracking; + +/** + * @event + * @name pc.XrImageTracking#error + * @param {Error} error - Error object related to a failure of image tracking. + * @description Fired when the XR session is started, but image tracking failed to process the provided images. + */ + +/** + * @function + * @name pc.XrImageTracking#add + * @description Add an image for image tracking. A width can also be provided to help the underlying system estimate the appropriate transformation. Modifying the tracked images list is only possible before an AR session is started. + * @param {HTMLCanvasElement|HTMLImageElement|SVGImageElement|HTMLVideoElement|Blob|ImageData|ImageBitmap} image - Image that is matching real world image as close as possible. Resolution of images should be at least 300x300. High resolution does NOT improve tracking performance. Color of image is irrelevant, so greyscale images can be used. Images with too many geometric features or repeating patterns will reduce tracking stability. + * @param {number} width - Width (in meters) of image in the real world. Providing this value as close to the real value will improve tracking quality. + * @returns {pc.XrTrackedImage} Tracked image object that will contain tracking information. + * @example + * // image with width of 20cm (0.2m) + * app.xr.imageTracking.add(bookCoverImg, 0.2); + */ +XrImageTracking.prototype.add = function (image, width) { + if (! this._supported || this._manager.active) return null; + + var trackedImage = new XrTrackedImage(image, width); + this._images.push(trackedImage); + return trackedImage; +}; + +/** + * @function + * @name pc.XrImageTracking#remove + * @description Remove an image from image tracking. + * @param {pc.XrTrackedImage} trackedImage - Tracked image to be removed. Modifying the tracked images list is only possible before an AR session is started. + */ +XrImageTracking.prototype.remove = function (trackedImage) { + if (this._manager.active) return; + + var ind = this._images.indexOf(trackedImage); + if (ind !== -1) { + trackedImage.destroy(); + this._images.splice(ind, 1); + } +}; + +XrImageTracking.prototype._onSessionStart = function () { + var self = this; + + this._manager.session.getTrackedImageScores().then(function (images) { + self._available = true; + + for (var i = 0; i < images.length; i++) { + self._images[i]._trackable = images[i] === 'trackable'; + } + }).catch(function (err) { + self._available = false; + self.fire('error', err); + }); +}; + +XrImageTracking.prototype._onSessionEnd = function () { + this._available = false; + + for (var i = 0; i < this._images.length; i++) { + this._images[i]._pose = null; + this._images[i]._measuredWidth = 0; + + if (this._images[i]._tracking) { + this._images[i]._tracking = false; + this._images[i].fire('untracked'); + } + } +}; + +XrImageTracking.prototype.prepareImages = function (callback) { + if (this._images.length) { + Promise.all(this._images.map(function (trackedImage) { + return trackedImage.prepare(); + })).then(function (bitmaps) { + callback(null, bitmaps); + }).catch(function (err) { + callback(err, null); + }); + } else { + callback(null, null); + } +}; + +XrImageTracking.prototype.update = function (frame) { + if (! this._available) return; + + var results = frame.getImageTrackingResults(); + var index = { }; + var i; + + for (i = 0; i < results.length; i++) { + index[results[i].index] = results[i]; + + var trackedImage = this._images[results[i].index]; + trackedImage._emulated = results[i].trackingState === 'emulated'; + trackedImage._measuredWidth = results[i].measuredWidthInMeters; + trackedImage._dirtyTransform = true; + trackedImage._pose = frame.getPose(results[i].imageSpace, this._manager._referenceSpace); + } + + for (i = 0; i < this._images.length; i++) { + if (this._images[i]._tracking && ! index[i]) { + this._images[i]._tracking = false; + this._images[i].fire('untracked'); + } else if (! this._images[i]._tracking && index[i]) { + this._images[i]._tracking = true; + this._images[i].fire('tracked'); + } + } +}; + +Object.defineProperty(XrImageTracking.prototype, 'supported', { + get: function () { + return this._supported; + } +}); + +Object.defineProperty(XrImageTracking.prototype, 'available', { + get: function () { + return this._available; + } +}); + +Object.defineProperty(XrImageTracking.prototype, 'images', { + get: function () { + return this._images; + } +}); + +export { XrImageTracking }; diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js index bc67f033d2c..309bc110876 100644 --- a/src/xr/xr-light-estimation.js +++ b/src/xr/xr-light-estimation.js @@ -194,7 +194,7 @@ Object.defineProperty(XrLightEstimation.prototype, 'supported', { */ Object.defineProperty(XrLightEstimation.prototype, 'available', { get: function () { - return !! this._available; + return this._available; } }); diff --git a/src/xr/xr-manager.js b/src/xr/xr-manager.js index 2593f9993a0..9f9fbcc9bf5 100644 --- a/src/xr/xr-manager.js +++ b/src/xr/xr-manager.js @@ -10,6 +10,7 @@ import { XRTYPE_INLINE, XRTYPE_VR, XRTYPE_AR } from './constants.js'; import { XrHitTest } from './xr-hit-test.js'; import { XrInput } from './xr-input.js'; import { XrLightEstimation } from './xr-light-estimation.js'; +import { XrImageTracking } from './xr-image-tracking.js'; import { XrDomOverlay } from './xr-dom-overlay.js'; /** @@ -55,6 +56,7 @@ function XrManager(app) { this.input = new XrInput(this); this.hitTest = new XrHitTest(this); this.lightEstimation = new XrLightEstimation(this); + this.imageTracking = new XrImageTracking(this); this.domOverlay = new XrDomOverlay(this); this._camera = null; @@ -138,7 +140,6 @@ XrManager.prototype.constructor = XrManager; * }); */ - /** * @event * @name pc.XrManager#error @@ -218,6 +219,10 @@ XrManager.prototype.start = function (camera, type, spaceType, options) { opts.optionalFeatures.push('light-estimation'); opts.optionalFeatures.push('hit-test'); + if (options && options.imageTracking) { + opts.optionalFeatures.push('image-tracking'); + } + if (this.domOverlay.root) { opts.optionalFeatures.push('dom-overlay'); opts.domOverlay = { root: this.domOverlay.root }; @@ -226,11 +231,31 @@ XrManager.prototype.start = function (camera, type, spaceType, options) { opts.optionalFeatures.push('hand-tracking'); } - if (options && options.optionalFeatures) { + if (options && options.optionalFeatures) opts.optionalFeatures = opts.optionalFeatures.concat(options.optionalFeatures); + + if (this.imageTracking.images.length) { + this.imageTracking.prepareImages(function (err, trackedImages) { + if (err) { + if (callback) callback(err); + self.fire('error', err); + return; + } + + if (trackedImages !== null) + opts.trackedImages = trackedImages; + + self._onStartOptionsReady(type, spaceType, opts, callback); + }); + } else { + self._onStartOptionsReady(type, spaceType, opts, callback); } +}; - navigator.xr.requestSession(type, opts).then(function (session) { +XrManager.prototype._onStartOptionsReady = function (type, spaceType, options, callback) { + var self = this; + + navigator.xr.requestSession(type, options).then(function (session) { self._onSessionStart(session, spaceType, callback); }).catch(function (ex) { self._camera.camera.xr = null; @@ -484,6 +509,9 @@ XrManager.prototype.update = function (frame) { if (this.lightEstimation.supported) { this.lightEstimation.update(frame); } + if (this.imageTracking.supported) { + this.imageTracking.update(frame); + } } this.fire('update', frame); diff --git a/src/xr/xr-tracked-image.js b/src/xr/xr-tracked-image.js new file mode 100644 index 00000000000..7d6ccaa3a80 --- /dev/null +++ b/src/xr/xr-tracked-image.js @@ -0,0 +1,139 @@ +import { EventHandler } from '../core/event-handler.js'; +import { Vec3 } from '../math/vec3.js'; +import { Quat } from '../math/quat.js'; + +/** + * @class + * @name pc.XrTrackedImage + * @classdesc The tracked image interface that is created by the Image Tracking system and is provided as a list from {@link pc.XrImageTracking#images}. It contains information about the tracking state as well as the position and rotation of the tracked image. + * @description The tracked image interface that is created by the Image Tracking system and is provided as a list from {@link pc.XrImageTracking#images}. It contains information about the tracking state as well as the position and rotation of the tracked image. + * @param {HTMLCanvasElement|HTMLImageElement|SVGImageElement|HTMLVideoElement|Blob|ImageData|ImageBitmap} image - Image that is matching the real world image as closely as possible. Resolution of images should be at least 300x300. High resolution does NOT improve tracking performance. Color of image is irrelevant, so greyscale images can be used. Images with too many geometric features or repeating patterns will reduce tracking stability. + * @param {number} width - Width (in meters) of image in real world. Providing this value as close to the real value will improve tracking quality. + * @property {HTMLCanvasElement|HTMLImageElement|SVGImageElement|HTMLVideoElement|Blob|ImageData|ImageBitmap} image Image that is used for tracking. + * @property {number} width Width that is provided to assist tracking performance. This property can be updated only when the AR session is not running. + * @property {boolean} trackable True if image is trackable. A too small resolution or invalid images can be untrackable by the underlying AR system. + * @property {boolean} tracking True if image is in tracking state and being tracked in real world by the underlying AR system. + * @property {boolean} emulated True if image was recently tracked but currently is not actively tracked due to inability of identifying the image by the underlying AR system. Position and rotation will be based on the previously known transformation assuming the tracked image has not moved. + */ +function XrTrackedImage(image, width) { + EventHandler.call(this); + + this._image = image; + this._bitmap = null; + this._width = width; + this._measuredWidth = 0; + this._trackable = false; + this._tracking = false; + this._emulated = false; + this._pose = null; + + this._position = new Vec3(); + this._rotation = new Quat(); +} +XrTrackedImage.prototype = Object.create(EventHandler.prototype); +XrTrackedImage.prototype.constructor = XrTrackedImage; + +/** + * @event + * @name pc.XrTrackedImage#tracked + * @description Fired when image becomes actively tracked. + */ + +/** + * @event + * @name pc.XrTrackedImage#untracked + * @description Fired when image is no more actively tracked. + */ + +XrTrackedImage.prototype.prepare = function () { + var self = this; + + if (this._bitmap) { + return { + image: this._bitmap, + widthInMeters: this._width + }; + } + + return createImageBitmap(this._image) + .then(function (bitmap) { + self._bitmap = bitmap; + return { + image: self._bitmap, + widthInMeters: self._width + }; + }); +}; + +XrTrackedImage.prototype.destroy = function () { + this._image = null; + this._pose = null; + + if (this._bitmap) { + this._bitmap.close(); + this._bitmap = null; + } +}; + +/** + * @function + * @name pc.XrTrackedImage#getPosition + * @description Get the position of the tracked image. The position is the most recent one based on the tracked image state. + * @returns {pc.Vec3} Position in world space. + * @example + * // update entity position to match tracked image position + * entity.setPosition(trackedImage.getPosition()); + */ +XrTrackedImage.prototype.getPosition = function () { + if (this._pose) this._position.copy(this._pose.transform.position); + return this._position; +}; + +/** + * @function + * @name pc.XrTrackedImage#getRotation + * @description Get the rotation of the tracked image. The rotation is the most recent based on the tracked image state. + * @returns {pc.Quat} Rotation in world space. + * @example + * // update entity rotation to match tracked image rotation + * entity.setRotation(trackedImage.getRotation()); + */ +XrTrackedImage.prototype.getRotation = function () { + if (this._pose) this._rotation.copy(this._pose.transform.orientation); + return this._rotation; +}; + +Object.defineProperty(XrTrackedImage.prototype, 'image', { + get: function () { + return this._image; + } +}); + +Object.defineProperty(XrTrackedImage.prototype, 'width', { + get: function () { + return this._width; + }, + set: function (value) { + this._width = value; + } +}); + +Object.defineProperty(XrTrackedImage.prototype, 'trackable', { + get: function () { + return this._trackable; + } +}); + +Object.defineProperty(XrTrackedImage.prototype, 'tracking', { + get: function () { + return this._tracking; + } +}); + +Object.defineProperty(XrTrackedImage.prototype, 'emulated', { + get: function () { + return this._emulated; + } +}); + +export { XrTrackedImage };