From 9296c0997bc83b963c9053d6ecf6c428451d0cbf Mon Sep 17 00:00:00 2001 From: Maksims Mihejevs Date: Mon, 4 May 2020 01:29:17 +0300 Subject: [PATCH 1/5] light estimation --- build/dependencies.txt | 1 + src/xr/xr-light-estimation.js | 186 ++++++++++++++++++++++++++++++++++ src/xr/xr-manager.js | 19 +++- 3 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/xr/xr-light-estimation.js diff --git a/build/dependencies.txt b/build/dependencies.txt index c25abdec56d..5c1e991d2b1 100644 --- a/build/dependencies.txt +++ b/build/dependencies.txt @@ -118,6 +118,7 @@ ../src/xr/xr-input-source.js ../src/xr/xr-hit-test.js ../src/xr/xr-hit-test-source.js +../src/xr/xr-light-estimation.js ../src/net/http.js ../src/script/script.js ../src/script/script-type.js diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js new file mode 100644 index 00000000000..be81f45cbbe --- /dev/null +++ b/src/xr/xr-light-estimation.js @@ -0,0 +1,186 @@ +Object.assign(pc, function () { + var vec3A = new pc.Vec3(); + var vec3B = new pc.Vec3(); + var mat4A = new pc.Mat4(); + var mat4B = new pc.Mat4(); + + /** + * @class + * @name pc.XrLightEstimation + * @augments pc.EventHandler + * @classdesc Light Estimation provides illimunation data from real world, which is estimated by underlying AR systems. + * It provides reflection Cube Map, that represents reflection estimation from viewer position. + * More simplified approximation of light is provided by L2 Spherical Harminics data. + * And the most simple level of light estimation is a most prominent directional light, its rotation, intensity and a color. + * @description Light Estimation provides illimunation data from real world, which is estimated by underlying AR systems. + * It provides reflection Cube Map, that represents reflection estimation from viewer position. + * More simplified approximation of light is provided by L2 Spherical Harminics data. + * And the most simple level of light estimation is a most prominent directional light, its rotation, intensity and a color. + * @param {pc.XrManager} manager - WebXR Manager. + * @property {boolean} supported True if Light Estimation is supported, this information is available only during active AR session. + * @property {number} intensity Intensity of estimated most prominent directional light. + * @property {pc.Color} color Color of estimated most prominent directional light. + * @property {pc.Quat} rotation Rotation of estimated most prominent directional light. + */ + var XrLightEstimation = function (manager) { + pc.EventHandler.call(this); + + this._manager = manager; + + this._supported = false; + this._available = false; + + this._lightProbeRequested = false; + this._lightProbe = null; + + this._intensity = 0; + this._rotation = new pc.Quat(); + this._color = new pc.Color(); + + this._manager.on('start', this._onSessionStart, this); + this._manager.on('end', this._onSessionEnd, this); + }; + XrLightEstimation.prototype = Object.create(pc.EventHandler.prototype); + XrLightEstimation.prototype.constructor = XrLightEstimation; + + XrLightEstimation.prototype._onSessionStart = function () { + var supported = !! this._manager.session.requestLightProbe; + if (! supported) return; + + this._supported = true; + this.fire('supported'); + }; + + XrLightEstimation.prototype._onSessionEnd = function () { + this._supported = false; + this._available = false; + + this._lightProbeRequested = false; + this._lightProbe = null; + + this._intensity = 1; + this._rotation.set(0, 0, 0, 1); + this._color.set(1, 1, 1); + }; + + /** + * @function + * @name pc.XrLightEstimation#start + * @description Start estimation of illimunation data. + * Availability of such data will come later, and `available` event will be fired. + * If it failed to start estimation, `error` event will be fired. + * @example + * app.xr.on('start', function () { + * if (app.xr.lightEstimation.supported) { + * app.xr.lightEstimation.start(); + * } + * }); + */ + XrLightEstimation.prototype.start = function () { + var err; + + if (! this._manager.session) + err = new Error('XR session is not running'); + + if (! err && this._manager.type !== pc.XRTYPE_AR) + err = new Error('XR session type is not AR'); + + if (! err && ! this._supported) + err = new Error('light-estimation is not supported'); + + if (! err && this._lightProbe || this._lightProbeRequested) + err = new Error('light estimation is already requested'); + + if (err) { + this.fire('error', err); + return; + } + + var self = this; + this._lightProbeRequested = true; + + this._manager.session.requestLightProbe( + ).then(function (lightProbe) { + self._lightProbeRequested = false; + + if (self._manager.active) { + self._lightProbe = lightProbe; + } else { + self.fire('error', new Error('XR session is not active')); + } + }).catch(function (ex) { + self._lightProbeRequested = false; + self.fire('error', ex); + }); + }; + + XrLightEstimation.prototype.update = function (frame) { + if (! this._lightProbe) return; + + var lightEstimate = frame.getLightEstimate(this._lightProbe); + if (! lightEstimate) return; + + if (! this._available) { + this._available = true; + this.fire('available'); + } + + // intensity + this._intensity = Math.max(1.0, Math.max(lightEstimate.primaryLightIntensity.x, Math.max(lightEstimate.primaryLightIntensity.y, lightEstimate.primaryLightIntensity.z))); + + // color + vec3A.copy(lightEstimate.primaryLightIntensity).scale(1 / this._intensity); + this._color.set(vec3A.x, vec3A.y, vec3A.z); + + // rotation + vec3A.set(0, 0, 0); + vec3B.copy(lightEstimate.primaryLightDirection); + mat4A.setLookAt(vec3B, vec3A, pc.Vec3.UP); + mat4B.setFromAxisAngle(pc.Vec3.RIGHT, 90); // direcitonal light is looking down + mat4A.mul(mat4B); + this._rotation.setFromMat4(mat4A); + }; + + Object.defineProperty(XrLightEstimation.prototype, 'supported', { + get: function () { + return this._supported; + } + }); + + /** + * @name pc.XrLightEstimation#available + * @type {boolean} + * @description True if estimated light information is available. + * @example + * if (app.xr.lightEstimation.available) { + * entity.light.intensity = app.xr.lightEstimation.intensity; + * } + */ + Object.defineProperty(XrLightEstimation.prototype, 'available', { + get: function () { + return !! this._available; + } + }); + + Object.defineProperty(XrLightEstimation.prototype, 'intensity', { + get: function () { + return this._intensity; + } + }); + + Object.defineProperty(XrLightEstimation.prototype, 'color', { + get: function () { + return this._color; + } + }); + + Object.defineProperty(XrLightEstimation.prototype, 'rotation', { + get: function () { + return this._rotation; + } + }); + + return { + XrLightEstimation: XrLightEstimation + }; +}()); diff --git a/src/xr/xr-manager.js b/src/xr/xr-manager.js index 6e27fa3f46f..390802814d9 100644 --- a/src/xr/xr-manager.js +++ b/src/xr/xr-manager.js @@ -123,6 +123,7 @@ Object.assign(pc, function () { this.input = new pc.XrInput(this); this.hitTest = new pc.XrHitTest(this); + this.lightEstimation = new pc.XrLightEstimation(this); this._camera = null; this.views = []; @@ -258,8 +259,14 @@ Object.assign(pc, function () { // 3. probably immersive-vr will fail to be created // 4. call makeXRCompatible, very likely will lead to context loss + var optionalFeatures = []; + + if (type === pc.XRTYPE_AR) + optionalFeatures.push('light-estimation'); + navigator.xr.requestSession(type, { - requiredFeatures: [spaceType] + requiredFeatures: [spaceType], + optionalFeatures: optionalFeatures }).then(function (session) { self._onSessionStart(session, spaceType, callback); }).catch(function (ex) { @@ -507,8 +514,14 @@ Object.assign(pc, function () { this.input.update(frame); - if (this._type === pc.XRTYPE_AR && this.hitTest.supported) - this.hitTest.update(frame); + if (this._type === pc.XRTYPE_AR) { + if (this.hitTest.supported) { + this.hitTest.update(frame); + } + if (this.lightEstimation.supported) { + this.lightEstimation.update(frame); + } + } this.fire('update'); }; From 92631495423bf49386035a88a168224c7209f58b Mon Sep 17 00:00:00 2001 From: Maksims Mihejevs Date: Mon, 4 May 2020 02:19:36 +0300 Subject: [PATCH 2/5] document events --- src/xr/xr-light-estimation.js | 51 ++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js index be81f45cbbe..76902ad4767 100644 --- a/src/xr/xr-light-estimation.js +++ b/src/xr/xr-light-estimation.js @@ -18,9 +18,9 @@ Object.assign(pc, function () { * And the most simple level of light estimation is a most prominent directional light, its rotation, intensity and a color. * @param {pc.XrManager} manager - WebXR Manager. * @property {boolean} supported True if Light Estimation is supported, this information is available only during active AR session. - * @property {number} intensity Intensity of estimated most prominent directional light. - * @property {pc.Color} color Color of estimated most prominent directional light. - * @property {pc.Quat} rotation Rotation of estimated most prominent directional light. + * @property {number|null} intensity Intensity of estimated most prominent directional light. Or null if data is not available. + * @property {pc.Color|null} color Color of estimated most prominent directional light. Or null if data is not available. + * @property {pc.Quat|null} rotation Rotation of estimated most prominent directional light. Or null if data is not available. */ var XrLightEstimation = function (manager) { pc.EventHandler.call(this); @@ -43,12 +43,27 @@ Object.assign(pc, function () { XrLightEstimation.prototype = Object.create(pc.EventHandler.prototype); XrLightEstimation.prototype.constructor = XrLightEstimation; + /** + * @event + * @name pc.XrLightEstimation#available + * @description Fired when light estimation data becomes available. + */ + + /** + * @event + * @name pc.XrLightEstimation#error + * @param {Error} error - Error object related to failure of light estimation start. + * @description Fired when light estimation has failed to start. + * @example + * app.xr.lightEstimation.on('error', function (ex) { + * // has failed to start + * }); + */ + XrLightEstimation.prototype._onSessionStart = function () { var supported = !! this._manager.session.requestLightProbe; if (! supported) return; - this._supported = true; - this.fire('supported'); }; XrLightEstimation.prototype._onSessionEnd = function () { @@ -57,10 +72,6 @@ Object.assign(pc, function () { this._lightProbeRequested = false; this._lightProbe = null; - - this._intensity = 1; - this._rotation.set(0, 0, 0, 1); - this._color.set(1, 1, 1); }; /** @@ -101,10 +112,13 @@ Object.assign(pc, function () { this._manager.session.requestLightProbe( ).then(function (lightProbe) { + var wasRequested = self._lightProbeRequested; self._lightProbeRequested = false; if (self._manager.active) { - self._lightProbe = lightProbe; + if (wasRequested) { + self._lightProbe = lightProbe; + } } else { self.fire('error', new Error('XR session is not active')); } @@ -114,6 +128,17 @@ Object.assign(pc, function () { }); }; + /** + * @function + * @name pc.XrLightEstimation#stop + * @description Stop estimation of illimunation data. + */ + XrLightEstimation.prototype.stop = function () { + this._lightProbeRequested = false; + this._lightProbe = null; + this._available = false; + }; + XrLightEstimation.prototype.update = function (frame) { if (! this._lightProbe) return; @@ -164,19 +189,19 @@ Object.assign(pc, function () { Object.defineProperty(XrLightEstimation.prototype, 'intensity', { get: function () { - return this._intensity; + return this._available ? this._intensity : null; } }); Object.defineProperty(XrLightEstimation.prototype, 'color', { get: function () { - return this._color; + return this._available ? this._color : null; } }); Object.defineProperty(XrLightEstimation.prototype, 'rotation', { get: function () { - return this._rotation; + return this._available ? this._rotation : null; } }); From 9dc49295b1ec1befb595be99d0942239cce996df Mon Sep 17 00:00:00 2001 From: Maksims Mihejevs Date: Mon, 4 May 2020 02:21:23 +0300 Subject: [PATCH 3/5] stop > end, for API consistency --- src/xr/xr-light-estimation.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js index 76902ad4767..74344c4ef85 100644 --- a/src/xr/xr-light-estimation.js +++ b/src/xr/xr-light-estimation.js @@ -130,10 +130,10 @@ Object.assign(pc, function () { /** * @function - * @name pc.XrLightEstimation#stop - * @description Stop estimation of illimunation data. + * @name pc.XrLightEstimation#end + * @description End estimation of illimunation data. */ - XrLightEstimation.prototype.stop = function () { + XrLightEstimation.prototype.end = function () { this._lightProbeRequested = false; this._lightProbe = null; this._available = false; From 19dfb7d6c2e4bd751d3db3370f4389c6ac1792b9 Mon Sep 17 00:00:00 2001 From: Will Eastcott Date: Mon, 4 May 2020 13:24:25 +0100 Subject: [PATCH 4/5] Small tweaks --- src/xr/xr-light-estimation.js | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js index 74344c4ef85..915ef0da541 100644 --- a/src/xr/xr-light-estimation.js +++ b/src/xr/xr-light-estimation.js @@ -8,19 +8,16 @@ Object.assign(pc, function () { * @class * @name pc.XrLightEstimation * @augments pc.EventHandler - * @classdesc Light Estimation provides illimunation data from real world, which is estimated by underlying AR systems. - * It provides reflection Cube Map, that represents reflection estimation from viewer position. - * More simplified approximation of light is provided by L2 Spherical Harminics data. - * And the most simple level of light estimation is a most prominent directional light, its rotation, intensity and a color. - * @description Light Estimation provides illimunation data from real world, which is estimated by underlying AR systems. - * It provides reflection Cube Map, that represents reflection estimation from viewer position. - * More simplified approximation of light is provided by L2 Spherical Harminics data. - * And the most simple level of light estimation is a most prominent directional light, its rotation, intensity and a color. + * @classdesc Light Estimation provides illimunation data from the real world, which is estimated by the underlying AR system. + * It provides a reflection Cube Map, that represents the reflection estimation from the viewer position. + * A more simplified approximation of light is provided by L2 Spherical Harmonics data. + * And the most simple level of light estimation is the most prominent directional light, its rotation, intensity and color. + * @description Creates a new XrLightEstimation. Note that this is created internally by the {@link pc.XrManager}. * @param {pc.XrManager} manager - WebXR Manager. - * @property {boolean} supported True if Light Estimation is supported, this information is available only during active AR session. - * @property {number|null} intensity Intensity of estimated most prominent directional light. Or null if data is not available. - * @property {pc.Color|null} color Color of estimated most prominent directional light. Or null if data is not available. - * @property {pc.Quat|null} rotation Rotation of estimated most prominent directional light. Or null if data is not available. + * @property {boolean} supported True if Light Estimation is supported. This information is available only during an active AR session. + * @property {number|null} intensity Intensity of what is estimated to be the most prominent directional light. Or null if data is not available. + * @property {pc.Color|null} color Color of what is estimated to be the most prominent directional light. Or null if data is not available. + * @property {pc.Quat|null} rotation Rotation of what is estimated to be the most prominent directional light. Or null if data is not available. */ var XrLightEstimation = function (manager) { pc.EventHandler.call(this); @@ -78,8 +75,8 @@ Object.assign(pc, function () { * @function * @name pc.XrLightEstimation#start * @description Start estimation of illimunation data. - * Availability of such data will come later, and `available` event will be fired. - * If it failed to start estimation, `error` event will be fired. + * Availability of such data will come later and an `available` event will be fired. + * If it failed to start estimation, an `error` event will be fired. * @example * app.xr.on('start', function () { * if (app.xr.lightEstimation.supported) { @@ -131,7 +128,7 @@ Object.assign(pc, function () { /** * @function * @name pc.XrLightEstimation#end - * @description End estimation of illimunation data. + * @description End estimation of illumination data. */ XrLightEstimation.prototype.end = function () { this._lightProbeRequested = false; @@ -151,10 +148,11 @@ Object.assign(pc, function () { } // intensity - this._intensity = Math.max(1.0, Math.max(lightEstimate.primaryLightIntensity.x, Math.max(lightEstimate.primaryLightIntensity.y, lightEstimate.primaryLightIntensity.z))); + var pli = lightEstimate.primaryLightIntensity; + this._intensity = Math.max(1.0, Math.max(pli.x, Math.max(pli.y, pli.z))); // color - vec3A.copy(lightEstimate.primaryLightIntensity).scale(1 / this._intensity); + vec3A.copy(pli).scale(1 / this._intensity); this._color.set(vec3A.x, vec3A.y, vec3A.z); // rotation From 2ff789bd2183c6d270acae5966ef9747b5c71747 Mon Sep 17 00:00:00 2001 From: Maksims Mihejevs Date: Thu, 7 May 2020 15:29:05 +0300 Subject: [PATCH 5/5] spherical harmonics from light estimation --- src/xr/xr-light-estimation.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/xr/xr-light-estimation.js b/src/xr/xr-light-estimation.js index 74344c4ef85..96c03457e18 100644 --- a/src/xr/xr-light-estimation.js +++ b/src/xr/xr-light-estimation.js @@ -37,6 +37,8 @@ Object.assign(pc, function () { this._rotation = new pc.Quat(); this._color = new pc.Color(); + this._sphericalHarmonics = new Float32Array(27); + this._manager.on('start', this._onSessionStart, this); this._manager.on('end', this._onSessionEnd, this); }; @@ -164,6 +166,9 @@ Object.assign(pc, function () { mat4B.setFromAxisAngle(pc.Vec3.RIGHT, 90); // direcitonal light is looking down mat4A.mul(mat4B); this._rotation.setFromMat4(mat4A); + + // spherical harmonics + this._sphericalHarmonics.set(lightEstimate.sphericalHarmonicsCoefficients); }; Object.defineProperty(XrLightEstimation.prototype, 'supported', { @@ -205,6 +210,12 @@ Object.assign(pc, function () { } }); + Object.defineProperty(XrLightEstimation.prototype, 'sphericalHarmonics', { + get: function () { + return this._available ? this._sphericalHarmonics : null; + } + }); + return { XrLightEstimation: XrLightEstimation };