From 123d045e227dba295dcc75b36f30e5680fba0a98 Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Sat, 4 Nov 2023 13:39:25 +0200 Subject: [PATCH 1/8] XrViews, XR Raw Camera Access --- examples/package-lock.json | 46 ++--- examples/src/examples/xr/ar-basic.mjs | 7 + examples/src/examples/xr/ar-camera-color.mjs | 197 +++++++++++++++++++ examples/src/examples/xr/index.mjs | 1 + src/framework/xr/constants.js | 4 + src/framework/xr/xr-depth-sensing.js | 4 +- src/framework/xr/xr-manager.js | 113 +++++------ src/framework/xr/xr-view.js | 164 +++++++++++++++ src/framework/xr/xr-views.js | 94 +++++++++ src/scene/renderer/forward-renderer.js | 8 +- src/scene/renderer/renderer.js | 31 +-- 11 files changed, 557 insertions(+), 112 deletions(-) create mode 100644 examples/src/examples/xr/ar-camera-color.mjs create mode 100644 src/framework/xr/xr-view.js create mode 100644 src/framework/xr/xr-views.js diff --git a/examples/package-lock.json b/examples/package-lock.json index 59943da9d95..8be323a30c2 100644 --- a/examples/package-lock.json +++ b/examples/package-lock.json @@ -57,23 +57,25 @@ "version": "1.67.0-dev", "dev": true, "license": "MIT", + "dependencies": { + "@types/webxr": "^0.5.7", + "@webgpu/types": "^0.1.38" + }, "devDependencies": { - "@babel/core": "^7.23.0", + "@babel/core": "^7.23.2", "@babel/eslint-parser": "^7.22.15", - "@babel/preset-env": "^7.22.20", + "@babel/preset-env": "^7.23.2", "@playcanvas/canvas-mock": "^1.0.1", "@playcanvas/eslint-config": "^1.7.1", "@playcanvas/jsdoc-template": "^1.1.2", - "@rollup/plugin-babel": "^6.0.3", - "@rollup/plugin-node-resolve": "^15.2.1", - "@rollup/plugin-strip": "^3.0.2", - "@rollup/plugin-terser": "^0.4.3", - "@rollup/pluginutils": "^5.0.4", - "@types/webxr": "^0.5.5", - "@webgpu/types": "^0.1.35", + "@rollup/plugin-babel": "^6.0.4", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-strip": "^3.0.4", + "@rollup/plugin-terser": "^0.4.4", + "@rollup/pluginutils": "^5.0.5", "c8": "^8.0.0", "chai": "^4.3.10", - "eslint": "^8.50.0", + "eslint": "^8.52.0", "fflate": "^0.8.1", "jsdoc": "^4.0.2", "jsdoc-tsimport-plugin": "^1.0.5", @@ -87,7 +89,7 @@ "rollup-plugin-jscc": "2.0.0", "rollup-plugin-visualizer": "^5.9.2", "serve": "^14.2.1", - "sinon": "^16.0.0", + "sinon": "^17.0.0", "typedoc": "^0.25.1", "typedoc-plugin-mdn-links": "^3.1.0", "typescript": "^5.2.2", @@ -9820,22 +9822,22 @@ "playcanvas": { "version": "file:..", "requires": { - "@babel/core": "^7.23.0", + "@babel/core": "^7.23.2", "@babel/eslint-parser": "^7.22.15", - "@babel/preset-env": "^7.22.20", + "@babel/preset-env": "^7.23.2", "@playcanvas/canvas-mock": "^1.0.1", "@playcanvas/eslint-config": "^1.7.1", "@playcanvas/jsdoc-template": "^1.1.2", - "@rollup/plugin-babel": "^6.0.3", - "@rollup/plugin-node-resolve": "^15.2.1", - "@rollup/plugin-strip": "^3.0.2", - "@rollup/plugin-terser": "^0.4.3", - "@rollup/pluginutils": "^5.0.4", - "@types/webxr": "^0.5.5", - "@webgpu/types": "^0.1.35", + "@rollup/plugin-babel": "^6.0.4", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-strip": "^3.0.4", + "@rollup/plugin-terser": "^0.4.4", + "@rollup/pluginutils": "^5.0.5", + "@types/webxr": "^0.5.7", + "@webgpu/types": "^0.1.38", "c8": "^8.0.0", "chai": "^4.3.10", - "eslint": "^8.50.0", + "eslint": "^8.52.0", "fflate": "^0.8.1", "jsdoc": "^4.0.2", "jsdoc-tsimport-plugin": "^1.0.5", @@ -9849,7 +9851,7 @@ "rollup-plugin-jscc": "2.0.0", "rollup-plugin-visualizer": "^5.9.2", "serve": "^14.2.1", - "sinon": "^16.0.0", + "sinon": "^17.0.0", "typedoc": "^0.25.1", "typedoc-plugin-mdn-links": "^3.1.0", "typescript": "^5.2.2", diff --git a/examples/src/examples/xr/ar-basic.mjs b/examples/src/examples/xr/ar-basic.mjs index ad68776a507..c41851c4806 100644 --- a/examples/src/examples/xr/ar-basic.mjs +++ b/examples/src/examples/xr/ar-basic.mjs @@ -15,6 +15,13 @@ async function example({ canvas }) { if (!el) { el = document.createElement('div'); el.classList.add('message'); + el.style.position = 'absolute'; + el.style.bottom = '96px'; + el.style.right = '0'; + el.style.padding = '8px 16px'; + el.style.fontFamily = 'Helvetica, Arial, sans-serif'; + el.style.color = '#fff'; + el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; document.body.append(el); } el.textContent = msg; diff --git a/examples/src/examples/xr/ar-camera-color.mjs b/examples/src/examples/xr/ar-camera-color.mjs new file mode 100644 index 00000000000..00c1b5941f3 --- /dev/null +++ b/examples/src/examples/xr/ar-camera-color.mjs @@ -0,0 +1,197 @@ +import * as pc from 'playcanvas'; + +/** + * @typedef {import('../../options.mjs').ExampleOptions} ExampleOptions + * @param {import('../../options.mjs').ExampleOptions} options - The example options. + * @returns {Promise} The example application. + */ +async function example({ canvas }) { + /** + * @param {string} msg - The message. + */ + const message = function (msg) { + /** @type {HTMLDivElement} */ + let el = document.querySelector('.message'); + if (!el) { + el = document.createElement('div'); + el.classList.add('message'); + el.style.position = 'absolute'; + el.style.bottom = '96px'; + el.style.right = '0'; + el.style.padding = '8px 16px'; + el.style.fontFamily = 'Helvetica, Arial, sans-serif'; + el.style.color = '#fff'; + el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; + document.body.append(el); + } + el.textContent = msg; + }; + + const app = new pc.Application(canvas, { + mouse: new pc.Mouse(canvas), + touch: new pc.TouchDevice(canvas), + keyboard: new pc.Keyboard(window), + graphicsDeviceOptions: { alpha: true } + }); + + app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); + app.setCanvasResolution(pc.RESOLUTION_AUTO); + + // Ensure canvas is resized when window changes size + const resize = () => app.resizeCanvas(); + window.addEventListener('resize', resize); + app.on('destroy', () => { + window.removeEventListener('resize', resize); + }); + + // use device pixel ratio + app.graphicsDevice.maxPixelRatio = window.devicePixelRatio; + + app.start(); + + // create camera + const c = new pc.Entity(); + c.addComponent('camera', { + clearColor: new pc.Color(0, 0, 0, 0), + farClip: 10000 + }); + app.root.addChild(c); + + const l = new pc.Entity(); + l.addComponent("light", { + type: "spot", + range: 30 + }); + l.translate(0, 10, 0); + app.root.addChild(l); + + const material = new pc.StandardMaterial(); + + /** + * @param {number} x - The x coordinate. + * @param {number} y - The y coordinate. + * @param {number} z - The z coordinate. + */ + const createCube = function (x, y, z) { + const cube = new pc.Entity(); + cube.addComponent("render", { + type: "box" + }); + cube.render.material = material; + cube.setLocalScale(0.5, 0.5, 0.5); + cube.translate(x * 0.5, y, z * 0.5); + app.root.addChild(cube); + }; + + // create a grid of cubes + const SIZE = 4; + for (let x = 0; x < SIZE; x++) { + for (let y = 0; y < SIZE; y++) { + createCube(2 * x - SIZE, 0.25, 2 * y - SIZE); + } + } + + if (app.xr.supported) { + const activate = function () { + if (app.xr.isAvailable(pc.XRTYPE_AR)) { + c.camera.startXr(pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { + cameraColor: true, + callback: function (err) { + if (err) message("WebXR Immersive AR failed to start: " + err.message); + } + }); + } else { + message("Immersive AR is not available"); + } + }; + + app.mouse.on("mousedown", function () { + if (!app.xr.active) + activate(); + }); + + if (app.touch) { + app.touch.on("touchend", function (evt) { + if (!app.xr.active) { + // if not in VR, activate + activate(); + } else { + // otherwise reset camera + c.camera.endXr(); + } + + evt.event.preventDefault(); + evt.event.stopPropagation(); + }); + } + + // end session by keyboard ESC + app.keyboard.on('keydown', function (evt) { + if (evt.key === pc.KEY_ESCAPE && app.xr.active) { + app.xr.end(); + } + }); + + app.xr.on('start', function () { + message("Immersive AR session has started"); + }); + app.xr.on('end', function () { + message("Immersive AR session has ended"); + }); + app.xr.on('available:' + pc.XRTYPE_AR, function (available) { + if (available) { + if (!app.xr.views.supportedColor) { + message("AR Camera Color is not supported"); + } else { + message("Touch screen to start AR session"); + } + } else { + message("Immersive AR is not available"); + } + }); + + app.on('update', () => { + if (app.xr.views.availableColor) { + for(let i = 0; i < app.xr.views.size; i++) { + const view = app.xr.views.list[i]; + if (!view.textureColor) + continue; + + if (!material.diffuseMap) { + material.diffuseMap = view.textureColor; + material.update(); + } + + app.drawTexture(0.5, -0.5, 1, -1, view.textureColor); + } + } + }); + + app.xr.on('end', () => { + if (!material.diffuseMap) + return; + + material.diffuseMap = null; + material.update(); + }) + + if (!app.xr.isAvailable(pc.XRTYPE_AR)) { + message("Immersive AR is not available"); + } else if (!app.xr.views.supportedColor) { + message("AR Camera Color is not supported"); + } else { + message("Touch screen to start AR session"); + } + } else { + message("WebXR is not supported"); + } + return app; +} + +class ArCameraColorExample { + static CATEGORY = 'XR'; + static NAME = 'AR Camera Color'; + static example = example; +} + +export { ArCameraColorExample }; diff --git a/examples/src/examples/xr/index.mjs b/examples/src/examples/xr/index.mjs index c40b96d7c91..1c9b6f43ad3 100644 --- a/examples/src/examples/xr/index.mjs +++ b/examples/src/examples/xr/index.mjs @@ -1,4 +1,5 @@ export * from "./ar-basic.mjs"; +export * from "./ar-camera-color.mjs"; export * from "./ar-hit-test.mjs"; export * from "./vr-basic.mjs"; export * from './vr-controllers.mjs'; diff --git a/src/framework/xr/constants.js b/src/framework/xr/constants.js index 0c3c0f01895..5309074cc28 100644 --- a/src/framework/xr/constants.js +++ b/src/framework/xr/constants.js @@ -100,6 +100,10 @@ export const XRTARGETRAY_SCREEN = 'screen'; */ export const XRTARGETRAY_POINTER = 'tracked-pointer'; +export const XREYE_NONE = 'none'; +export const XREYE_LEFT = 'left'; +export const XREYE_RIGHT = 'right'; + /** * None - input source is not meant to be held in hands. * diff --git a/src/framework/xr/xr-depth-sensing.js b/src/framework/xr/xr-depth-sensing.js index affc289b964..eeb232d54e4 100644 --- a/src/framework/xr/xr-depth-sensing.js +++ b/src/framework/xr/xr-depth-sensing.js @@ -268,10 +268,12 @@ class XrDepthSensing extends EventHandler { * @param {*} view - First XRView of viewer XRPose. * @ignore */ - update(frame, view) { + update(frame, pose) { if (!this._usage) return; + const view = pose.views[0]; + let depthInfoCpu = null; let depthInfoGpu = null; if (this._usage === XRDEPTHSENSINGUSAGE_CPU && view) { diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index 465b9ee8092..6a8a03e1f19 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -9,6 +9,7 @@ import { Vec3 } from '../../core/math/vec3.js'; import { Vec4 } from '../../core/math/vec4.js'; import { XRTYPE_INLINE, XRTYPE_VR, XRTYPE_AR, XRDEPTHSENSINGUSAGE_CPU, XRDEPTHSENSINGFORMAT_L8A8 } from './constants.js'; +import { DEVICETYPE_WEBGL1, DEVICETYPE_WEBGL2 } from '../../platform/graphics/constants.js'; import { XrDepthSensing } from './xr-depth-sensing.js'; import { XrDomOverlay } from './xr-dom-overlay.js'; import { XrHitTest } from './xr-hit-test.js'; @@ -17,6 +18,7 @@ import { XrInput } from './xr-input.js'; import { XrLightEstimation } from './xr-light-estimation.js'; import { XrPlaneDetection } from './xr-plane-detection.js'; import { XrAnchors } from './xr-anchors.js'; +import { XrViews } from './xr-views.js'; /** * Callback used by {@link XrManager#endXr} and {@link XrManager#startXr}. @@ -74,6 +76,12 @@ class XrManager extends EventHandler { */ _baseLayer = null; + /** + * @type {XRWebGLBinding|null} + * @ignore + */ + webglBinding = null; + /** * @type {XRReferenceSpace|null} * @ignore @@ -136,22 +144,30 @@ class XrManager extends EventHandler { lightEstimation; /** - * @type {import('../components/camera/component.js').CameraComponent} - * @private - */ - _camera = null; - - /** - * @type {Array<*>} + * Provides access to views. + * + * @type {XrViews} * @ignore */ - views = []; + views; /** - * @type {Array<*>} - * @ignore + * @type {import('../components/camera/component.js').CameraComponent|null} + * @private */ - viewsPool = []; + _camera = null; + + // /** + // * @type {Array<*>} + // * @ignore + // */ + // views = []; + + // /** + // * @type {Array<*>} + // * @ignore + // */ + // viewsPool = []; /** * @type {Vec3} @@ -213,6 +229,7 @@ class XrManager extends EventHandler { this.input = new XrInput(this); this.lightEstimation = new XrLightEstimation(this); this.anchors = new XrAnchors(this); + this.views = new XrViews(this); // TODO // 1. HMD class with its params @@ -451,6 +468,10 @@ class XrManager extends EventHandler { dataFormatPreference: dataFormatPreference }; } + + if (options && options.cameraColor && this.views.supportedColor) { + opts.optionalFeatures.push('camera-access'); + } } else if (type === XRTYPE_VR) { opts.optionalFeatures.push('hand-tracking'); } @@ -603,7 +624,7 @@ class XrManager extends EventHandler { this._session = null; this._referenceSpace = null; - this.views = []; + // this.views = []; this._width = 0; this._height = 0; this._type = null; @@ -611,7 +632,8 @@ class XrManager extends EventHandler { // old requestAnimationFrame will never be triggered, // so queue up new tick - this.app.tick(); + if (this.app.systems) + this.app.tick(); }; session.addEventListener('end', onEnd); @@ -636,6 +658,17 @@ class XrManager extends EventHandler { antialias: false }); + if (platform.browser) { + const deviceType = this.app.graphicsDevice.deviceType; + if ((deviceType === DEVICETYPE_WEBGL1 || deviceType === DEVICETYPE_WEBGL2) && window.XRWebGLBinding) { + try { + this.webglBinding = new XRWebGLBinding(session, this.app.graphicsDevice.gl); + } catch(ex) { + this.fire('error', ex); + } + } + } + session.updateRenderState({ baseLayer: this._baseLayer, depthNear: this._depthNear, @@ -705,32 +738,11 @@ class XrManager extends EventHandler { if (!pose) return false; - const lengthOld = this.views.length; - const lengthNew = pose.views.length; - - while (lengthNew > this.views.length) { - let view = this.viewsPool.pop(); - if (!view) { - view = { - viewport: new Vec4(), - projMat: new Mat4(), - viewMat: new Mat4(), - viewOffMat: new Mat4(), - viewInvMat: new Mat4(), - viewInvOffMat: new Mat4(), - projViewOffMat: new Mat4(), - viewMat3: new Mat3(), - position: new Float32Array(3), - rotation: new Quat() - }; - } + const lengthOld = this.views.size; + // const lengthNew = pose.views.length; - this.views.push(view); - } - // remove views from list into pool - while (lengthNew < this.views.length) { - this.viewsPool.push(this.views.pop()); - } + // add views + this.views.update(frame, pose.views); // reset position const posePosition = pose.transform.position; @@ -738,28 +750,10 @@ class XrManager extends EventHandler { this._localPosition.set(posePosition.x, posePosition.y, posePosition.z); this._localRotation.set(poseOrientation.x, poseOrientation.y, poseOrientation.z, poseOrientation.w); - const layer = frame.session.renderState.baseLayer; - - for (let i = 0; i < pose.views.length; i++) { - // for each view, calculate matrices - const viewRaw = pose.views[i]; - const view = this.views[i]; - const viewport = layer.getViewport(viewRaw); - - view.viewport.x = viewport.x; - view.viewport.y = viewport.y; - view.viewport.z = viewport.width; - view.viewport.w = viewport.height; - - view.projMat.set(viewRaw.projectionMatrix); - view.viewMat.set(viewRaw.transform.inverse.matrix); - view.viewInvMat.set(viewRaw.transform.matrix); - } - // update the camera fov properties only when we had 0 views - if (lengthOld === 0 && this.views.length > 0) { + if (lengthOld === 0 && this.views.size > 0) { const viewProjMat = new Mat4(); - const view = this.views[0]; + const view = this.views.list[0]; viewProjMat.copy(view.projMat); const data = viewProjMat.data; @@ -770,7 +764,6 @@ class XrManager extends EventHandler { const nearClip = data[14] / (data[10] - 1); const horizontalFov = false; - const camera = this._camera.camera; camera.setXrProperties({ aspectRatio, @@ -795,7 +788,7 @@ class XrManager extends EventHandler { this.lightEstimation.update(frame); if (this.depthSensing.supported) - this.depthSensing.update(frame, pose && pose.views[0]); + this.depthSensing.update(frame, pose); if (this.imageTracking.supported) this.imageTracking.update(frame); diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js new file mode 100644 index 00000000000..b2df2808316 --- /dev/null +++ b/src/framework/xr/xr-view.js @@ -0,0 +1,164 @@ +import { EventHandler } from "../../core/event-handler.js"; +import { Texture } from '../../platform/graphics/texture.js'; +import { Vec4 } from "../../core/math/vec4.js"; +import { Mat3 } from "../../core/math/mat3.js"; +import { Mat4 } from "../../core/math/mat4.js"; + +import { ADDRESS_CLAMP_TO_EDGE, PIXELFORMAT_RGB8, FILTER_LINEAR, PIXELFORMAT_RGBA8 } from '../../platform/graphics/constants.js'; + +class XrView extends EventHandler { + _manager; + _xrView; + + _positionData = new Float32Array(3); + _viewport = new Vec4(); + + _projMat = new Mat4(); + _projViewOffMat = new Mat4(); + _viewMat = new Mat4(); + _viewOffMat = new Mat4(); + _viewMat3 = new Mat3(); + _viewInvMat = new Mat4(); + _viewInvOffMat = new Mat4(); + + _xrCamera = null; + _textureColor = null; + + constructor(manager, xrView) { + super(); + + this._manager = manager; + this._xrView = xrView; + + if (this._manager.views.supportedColor) + this._xrCamera = this._xrView.camera; + + this._updateTextureColor(); + } + + _updateTextureColor() { + if (!this._manager.views.availableColor || !this._xrCamera) + return; + + const binding = this._manager.webglBinding; + if (!binding) + return; + + const texture = binding.getCameraImage(this._xrCamera); + if (!texture) + return; + + if (!this._textureColor) { + this._textureColor = new Texture(this._manager.app.graphicsDevice, { + format: PIXELFORMAT_RGBA8, + mipmaps: false, + flipY: true, + addressU: ADDRESS_CLAMP_TO_EDGE, + addressV: ADDRESS_CLAMP_TO_EDGE, + minFilter: FILTER_LINEAR, + magFilter: FILTER_LINEAR, + width: this._xrCamera.width, + height: this._xrCamera.height, + name: `XrView-${this._xrView.eye}-Color` + }); + this._textureColor.upload(); + } + + // force texture initialization + if (!this._textureColor.impl._glTexture) { + this._textureColor.impl.initialize(this._manager.app.graphicsDevice, this._textureColor); + this._textureColor.impl.upload = () => { }; + this._textureColor._needsUpload = false; + } + + this._textureColor.impl._glCreated = true; + this._textureColor.impl._glTexture = texture; + } + + updateTransforms(transform) { + if (transform) { + this._viewInvOffMat.mul2(transform, this._viewInvMat); + this.viewOffMat.copy(this._viewInvOffMat).invert(); + } else { + this._viewInvOffMat.copy(this._viewInvMat); + this.viewOffMat.copy(this._viewMat); + } + + this._viewMat3.setFromMat4(this._viewOffMat); + this._projViewOffMat.mul2(this._projMat, this._viewOffMat); + + this._positionData[0] = this._viewInvOffMat.data[12]; + this._positionData[1] = this._viewInvOffMat.data[13]; + this._positionData[2] = this._viewInvOffMat.data[14]; + } + + update(frame, xrView) { + this._xrView = xrView; + if (this._manager.views.supportedColor) + this._xrCamera = this._xrView.camera; + + const layer = frame.session.renderState.baseLayer; + + // viewport + const viewport = layer.getViewport(this._xrView); + this._viewport.x = viewport.x; + this._viewport.y = viewport.y; + this._viewport.z = viewport.width; + this._viewport.w = viewport.height; + + // matrices + this._projMat.set(this._xrView.projectionMatrix); + this._viewMat.set(this._xrView.transform.inverse.matrix); + this._viewInvMat.set(this._xrView.transform.matrix); + + this._updateTextureColor(); + } + + destroy() { + if (this._textureColor) { + // TODO + // ensure there is no use of this texture after session ended + this._textureColor.impl._glTexture = null; + this._textureColor.destroy(); + this._textureColor = null; + } + } + + get textureColor() { + return this._textureColor; + } + + get eye() { + return this._xrView.eye; + } + + get viewport() { + return this._viewport; + } + + get projMat() { + return this._projMat; + } + + get projViewOffMat() { + return this._projViewOffMat; + } + + get viewOffMat() { + return this._viewOffMat; + } + + get viewInvOffMat() { + return this._viewInvOffMat; + } + + get viewMat3() { + return this._viewMat3; + } + + get positionData() { + return this._positionData; + } +} + +export { XrView }; diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js new file mode 100644 index 00000000000..ee5d6e2465f --- /dev/null +++ b/src/framework/xr/xr-views.js @@ -0,0 +1,94 @@ +import { platform } from '../../core/platform.js'; +import { EventHandler } from "../../core/event-handler.js"; +import { XrView } from "./xr-view.js"; +import { XRTYPE_AR } from "./constants.js"; + +class XrViews extends EventHandler { + _manager; + _index = new Map(); + _list = []; + _indexTemporary = new Map(); + _supportedColor = platform.browser && !!window.XRCamera && !!window.XRWebGLBinding; + _availableColor = false; + + constructor(manager) { + super(); + + this._manager = manager; + this._manager.on('start', this._onSessionStart, this); + this._manager.on('end', this._onSessionEnd, this); + } + + _onSessionStart() { + if (this._manager.type !== XRTYPE_AR) + return; + this._availableColor = this._manager.session.enabledFeatures.indexOf('camera-access') !== -1; + } + + _onSessionEnd() { + for(const view of this._index.values()) { + view.destroy(); + } + this._index.clear(); + this._availableColor = false; + this._list.length = 0; + } + + update(frame, xrViews) { + for(let i = 0; i < xrViews.length; i++) { + this._indexTemporary.set(xrViews[i].eye, xrViews[i]); + } + + for(const [ eye, xrView ] of this._indexTemporary) { + let view = this._index.get(eye); + + if (!view) { + // add new view + view = new XrView(this._manager, xrView); + this._index.set(eye, view); + this._list.push(view); + view.update(frame, xrView); + this.fire('add', view); + } else { + // update existing view0 + view.update(frame, xrView); + } + } + + // remove views + for(const [ eye, view ] of this._index) { + if (this._indexTemporary.has(eye)) + continue; + + view.destroy(); + this._index.delete(eye); + const ind = this._list.indexOf(view); + if (ind !== -1) this._list.splice(ind, 1); + this.fire('remove', view); + } + + this._indexTemporary.clear(); + } + + get(name) { + return this._index.get(name) || null; + } + + get list() { + return this._list; + } + + get size() { + return this._list.length; + } + + get supportedColor() { + return this._supportedColor; + } + + get availableColor() { + return this._availableColor; + } +} + +export { XrViews }; diff --git a/src/scene/renderer/forward-renderer.js b/src/scene/renderer/forward-renderer.js index 79191c46b12..6b6d14c0cce 100644 --- a/src/scene/renderer/forward-renderer.js +++ b/src/scene/renderer/forward-renderer.js @@ -621,11 +621,11 @@ class ForwardRenderer extends Renderer { drawCallback?.(drawCall, i); - if (camera.xr && camera.xr.session && camera.xr.views.length) { + if (camera.xr && camera.xr.session && camera.xr.views.size) { const views = camera.xr.views; - for (let v = 0; v < views.length; v++) { - const view = views[v]; + for (let v = 0; v < views.size; v++) { + const view = views.list[v]; device.setViewport(view.viewport.x, view.viewport.y, view.viewport.z, view.viewport.w); @@ -635,7 +635,7 @@ class ForwardRenderer extends Renderer { this.viewInvId.setValue(view.viewInvOffMat.data); this.viewId3.setValue(view.viewMat3.data); this.viewProjId.setValue(view.projViewOffMat.data); - this.viewPosId.setValue(view.position); + this.viewPosId.setValue(view.positionData); if (v === 0) { this.drawInstance(device, drawCall, mesh, style, true); diff --git a/src/scene/renderer/renderer.js b/src/scene/renderer/renderer.js index a5c08aa1cc3..842d4079077 100644 --- a/src/scene/renderer/renderer.js +++ b/src/scene/renderer/renderer.js @@ -295,31 +295,12 @@ class Renderer { let viewCount = 1; if (camera.xr && camera.xr.session) { - let transform; - const parent = camera._node.parent; - if (parent) - transform = parent.getWorldTransform(); - + const transform = camera._node?.parent?.getWorldTransform() || null; const views = camera.xr.views; - viewCount = views.length; + viewCount = views.size; for (let v = 0; v < viewCount; v++) { - const view = views[v]; - - if (parent) { - view.viewInvOffMat.mul2(transform, view.viewInvMat); - view.viewOffMat.copy(view.viewInvOffMat).invert(); - } else { - view.viewInvOffMat.copy(view.viewInvMat); - view.viewOffMat.copy(view.viewMat); - } - - view.viewMat3.setFromMat4(view.viewOffMat); - view.projViewOffMat.mul2(view.projMat, view.viewOffMat); - - view.position[0] = view.viewInvOffMat.data[12]; - view.position[1] = view.viewInvOffMat.data[13]; - view.position[2] = view.viewInvOffMat.data[14]; - + const view = views.list[v]; + view.updateTransforms(transform); camera.frustum.setFromMat4(view.projViewOffMat); } } else { @@ -497,9 +478,9 @@ class Renderer { updateCameraFrustum(camera) { - if (camera.xr && camera.xr.views.length) { + if (camera.xr && camera.xr.views.size) { // calculate frustum based on XR view - const view = camera.xr.views[0]; + const view = camera.xr.views.list[0]; viewProjMat.mul2(view.projMat, view.viewOffMat); camera.frustum.setFromMat4(viewProjMat); return; From 40877655d129ff58f227d0a68840644283ee0960 Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Sun, 5 Nov 2023 15:46:38 +0200 Subject: [PATCH 2/8] docs and lint --- examples/src/examples/xr/ar-basic.mjs | 7 - examples/src/examples/xr/ar-camera-color.mjs | 8 +- src/framework/xr/constants.js | 17 ++ src/framework/xr/xr-depth-sensing.js | 4 +- src/framework/xr/xr-manager.js | 24 +- src/framework/xr/xr-view.js | 267 ++++++++++++++----- src/framework/xr/xr-views.js | 148 +++++++--- 7 files changed, 346 insertions(+), 129 deletions(-) diff --git a/examples/src/examples/xr/ar-basic.mjs b/examples/src/examples/xr/ar-basic.mjs index c41851c4806..ad68776a507 100644 --- a/examples/src/examples/xr/ar-basic.mjs +++ b/examples/src/examples/xr/ar-basic.mjs @@ -15,13 +15,6 @@ async function example({ canvas }) { if (!el) { el = document.createElement('div'); el.classList.add('message'); - el.style.position = 'absolute'; - el.style.bottom = '96px'; - el.style.right = '0'; - el.style.padding = '8px 16px'; - el.style.fontFamily = 'Helvetica, Arial, sans-serif'; - el.style.color = '#fff'; - el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; document.body.append(el); } el.textContent = msg; diff --git a/examples/src/examples/xr/ar-camera-color.mjs b/examples/src/examples/xr/ar-camera-color.mjs index 00c1b5941f3..5c1b9ae5ec3 100644 --- a/examples/src/examples/xr/ar-camera-color.mjs +++ b/examples/src/examples/xr/ar-camera-color.mjs @@ -95,7 +95,7 @@ async function example({ canvas }) { const activate = function () { if (app.xr.isAvailable(pc.XRTYPE_AR)) { c.camera.startXr(pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { - cameraColor: true, + cameraColor: true, // request access to camera color callback: function (err) { if (err) message("WebXR Immersive AR failed to start: " + err.message); } @@ -151,17 +151,20 @@ async function example({ canvas }) { }); app.on('update', () => { + // if camera color is available if (app.xr.views.availableColor) { for(let i = 0; i < app.xr.views.size; i++) { const view = app.xr.views.list[i]; - if (!view.textureColor) + if (!view.textureColor) // check if color texture is available continue; + // apply camera color texture to material diffuse channel if (!material.diffuseMap) { material.diffuseMap = view.textureColor; material.update(); } + // debug draw camera color texture on the screen app.drawTexture(0.5, -0.5, 1, -1, view.textureColor); } } @@ -171,6 +174,7 @@ async function example({ canvas }) { if (!material.diffuseMap) return; + // clear camera color texture when XR session ends material.diffuseMap = null; material.update(); }) diff --git a/src/framework/xr/constants.js b/src/framework/xr/constants.js index 5309074cc28..fe464eed166 100644 --- a/src/framework/xr/constants.js +++ b/src/framework/xr/constants.js @@ -100,8 +100,25 @@ export const XRTARGETRAY_SCREEN = 'screen'; */ export const XRTARGETRAY_POINTER = 'tracked-pointer'; +/** + * None - view associated with a monoscopic screen, such as mobile phone screens. + * + * @type {string} + */ export const XREYE_NONE = 'none'; + +/** + * Left - view associated with left eye. + * + * @type {string} + */ export const XREYE_LEFT = 'left'; + +/** + * Right - view associated with right eye. + * + * @type {string} + */ export const XREYE_RIGHT = 'right'; /** diff --git a/src/framework/xr/xr-depth-sensing.js b/src/framework/xr/xr-depth-sensing.js index eeb232d54e4..affc289b964 100644 --- a/src/framework/xr/xr-depth-sensing.js +++ b/src/framework/xr/xr-depth-sensing.js @@ -268,12 +268,10 @@ class XrDepthSensing extends EventHandler { * @param {*} view - First XRView of viewer XRPose. * @ignore */ - update(frame, pose) { + update(frame, view) { if (!this._usage) return; - const view = pose.views[0]; - let depthInfoCpu = null; let depthInfoGpu = null; if (this._usage === XRDEPTHSENSINGUSAGE_CPU && view) { diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index 6a8a03e1f19..11b6fdb71b3 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -2,11 +2,9 @@ import { Debug } from "../../core/debug.js"; import { EventHandler } from '../../core/event-handler.js'; import { platform } from '../../core/platform.js'; -import { Mat3 } from '../../core/math/mat3.js'; import { Mat4 } from '../../core/math/mat4.js'; import { Quat } from '../../core/math/quat.js'; import { Vec3 } from '../../core/math/vec3.js'; -import { Vec4 } from '../../core/math/vec4.js'; import { XRTYPE_INLINE, XRTYPE_VR, XRTYPE_AR, XRDEPTHSENSINGUSAGE_CPU, XRDEPTHSENSINGFORMAT_L8A8 } from './constants.js'; import { DEVICETYPE_WEBGL1, DEVICETYPE_WEBGL2 } from '../../platform/graphics/constants.js'; @@ -144,7 +142,7 @@ class XrManager extends EventHandler { lightEstimation; /** - * Provides access to views. + * Provides access to views and their capabilities. * * @type {XrViews} * @ignore @@ -157,18 +155,6 @@ class XrManager extends EventHandler { */ _camera = null; - // /** - // * @type {Array<*>} - // * @ignore - // */ - // views = []; - - // /** - // * @type {Array<*>} - // * @ignore - // */ - // viewsPool = []; - /** * @type {Vec3} * @private @@ -624,7 +610,6 @@ class XrManager extends EventHandler { this._session = null; this._referenceSpace = null; - // this.views = []; this._width = 0; this._height = 0; this._type = null; @@ -662,8 +647,8 @@ class XrManager extends EventHandler { const deviceType = this.app.graphicsDevice.deviceType; if ((deviceType === DEVICETYPE_WEBGL1 || deviceType === DEVICETYPE_WEBGL2) && window.XRWebGLBinding) { try { - this.webglBinding = new XRWebGLBinding(session, this.app.graphicsDevice.gl); - } catch(ex) { + this.webglBinding = new XRWebGLBinding(session, this.app.graphicsDevice.gl); // eslint-disable-line no-undef + } catch (ex) { this.fire('error', ex); } } @@ -739,7 +724,6 @@ class XrManager extends EventHandler { if (!pose) return false; const lengthOld = this.views.size; - // const lengthNew = pose.views.length; // add views this.views.update(frame, pose.views); @@ -788,7 +772,7 @@ class XrManager extends EventHandler { this.lightEstimation.update(frame); if (this.depthSensing.supported) - this.depthSensing.update(frame, pose); + this.depthSensing.update(frame, pose && pose.views[0]); if (this.imageTracking.supported) this.imageTracking.update(frame); diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js index b2df2808316..ce764487942 100644 --- a/src/framework/xr/xr-view.js +++ b/src/framework/xr/xr-view.js @@ -1,32 +1,103 @@ -import { EventHandler } from "../../core/event-handler.js"; import { Texture } from '../../platform/graphics/texture.js'; import { Vec4 } from "../../core/math/vec4.js"; import { Mat3 } from "../../core/math/mat3.js"; import { Mat4 } from "../../core/math/mat4.js"; -import { ADDRESS_CLAMP_TO_EDGE, PIXELFORMAT_RGB8, FILTER_LINEAR, PIXELFORMAT_RGBA8 } from '../../platform/graphics/constants.js'; - -class XrView extends EventHandler { +import { ADDRESS_CLAMP_TO_EDGE, FILTER_LINEAR, PIXELFORMAT_RGB8 } from '../../platform/graphics/constants.js'; + +/** + * Represents XR View which represents a screen (mobile phone context) or an eye (HMD context). + * + * @category XR + */ +class XrView { + /** + * @type {import('./xr-manager.js').XrManager} + * @private + */ _manager; + + /** + * @type {XRView} + * @private + */ _xrView; + /** + * @type {Float32Array} + * @private + */ _positionData = new Float32Array(3); + + /** + * @type {Vec4} + * @private + */ _viewport = new Vec4(); + /** + * @type {Mat4} + * @private + */ _projMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ _projViewOffMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ _viewMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ _viewOffMat = new Mat4(); + + /** + * @type {Mat3} + * @private + */ _viewMat3 = new Mat3(); + + /** + * @type {Mat4} + * @private + */ _viewInvMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ _viewInvOffMat = new Mat4(); + /** + * @type {XRCamera} + * @private + */ _xrCamera = null; + + /** + * @type {Texture|null} + * @private + */ _textureColor = null; + /** + * Create a new XrView instance. + * + * @param {import('./xr-manager.js').XrManager} manager - WebXR Manager. + * @param {XRView} xrView - [XRView](https://developer.mozilla.org/en-US/docs/Web/API/XRView) + * object that is created by WebXR API. + * @hideconstructor + */ constructor(manager, xrView) { - super(); - this._manager = manager; this._xrView = xrView; @@ -36,6 +107,121 @@ class XrView extends EventHandler { this._updateTextureColor(); } + /** + * Texture associated with this view's camera color. Equals to null if camera color is + * not available or not supported. + * + * @type {Texture|null} + * @readonly + */ + get textureColor() { + return this._textureColor; + } + + /** + * An eye with which this view is associated. Can be any of: + * + * - {@link XREYE_NONE}: None - inidcates a monoscopic view (likely mobile phone screen). + * - {@link XREYE_LEFT}: Left - indicates left eye view. + * - {@link XREYE_RIGHT}: Right - indicates a right eye view. + * + * @type {string} + * @readonly + */ + get eye() { + return this._xrView.eye; + } + + /** + * A Vec4 (x, y, width, height) that represents a view's viewport. For monoscopic screen + * it will define fullscreen view, but for stereoscopic views (left/right eye) it will define + * a part of a whole screen that view is occupying. + * + * @type {Vec4} + * @readonly + */ + get viewport() { + return this._viewport; + } + + /** + * @type {Mat4} + * @ignore + */ + get projMat() { + return this._projMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get projViewOffMat() { + return this._projViewOffMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get viewOffMat() { + return this._viewOffMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get viewInvOffMat() { + return this._viewInvOffMat; + } + + /** + * @type {Mat3} + * @ignore + */ + get viewMat3() { + return this._viewMat3; + } + + /** + * @type {Float32Array} + * @ignore + */ + get positionData() { + return this._positionData; + } + + /** + * @param {*} frame - XRFrame from requestAnimationFrame callback. + * @param {XRView} xrView - XRView from WebXR API. + * @ignore + */ + update(frame, xrView) { + this._xrView = xrView; + if (this._manager.views.supportedColor) + this._xrCamera = this._xrView.camera; + + const layer = frame.session.renderState.baseLayer; + + // viewport + const viewport = layer.getViewport(this._xrView); + this._viewport.x = viewport.x; + this._viewport.y = viewport.y; + this._viewport.z = viewport.width; + this._viewport.w = viewport.height; + + // matrices + this._projMat.set(this._xrView.projectionMatrix); + this._viewMat.set(this._xrView.transform.inverse.matrix); + this._viewInvMat.set(this._xrView.transform.matrix); + + this._updateTextureColor(); + } + + /** + * @private + */ _updateTextureColor() { if (!this._manager.views.availableColor || !this._xrCamera) return; @@ -50,9 +236,9 @@ class XrView extends EventHandler { if (!this._textureColor) { this._textureColor = new Texture(this._manager.app.graphicsDevice, { - format: PIXELFORMAT_RGBA8, + format: PIXELFORMAT_RGB8, mipmaps: false, - flipY: true, + flipY: false, addressU: ADDRESS_CLAMP_TO_EDGE, addressV: ADDRESS_CLAMP_TO_EDGE, minFilter: FILTER_LINEAR, @@ -75,6 +261,10 @@ class XrView extends EventHandler { this._textureColor.impl._glTexture = texture; } + /** + * @param {Mat4|null} transform - World Transform of a parents GraphNode. + * @ignore + */ updateTransforms(transform) { if (transform) { this._viewInvOffMat.mul2(transform, this._viewInvMat); @@ -92,28 +282,9 @@ class XrView extends EventHandler { this._positionData[2] = this._viewInvOffMat.data[14]; } - update(frame, xrView) { - this._xrView = xrView; - if (this._manager.views.supportedColor) - this._xrCamera = this._xrView.camera; - - const layer = frame.session.renderState.baseLayer; - - // viewport - const viewport = layer.getViewport(this._xrView); - this._viewport.x = viewport.x; - this._viewport.y = viewport.y; - this._viewport.z = viewport.width; - this._viewport.w = viewport.height; - - // matrices - this._projMat.set(this._xrView.projectionMatrix); - this._viewMat.set(this._xrView.transform.inverse.matrix); - this._viewInvMat.set(this._xrView.transform.matrix); - - this._updateTextureColor(); - } - + /** + * @ignore + */ destroy() { if (this._textureColor) { // TODO @@ -123,42 +294,6 @@ class XrView extends EventHandler { this._textureColor = null; } } - - get textureColor() { - return this._textureColor; - } - - get eye() { - return this._xrView.eye; - } - - get viewport() { - return this._viewport; - } - - get projMat() { - return this._projMat; - } - - get projViewOffMat() { - return this._projViewOffMat; - } - - get viewOffMat() { - return this._viewOffMat; - } - - get viewInvOffMat() { - return this._viewInvOffMat; - } - - get viewMat3() { - return this._viewMat3; - } - - get positionData() { - return this._positionData; - } } export { XrView }; diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js index ee5d6e2465f..f88176e9b01 100644 --- a/src/framework/xr/xr-views.js +++ b/src/framework/xr/xr-views.js @@ -3,14 +3,53 @@ import { EventHandler } from "../../core/event-handler.js"; import { XrView } from "./xr-view.js"; import { XRTYPE_AR } from "./constants.js"; +/** + * Provides access to list of {@link XrView}'s. And information about their capabilities, + * such as support and availability of view's camera color texture. + * + * @category XR + */ class XrViews extends EventHandler { + /** + * @type {import('./xr-manager.js').XrManager} + * @private + */ _manager; + + /** + * @type {Map} + * @private + */ _index = new Map(); + + /** + * @type {Map} + * @private + */ + _indexTmp = new Map(); + + /** + * @type {XrView[]} + * @private + */ _list = []; - _indexTemporary = new Map(); + + /** + * @type {boolean} + * @private + */ _supportedColor = platform.browser && !!window.XRCamera && !!window.XRWebGLBinding; + + /** + * @type {boolean} + * @private + */ _availableColor = false; + /** + * @param {import('./xr-manager.js').XrManager} manager - WebXR Manager. + * @hideconstructor + */ constructor(manager) { super(); @@ -19,27 +58,65 @@ class XrViews extends EventHandler { this._manager.on('end', this._onSessionEnd, this); } - _onSessionStart() { - if (this._manager.type !== XRTYPE_AR) - return; - this._availableColor = this._manager.session.enabledFeatures.indexOf('camera-access') !== -1; + // TODO + // events + + /** + * An array of {@link XrView}'s of this session. Views are not available straight + * away on session start, and can be added/removed mid-session. So use of add/remove + * events is required for accessing views. + * + * @type {XrView[]} + * @readonly + */ + get list() { + return this._list; } - _onSessionEnd() { - for(const view of this._index.values()) { - view.destroy(); - } - this._index.clear(); - this._availableColor = false; - this._list.length = 0; + /** + * How many views are available. Views can be added/removed mid-session by underlying + * WebXR system. + * + * @type {number} + * @readonly + */ + get size() { + return this._list.length; } + /** + * Check if Camera Color is supported. It might be still unavailable even if requested, + * based on hardware capabilities and granted permissions. + * + * @type {boolean} + * @readonly + */ + get supportedColor() { + return this._supportedColor; + } + + /** + * Check if Camera Color is available. This information becomes available only after + * session has started. + * + * @type {boolean} + * @readonly + */ + get availableColor() { + return this._availableColor; + } + + /** + * @param {*} frame - XRFrame from requestAnimationFrame callback. + * @param {XRView} xrView - XRView from WebXR API. + * @ignore + */ update(frame, xrViews) { - for(let i = 0; i < xrViews.length; i++) { - this._indexTemporary.set(xrViews[i].eye, xrViews[i]); + for (let i = 0; i < xrViews.length; i++) { + this._indexTmp.set(xrViews[i].eye, xrViews[i]); } - for(const [ eye, xrView ] of this._indexTemporary) { + for (const [eye, xrView] of this._indexTmp) { let view = this._index.get(eye); if (!view) { @@ -56,8 +133,8 @@ class XrViews extends EventHandler { } // remove views - for(const [ eye, view ] of this._index) { - if (this._indexTemporary.has(eye)) + for (const [eye, view] of this._index) { + if (this._indexTmp.has(eye)) continue; view.destroy(); @@ -67,27 +144,36 @@ class XrViews extends EventHandler { this.fire('remove', view); } - this._indexTemporary.clear(); + this._indexTmp.clear(); } - get(name) { + /** + * @param {string} eye - An XREYE_* view is associated with. Can be 'none' for monoscope views. + * @returns {XrView|null} View or null if view of such eye is not available. + */ + get(eye) { return this._index.get(name) || null; } - get list() { - return this._list; - } - - get size() { - return this._list.length; - } - - get supportedColor() { - return this._supportedColor; + /** + * @private + */ + _onSessionStart() { + if (this._manager.type !== XRTYPE_AR) + return; + this._availableColor = this._manager.session.enabledFeatures.indexOf('camera-access') !== -1; } - get availableColor() { - return this._availableColor; + /** + * @private + */ + _onSessionEnd() { + for (const view of this._index.values()) { + view.destroy(); + } + this._index.clear(); + this._availableColor = false; + this._list.length = 0; } } From 7b0007b94ae33598dd8f57bb673056ca4c05bcf0 Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Sun, 5 Nov 2023 15:53:35 +0200 Subject: [PATCH 3/8] xr views events --- src/framework/xr/xr-views.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js index f88176e9b01..bf46c249081 100644 --- a/src/framework/xr/xr-views.js +++ b/src/framework/xr/xr-views.js @@ -58,8 +58,28 @@ class XrViews extends EventHandler { this._manager.on('end', this._onSessionEnd, this); } - // TODO - // events + /** + * Fired when view has been added. Views are not available straight away on session start + * and are added mid-session. They can be added/removed mid session by underlyng system. + * + * @event XrViews#add + * @param {XrView} view - XrView that has been added. + * @example + * xr.views.on('add', function (view) { + * // view that has been added + * }); + */ + + /** + * Fired when view has been removed. They can be added/removed mid session by underlyng system. + * + * @event XrViews#remove + * @param {XrView} view - XrView that has been removed. + * @example + * xr.views.on('remove', function (view) { + * // view that has been added + * }); + */ /** * An array of {@link XrView}'s of this session. Views are not available straight From 368d551006da6f7db3ec02212976ed98fb79e271 Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Thu, 9 Nov 2023 13:30:07 +0200 Subject: [PATCH 4/8] implement camera color texture copying --- examples/src/examples/xr/ar-camera-color.mjs | 2 +- src/framework/xr/xr-manager.js | 4 +- src/framework/xr/xr-view.js | 75 +++++++++++++++----- src/framework/xr/xr-views.js | 4 -- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/examples/src/examples/xr/ar-camera-color.mjs b/examples/src/examples/xr/ar-camera-color.mjs index 5c1b9ae5ec3..ddfb4f369cb 100644 --- a/examples/src/examples/xr/ar-camera-color.mjs +++ b/examples/src/examples/xr/ar-camera-color.mjs @@ -165,7 +165,7 @@ async function example({ canvas }) { } // debug draw camera color texture on the screen - app.drawTexture(0.5, -0.5, 1, -1, view.textureColor); + app.drawTexture(0.5, -0.5, 1, 1, view.textureColor); } } }); diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index 64bcfade0e3..15d02155783 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -417,6 +417,8 @@ class XrManager extends EventHandler { optionalFeatures: [] }; + const webgl = this.app.graphicsDevice?.isWebGL1 || this.app.graphicsDevice?.isWebGL2; + if (type === XRTYPE_AR) { opts.optionalFeatures.push('light-estimation'); opts.optionalFeatures.push('hit-test'); @@ -462,7 +464,7 @@ class XrManager extends EventHandler { }; } - if (options && options.cameraColor && this.views.supportedColor) { + if (webgl && options && options.cameraColor && this.views.supportedColor) { opts.optionalFeatures.push('camera-access'); } } else if (type === XRTYPE_VR) { diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js index ce764487942..6eae5da01c9 100644 --- a/src/framework/xr/xr-view.js +++ b/src/framework/xr/xr-view.js @@ -112,7 +112,6 @@ class XrView { * not available or not supported. * * @type {Texture|null} - * @readonly */ get textureColor() { return this._textureColor; @@ -126,7 +125,6 @@ class XrView { * - {@link XREYE_RIGHT}: Right - indicates a right eye view. * * @type {string} - * @readonly */ get eye() { return this._xrView.eye; @@ -138,7 +136,6 @@ class XrView { * a part of a whole screen that view is occupying. * * @type {Vec4} - * @readonly */ get viewport() { return this._viewport; @@ -199,7 +196,7 @@ class XrView { */ update(frame, xrView) { this._xrView = xrView; - if (this._manager.views.supportedColor) + if (this._manager.views.availableColor) this._xrCamera = this._xrView.camera; const layer = frame.session.renderState.baseLayer; @@ -234,31 +231,68 @@ class XrView { if (!texture) return; + const device = this._manager.app.graphicsDevice; + const gl = device.gl; + const attachmentBaseConstant = device.isWebGL2 ? gl.COLOR_ATTACHMENT0 : (device.extDrawBuffers?.COLOR_ATTACHMENT0_WEBGL ?? gl.COLOR_ATTACHMENT0); + + const width = this._xrCamera.width; + const height = this._xrCamera.height; + if (!this._textureColor) { + // color texture this._textureColor = new Texture(this._manager.app.graphicsDevice, { format: PIXELFORMAT_RGB8, mipmaps: false, - flipY: false, addressU: ADDRESS_CLAMP_TO_EDGE, addressV: ADDRESS_CLAMP_TO_EDGE, minFilter: FILTER_LINEAR, magFilter: FILTER_LINEAR, - width: this._xrCamera.width, - height: this._xrCamera.height, + width: width, + height: height, name: `XrView-${this._xrView.eye}-Color` }); + + // force initialize texture this._textureColor.upload(); - } - // force texture initialization - if (!this._textureColor.impl._glTexture) { - this._textureColor.impl.initialize(this._manager.app.graphicsDevice, this._textureColor); - this._textureColor.impl.upload = () => { }; - this._textureColor._needsUpload = false; + // create frame buffer to read from + this._frameBufferSource = gl.createFramebuffer(); + + // create frame buffer to write to + this._frameBuffer = gl.createFramebuffer(); } - this._textureColor.impl._glCreated = true; - this._textureColor.impl._glTexture = texture; + // set frame buffer to read from + device.setFramebuffer(this._frameBufferSource); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + texture, + 0 + ); + + // set frame buffer to write to + device.setFramebuffer(this._frameBuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + this._textureColor.impl._glTexture, + 0 + ); + + // bind buffers + gl.bindFramebuffer(gl.READ_FRAMEBUFFER, this._frameBufferSource); + let ready = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE; + + gl.bindFramebuffer(gl.DRAW_FRAMEBUFFER, this._frameBuffer); + if (ready) ready = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE; + + if (ready) { + // copy buffers with flip Y + gl.blitFramebuffer(0, height, width, 0, 0, 0, width, height, gl.COLOR_BUFFER_BIT, gl.NEAREST); + } } /** @@ -287,11 +321,16 @@ class XrView { */ destroy() { if (this._textureColor) { - // TODO - // ensure there is no use of this texture after session ended - this._textureColor.impl._glTexture = null; this._textureColor.destroy(); this._textureColor = null; + + const gl = this._manager.app.graphicsDevice.gl; + + gl.deleteFramebuffer(this._frameBufferSource); + this._frameBufferSource = null; + + gl.deleteFramebuffer(this._frameBuffer); + this._frameBuffer = null; } } } diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js index bf46c249081..670d48dab4f 100644 --- a/src/framework/xr/xr-views.js +++ b/src/framework/xr/xr-views.js @@ -87,7 +87,6 @@ class XrViews extends EventHandler { * events is required for accessing views. * * @type {XrView[]} - * @readonly */ get list() { return this._list; @@ -98,7 +97,6 @@ class XrViews extends EventHandler { * WebXR system. * * @type {number} - * @readonly */ get size() { return this._list.length; @@ -109,7 +107,6 @@ class XrViews extends EventHandler { * based on hardware capabilities and granted permissions. * * @type {boolean} - * @readonly */ get supportedColor() { return this._supportedColor; @@ -120,7 +117,6 @@ class XrViews extends EventHandler { * session has started. * * @type {boolean} - * @readonly */ get availableColor() { return this._availableColor; From fc1f31d64c8c9f962d6331833ccf253dc4ea6afc Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Fri, 24 Nov 2023 11:21:03 +0200 Subject: [PATCH 5/8] PR comments --- examples/src/examples/xr/ar-camera-color.mjs | 2 +- src/framework/xr/xr-manager.js | 4 ++-- src/framework/xr/xr-view.js | 2 +- src/framework/xr/xr-views.js | 10 ---------- src/scene/renderer/forward-renderer.js | 4 ++-- src/scene/renderer/renderer.js | 4 ++-- 6 files changed, 8 insertions(+), 18 deletions(-) diff --git a/examples/src/examples/xr/ar-camera-color.mjs b/examples/src/examples/xr/ar-camera-color.mjs index ddfb4f369cb..ffc06325413 100644 --- a/examples/src/examples/xr/ar-camera-color.mjs +++ b/examples/src/examples/xr/ar-camera-color.mjs @@ -153,7 +153,7 @@ async function example({ canvas }) { app.on('update', () => { // if camera color is available if (app.xr.views.availableColor) { - for(let i = 0; i < app.xr.views.size; i++) { + for(let i = 0; i < app.xr.views.list.length; i++) { const view = app.xr.views.list[i]; if (!view.textureColor) // check if color texture is available continue; diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index c16fa12d499..b7353c862a1 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -773,7 +773,7 @@ class XrManager extends EventHandler { if (!pose) return false; - const lengthOld = this.views.size; + const lengthOld = this.views.list.length; // add views this.views.update(frame, pose.views); @@ -785,7 +785,7 @@ class XrManager extends EventHandler { this._localRotation.set(poseOrientation.x, poseOrientation.y, poseOrientation.z, poseOrientation.w); // update the camera fov properties only when we had 0 views - if (lengthOld === 0 && this.views.size > 0) { + if (lengthOld === 0 && this.views.list.length > 0) { const viewProjMat = new Mat4(); const view = this.views.list[0]; diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js index 6eae5da01c9..845322ca964 100644 --- a/src/framework/xr/xr-view.js +++ b/src/framework/xr/xr-view.js @@ -240,7 +240,7 @@ class XrView { if (!this._textureColor) { // color texture - this._textureColor = new Texture(this._manager.app.graphicsDevice, { + this._textureColor = new Texture(device, { format: PIXELFORMAT_RGB8, mipmaps: false, addressU: ADDRESS_CLAMP_TO_EDGE, diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js index 670d48dab4f..5ccb265712f 100644 --- a/src/framework/xr/xr-views.js +++ b/src/framework/xr/xr-views.js @@ -92,16 +92,6 @@ class XrViews extends EventHandler { return this._list; } - /** - * How many views are available. Views can be added/removed mid-session by underlying - * WebXR system. - * - * @type {number} - */ - get size() { - return this._list.length; - } - /** * Check if Camera Color is supported. It might be still unavailable even if requested, * based on hardware capabilities and granted permissions. diff --git a/src/scene/renderer/forward-renderer.js b/src/scene/renderer/forward-renderer.js index 6b6d14c0cce..8e459a3f20e 100644 --- a/src/scene/renderer/forward-renderer.js +++ b/src/scene/renderer/forward-renderer.js @@ -621,10 +621,10 @@ class ForwardRenderer extends Renderer { drawCallback?.(drawCall, i); - if (camera.xr && camera.xr.session && camera.xr.views.size) { + if (camera.xr && camera.xr.session && camera.xr.views.list.length) { const views = camera.xr.views; - for (let v = 0; v < views.size; v++) { + for (let v = 0; v < views.list.length; v++) { const view = views.list[v]; device.setViewport(view.viewport.x, view.viewport.y, view.viewport.z, view.viewport.w); diff --git a/src/scene/renderer/renderer.js b/src/scene/renderer/renderer.js index 842d4079077..e2219712fed 100644 --- a/src/scene/renderer/renderer.js +++ b/src/scene/renderer/renderer.js @@ -297,7 +297,7 @@ class Renderer { if (camera.xr && camera.xr.session) { const transform = camera._node?.parent?.getWorldTransform() || null; const views = camera.xr.views; - viewCount = views.size; + viewCount = views.list.length; for (let v = 0; v < viewCount; v++) { const view = views.list[v]; view.updateTransforms(transform); @@ -478,7 +478,7 @@ class Renderer { updateCameraFrustum(camera) { - if (camera.xr && camera.xr.views.size) { + if (camera.xr && camera.xr.views.list.length) { // calculate frustum based on XR view const view = camera.xr.views.list[0]; viewProjMat.mul2(view.projMat, view.viewOffMat); From 5f0a8057a0cc610dec3278286a34d8f48c28e67f Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Fri, 24 Nov 2023 14:36:29 +0200 Subject: [PATCH 6/8] avoid FBO checks --- src/framework/xr/xr-view.js | 101 +++++++++++++++++------------------- 1 file changed, 48 insertions(+), 53 deletions(-) diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js index 845322ca964..78cbcd37a57 100644 --- a/src/framework/xr/xr-view.js +++ b/src/framework/xr/xr-view.js @@ -101,10 +101,24 @@ class XrView { this._manager = manager; this._xrView = xrView; - if (this._manager.views.supportedColor) + if (this._manager.views.supportedColor) { this._xrCamera = this._xrView.camera; - this._updateTextureColor(); + // color texture + if (this._manager.views.availableColor && this._xrCamera) { + this._textureColor = new Texture(this._manager.app.graphicsDevice, { + format: PIXELFORMAT_RGB8, + mipmaps: false, + addressU: ADDRESS_CLAMP_TO_EDGE, + addressV: ADDRESS_CLAMP_TO_EDGE, + minFilter: FILTER_LINEAR, + magFilter: FILTER_LINEAR, + width: this._xrCamera.width, + height: this._xrCamera.height, + name: `XrView-${this._xrView.eye}-Color` + }); + } + } } /** @@ -220,7 +234,7 @@ class XrView { * @private */ _updateTextureColor() { - if (!this._manager.views.availableColor || !this._xrCamera) + if (!this._manager.views.availableColor || !this._xrCamera || !this._textureColor) return; const binding = this._manager.webglBinding; @@ -233,63 +247,42 @@ class XrView { const device = this._manager.app.graphicsDevice; const gl = device.gl; - const attachmentBaseConstant = device.isWebGL2 ? gl.COLOR_ATTACHMENT0 : (device.extDrawBuffers?.COLOR_ATTACHMENT0_WEBGL ?? gl.COLOR_ATTACHMENT0); - - const width = this._xrCamera.width; - const height = this._xrCamera.height; - - if (!this._textureColor) { - // color texture - this._textureColor = new Texture(device, { - format: PIXELFORMAT_RGB8, - mipmaps: false, - addressU: ADDRESS_CLAMP_TO_EDGE, - addressV: ADDRESS_CLAMP_TO_EDGE, - minFilter: FILTER_LINEAR, - magFilter: FILTER_LINEAR, - width: width, - height: height, - name: `XrView-${this._xrView.eye}-Color` - }); - - // force initialize texture - this._textureColor.upload(); + if (!this._frameBufferSource) { // create frame buffer to read from this._frameBufferSource = gl.createFramebuffer(); // create frame buffer to write to this._frameBuffer = gl.createFramebuffer(); - } + } else { + const attachmentBaseConstant = device.isWebGL2 ? gl.COLOR_ATTACHMENT0 : (device.extDrawBuffers?.COLOR_ATTACHMENT0_WEBGL ?? gl.COLOR_ATTACHMENT0); + const width = this._xrCamera.width; + const height = this._xrCamera.height; + + // set frame buffer to read from + device.setFramebuffer(this._frameBufferSource); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + texture, + 0 + ); + + // set frame buffer to write to + device.setFramebuffer(this._frameBuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + this._textureColor.impl._glTexture, + 0 + ); + + // bind buffers + gl.bindFramebuffer(gl.READ_FRAMEBUFFER, this._frameBufferSource); + gl.bindFramebuffer(gl.DRAW_FRAMEBUFFER, this._frameBuffer); - // set frame buffer to read from - device.setFramebuffer(this._frameBufferSource); - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - attachmentBaseConstant, - gl.TEXTURE_2D, - texture, - 0 - ); - - // set frame buffer to write to - device.setFramebuffer(this._frameBuffer); - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - attachmentBaseConstant, - gl.TEXTURE_2D, - this._textureColor.impl._glTexture, - 0 - ); - - // bind buffers - gl.bindFramebuffer(gl.READ_FRAMEBUFFER, this._frameBufferSource); - let ready = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE; - - gl.bindFramebuffer(gl.DRAW_FRAMEBUFFER, this._frameBuffer); - if (ready) ready = gl.checkFramebufferStatus(gl.FRAMEBUFFER) === gl.FRAMEBUFFER_COMPLETE; - - if (ready) { // copy buffers with flip Y gl.blitFramebuffer(0, height, width, 0, 0, 0, width, height, gl.COLOR_BUFFER_BIT, gl.NEAREST); } @@ -323,7 +316,9 @@ class XrView { if (this._textureColor) { this._textureColor.destroy(); this._textureColor = null; + } + if (this._frameBufferSource) { const gl = this._manager.app.graphicsDevice.gl; gl.deleteFramebuffer(this._frameBufferSource); From 70067e5cfea17d854ad6e05115fc14e101083dfe Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Fri, 24 Nov 2023 14:46:00 +0200 Subject: [PATCH 7/8] fix --- src/framework/xr/xr-views.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js index 5ccb265712f..5d34b59a768 100644 --- a/src/framework/xr/xr-views.js +++ b/src/framework/xr/xr-views.js @@ -158,7 +158,7 @@ class XrViews extends EventHandler { * @returns {XrView|null} View or null if view of such eye is not available. */ get(eye) { - return this._index.get(name) || null; + return this._index.get(eye) || null; } /** From eb766698dc453fdeb807bedc47d81c428041122a Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Sat, 25 Nov 2023 13:41:51 +0200 Subject: [PATCH 8/8] handle webgl device lost --- src/framework/xr/xr-manager.js | 34 ++++++++++++++++++++++++++++++++++ src/framework/xr/xr-view.js | 7 +++++++ 2 files changed, 41 insertions(+) diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index b7353c862a1..29d6c108899 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -241,6 +241,9 @@ class XrManager extends EventHandler { this._deviceAvailabilityCheck(); }); this._deviceAvailabilityCheck(); + + this.app.graphicsDevice.on('devicelost', this._onDeviceLost, this); + this.app.graphicsDevice.on('devicerestored', this._onDeviceRestored, this); } } @@ -520,6 +523,35 @@ class XrManager extends EventHandler { }); } + /** + * @private + */ + _onDeviceLost() { + if (this.webglBinding) + this.webglBinding = null; + } + + /** + * @private + */ + _onDeviceRestored() { + if (!this._session) + return; + + this.webglBinding = null; + + if (platform.browser) { + const deviceType = this.app.graphicsDevice.deviceType; + if ((deviceType === DEVICETYPE_WEBGL1 || deviceType === DEVICETYPE_WEBGL2) && window.XRWebGLBinding) { + try { + this.webglBinding = new XRWebGLBinding(this._session, this.app.graphicsDevice.gl); // eslint-disable-line no-undef + } catch (ex) { + this.fire('error', ex); + } + } + } + } + /** * Attempts to end XR session and optionally fires callback when session is ended or failed to * end. @@ -540,6 +572,8 @@ class XrManager extends EventHandler { return; } + this.webglBinding = null; + if (callback) this.once('end', callback); this._session.end(); diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js index 78cbcd37a57..ecb9b688188 100644 --- a/src/framework/xr/xr-view.js +++ b/src/framework/xr/xr-view.js @@ -117,6 +117,8 @@ class XrView { height: this._xrCamera.height, name: `XrView-${this._xrView.eye}-Color` }); + + this._manager.app.graphicsDevice?.on('devicelost', this._onDeviceLost, this); } } } @@ -309,6 +311,11 @@ class XrView { this._positionData[2] = this._viewInvOffMat.data[14]; } + _onDeviceLost() { + this._frameBufferSource = null; + this._frameBuffer = null; + } + /** * @ignore */