diff --git a/examples/package-lock.json b/examples/package-lock.json index 59943da9d95..8be323a30c2 100644 --- a/examples/package-lock.json +++ b/examples/package-lock.json @@ -57,23 +57,25 @@ "version": "1.67.0-dev", "dev": true, "license": "MIT", + "dependencies": { + "@types/webxr": "^0.5.7", + "@webgpu/types": "^0.1.38" + }, "devDependencies": { - "@babel/core": "^7.23.0", + "@babel/core": "^7.23.2", "@babel/eslint-parser": "^7.22.15", - "@babel/preset-env": "^7.22.20", + "@babel/preset-env": "^7.23.2", "@playcanvas/canvas-mock": "^1.0.1", "@playcanvas/eslint-config": "^1.7.1", "@playcanvas/jsdoc-template": "^1.1.2", - "@rollup/plugin-babel": "^6.0.3", - "@rollup/plugin-node-resolve": "^15.2.1", - "@rollup/plugin-strip": "^3.0.2", - "@rollup/plugin-terser": "^0.4.3", - "@rollup/pluginutils": "^5.0.4", - "@types/webxr": "^0.5.5", - "@webgpu/types": "^0.1.35", + "@rollup/plugin-babel": "^6.0.4", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-strip": "^3.0.4", + "@rollup/plugin-terser": "^0.4.4", + "@rollup/pluginutils": "^5.0.5", "c8": "^8.0.0", "chai": "^4.3.10", - "eslint": "^8.50.0", + "eslint": "^8.52.0", "fflate": "^0.8.1", "jsdoc": "^4.0.2", "jsdoc-tsimport-plugin": "^1.0.5", @@ -87,7 +89,7 @@ "rollup-plugin-jscc": "2.0.0", "rollup-plugin-visualizer": "^5.9.2", "serve": "^14.2.1", - "sinon": "^16.0.0", + "sinon": "^17.0.0", "typedoc": "^0.25.1", "typedoc-plugin-mdn-links": "^3.1.0", "typescript": "^5.2.2", @@ -9820,22 +9822,22 @@ "playcanvas": { "version": "file:..", "requires": { - "@babel/core": "^7.23.0", + "@babel/core": "^7.23.2", "@babel/eslint-parser": "^7.22.15", - "@babel/preset-env": "^7.22.20", + "@babel/preset-env": "^7.23.2", "@playcanvas/canvas-mock": "^1.0.1", "@playcanvas/eslint-config": "^1.7.1", "@playcanvas/jsdoc-template": "^1.1.2", - "@rollup/plugin-babel": "^6.0.3", - "@rollup/plugin-node-resolve": "^15.2.1", - "@rollup/plugin-strip": "^3.0.2", - "@rollup/plugin-terser": "^0.4.3", - "@rollup/pluginutils": "^5.0.4", - "@types/webxr": "^0.5.5", - "@webgpu/types": "^0.1.35", + "@rollup/plugin-babel": "^6.0.4", + "@rollup/plugin-node-resolve": "^15.2.3", + "@rollup/plugin-strip": "^3.0.4", + "@rollup/plugin-terser": "^0.4.4", + "@rollup/pluginutils": "^5.0.5", + "@types/webxr": "^0.5.7", + "@webgpu/types": "^0.1.38", "c8": "^8.0.0", "chai": "^4.3.10", - "eslint": "^8.50.0", + "eslint": "^8.52.0", "fflate": "^0.8.1", "jsdoc": "^4.0.2", "jsdoc-tsimport-plugin": "^1.0.5", @@ -9849,7 +9851,7 @@ "rollup-plugin-jscc": "2.0.0", "rollup-plugin-visualizer": "^5.9.2", "serve": "^14.2.1", - "sinon": "^16.0.0", + "sinon": "^17.0.0", "typedoc": "^0.25.1", "typedoc-plugin-mdn-links": "^3.1.0", "typescript": "^5.2.2", diff --git a/examples/src/examples/xr/ar-camera-color.mjs b/examples/src/examples/xr/ar-camera-color.mjs new file mode 100644 index 00000000000..ffc06325413 --- /dev/null +++ b/examples/src/examples/xr/ar-camera-color.mjs @@ -0,0 +1,201 @@ +import * as pc from 'playcanvas'; + +/** + * @typedef {import('../../options.mjs').ExampleOptions} ExampleOptions + * @param {import('../../options.mjs').ExampleOptions} options - The example options. + * @returns {Promise} The example application. + */ +async function example({ canvas }) { + /** + * @param {string} msg - The message. + */ + const message = function (msg) { + /** @type {HTMLDivElement} */ + let el = document.querySelector('.message'); + if (!el) { + el = document.createElement('div'); + el.classList.add('message'); + el.style.position = 'absolute'; + el.style.bottom = '96px'; + el.style.right = '0'; + el.style.padding = '8px 16px'; + el.style.fontFamily = 'Helvetica, Arial, sans-serif'; + el.style.color = '#fff'; + el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; + document.body.append(el); + } + el.textContent = msg; + }; + + const app = new pc.Application(canvas, { + mouse: new pc.Mouse(canvas), + touch: new pc.TouchDevice(canvas), + keyboard: new pc.Keyboard(window), + graphicsDeviceOptions: { alpha: true } + }); + + app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); + app.setCanvasResolution(pc.RESOLUTION_AUTO); + + // Ensure canvas is resized when window changes size + const resize = () => app.resizeCanvas(); + window.addEventListener('resize', resize); + app.on('destroy', () => { + window.removeEventListener('resize', resize); + }); + + // use device pixel ratio + app.graphicsDevice.maxPixelRatio = window.devicePixelRatio; + + app.start(); + + // create camera + const c = new pc.Entity(); + c.addComponent('camera', { + clearColor: new pc.Color(0, 0, 0, 0), + farClip: 10000 + }); + app.root.addChild(c); + + const l = new pc.Entity(); + l.addComponent("light", { + type: "spot", + range: 30 + }); + l.translate(0, 10, 0); + app.root.addChild(l); + + const material = new pc.StandardMaterial(); + + /** + * @param {number} x - The x coordinate. + * @param {number} y - The y coordinate. + * @param {number} z - The z coordinate. + */ + const createCube = function (x, y, z) { + const cube = new pc.Entity(); + cube.addComponent("render", { + type: "box" + }); + cube.render.material = material; + cube.setLocalScale(0.5, 0.5, 0.5); + cube.translate(x * 0.5, y, z * 0.5); + app.root.addChild(cube); + }; + + // create a grid of cubes + const SIZE = 4; + for (let x = 0; x < SIZE; x++) { + for (let y = 0; y < SIZE; y++) { + createCube(2 * x - SIZE, 0.25, 2 * y - SIZE); + } + } + + if (app.xr.supported) { + const activate = function () { + if (app.xr.isAvailable(pc.XRTYPE_AR)) { + c.camera.startXr(pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { + cameraColor: true, // request access to camera color + callback: function (err) { + if (err) message("WebXR Immersive AR failed to start: " + err.message); + } + }); + } else { + message("Immersive AR is not available"); + } + }; + + app.mouse.on("mousedown", function () { + if (!app.xr.active) + activate(); + }); + + if (app.touch) { + app.touch.on("touchend", function (evt) { + if (!app.xr.active) { + // if not in VR, activate + activate(); + } else { + // otherwise reset camera + c.camera.endXr(); + } + + evt.event.preventDefault(); + evt.event.stopPropagation(); + }); + } + + // end session by keyboard ESC + app.keyboard.on('keydown', function (evt) { + if (evt.key === pc.KEY_ESCAPE && app.xr.active) { + app.xr.end(); + } + }); + + app.xr.on('start', function () { + message("Immersive AR session has started"); + }); + app.xr.on('end', function () { + message("Immersive AR session has ended"); + }); + app.xr.on('available:' + pc.XRTYPE_AR, function (available) { + if (available) { + if (!app.xr.views.supportedColor) { + message("AR Camera Color is not supported"); + } else { + message("Touch screen to start AR session"); + } + } else { + message("Immersive AR is not available"); + } + }); + + app.on('update', () => { + // if camera color is available + if (app.xr.views.availableColor) { + for(let i = 0; i < app.xr.views.list.length; i++) { + const view = app.xr.views.list[i]; + if (!view.textureColor) // check if color texture is available + continue; + + // apply camera color texture to material diffuse channel + if (!material.diffuseMap) { + material.diffuseMap = view.textureColor; + material.update(); + } + + // debug draw camera color texture on the screen + app.drawTexture(0.5, -0.5, 1, 1, view.textureColor); + } + } + }); + + app.xr.on('end', () => { + if (!material.diffuseMap) + return; + + // clear camera color texture when XR session ends + material.diffuseMap = null; + material.update(); + }) + + if (!app.xr.isAvailable(pc.XRTYPE_AR)) { + message("Immersive AR is not available"); + } else if (!app.xr.views.supportedColor) { + message("AR Camera Color is not supported"); + } else { + message("Touch screen to start AR session"); + } + } else { + message("WebXR is not supported"); + } + return app; +} + +class ArCameraColorExample { + static CATEGORY = 'XR'; + static NAME = 'AR Camera Color'; + static example = example; +} + +export { ArCameraColorExample }; diff --git a/examples/src/examples/xr/index.mjs b/examples/src/examples/xr/index.mjs index dc6f77415bd..27fd04e8eb5 100644 --- a/examples/src/examples/xr/index.mjs +++ b/examples/src/examples/xr/index.mjs @@ -1,4 +1,5 @@ export * from "./ar-basic.mjs"; +export * from "./ar-camera-color.mjs"; export * from "./ar-hit-test.mjs"; export * from "./ar-hit-test-anchors.mjs"; export * from "./ar-anchors-persistence.mjs"; diff --git a/src/framework/xr/constants.js b/src/framework/xr/constants.js index 0c3c0f01895..fe464eed166 100644 --- a/src/framework/xr/constants.js +++ b/src/framework/xr/constants.js @@ -100,6 +100,27 @@ export const XRTARGETRAY_SCREEN = 'screen'; */ export const XRTARGETRAY_POINTER = 'tracked-pointer'; +/** + * None - view associated with a monoscopic screen, such as mobile phone screens. + * + * @type {string} + */ +export const XREYE_NONE = 'none'; + +/** + * Left - view associated with left eye. + * + * @type {string} + */ +export const XREYE_LEFT = 'left'; + +/** + * Right - view associated with right eye. + * + * @type {string} + */ +export const XREYE_RIGHT = 'right'; + /** * None - input source is not meant to be held in hands. * diff --git a/src/framework/xr/xr-manager.js b/src/framework/xr/xr-manager.js index 5c43ecfd25e..707aa28a242 100644 --- a/src/framework/xr/xr-manager.js +++ b/src/framework/xr/xr-manager.js @@ -2,13 +2,12 @@ import { Debug } from "../../core/debug.js"; import { EventHandler } from '../../core/event-handler.js'; import { platform } from '../../core/platform.js'; -import { Mat3 } from '../../core/math/mat3.js'; import { Mat4 } from '../../core/math/mat4.js'; import { Quat } from '../../core/math/quat.js'; import { Vec3 } from '../../core/math/vec3.js'; -import { Vec4 } from '../../core/math/vec4.js'; import { XRTYPE_INLINE, XRTYPE_VR, XRTYPE_AR, XRDEPTHSENSINGUSAGE_CPU, XRDEPTHSENSINGFORMAT_L8A8 } from './constants.js'; +import { DEVICETYPE_WEBGL1, DEVICETYPE_WEBGL2 } from '../../platform/graphics/constants.js'; import { XrDepthSensing } from './xr-depth-sensing.js'; import { XrDomOverlay } from './xr-dom-overlay.js'; import { XrHitTest } from './xr-hit-test.js'; @@ -17,6 +16,7 @@ import { XrInput } from './xr-input.js'; import { XrLightEstimation } from './xr-light-estimation.js'; import { XrPlaneDetection } from './xr-plane-detection.js'; import { XrAnchors } from './xr-anchors.js'; +import { XrViews } from './xr-views.js'; /** * Callback used by {@link XrManager#endXr} and {@link XrManager#startXr}. @@ -81,6 +81,12 @@ class XrManager extends EventHandler { */ _baseLayer = null; + /** + * @type {XRWebGLBinding|null} + * @ignore + */ + webglBinding = null; + /** * @type {XRReferenceSpace|null} * @ignore @@ -142,6 +148,14 @@ class XrManager extends EventHandler { */ lightEstimation; + /** + * Provides access to views and their capabilities. + * + * @type {XrViews} + * @ignore + */ + views; + /** * Provides access to Anchors. * @@ -155,18 +169,6 @@ class XrManager extends EventHandler { */ _camera = null; - /** - * @type {Array<*>} - * @ignore - */ - views = []; - - /** - * @type {Array<*>} - * @ignore - */ - viewsPool = []; - /** * @type {Vec3} * @private @@ -227,6 +229,7 @@ class XrManager extends EventHandler { this.input = new XrInput(this); this.lightEstimation = new XrLightEstimation(this); this.anchors = new XrAnchors(this); + this.views = new XrViews(this); // TODO // 1. HMD class with its params @@ -424,6 +427,8 @@ class XrManager extends EventHandler { optionalFeatures: [] }; + const webgl = this.app.graphicsDevice?.isWebGL1 || this.app.graphicsDevice?.isWebGL2; + if (type === XRTYPE_AR) { opts.optionalFeatures.push('light-estimation'); opts.optionalFeatures.push('hit-test'); @@ -468,6 +473,10 @@ class XrManager extends EventHandler { dataFormatPreference: dataFormatPreference }; } + + if (webgl && options && options.cameraColor && this.views.supportedColor) { + opts.optionalFeatures.push('camera-access'); + } } else if (type === XRTYPE_VR) { opts.optionalFeatures.push('hand-tracking'); } @@ -534,6 +543,8 @@ class XrManager extends EventHandler { return; } + this.webglBinding = null; + if (callback) this.once('end', callback); this._session.end(); @@ -654,7 +665,6 @@ class XrManager extends EventHandler { this._session = null; this._referenceSpace = null; - this.views = []; this._width = 0; this._height = 0; this._type = null; @@ -662,7 +672,8 @@ class XrManager extends EventHandler { // old requestAnimationFrame will never be triggered, // so queue up new tick - this.app.tick(); + if (this.app.systems) + this.app.tick(); }; session.addEventListener('end', onEnd); @@ -720,9 +731,10 @@ class XrManager extends EventHandler { } _createBaseLayer() { - const framebufferScaleFactor = this.app.graphicsDevice.maxPixelRatio / window.devicePixelRatio; + const device = this.app.graphicsDevice; + const framebufferScaleFactor = device.maxPixelRatio / window.devicePixelRatio; - this._baseLayer = new XRWebGLLayer(this._session, this.app.graphicsDevice.gl, { + this._baseLayer = new XRWebGLLayer(this._session, device.gl, { alpha: true, depth: true, stencil: true, @@ -730,6 +742,15 @@ class XrManager extends EventHandler { antialias: false }); + const deviceType = device.deviceType; + if ((deviceType === DEVICETYPE_WEBGL1 || deviceType === DEVICETYPE_WEBGL2) && window.XRWebGLBinding) { + try { + this.webglBinding = new XRWebGLBinding(this._session, device.gl); // eslint-disable-line no-undef + } catch (ex) { + this.fire('error', ex); + } + } + this._session.updateRenderState({ baseLayer: this._baseLayer, depthNear: this._depthNear, @@ -742,6 +763,9 @@ class XrManager extends EventHandler { if (!this._session) return; + if (this.webglBinding) + this.webglBinding = null; + this._baseLayer = null; this._session.updateRenderState({ @@ -789,32 +813,10 @@ class XrManager extends EventHandler { if (!pose) return false; - const lengthOld = this.views.length; - const lengthNew = pose.views.length; - - while (lengthNew > this.views.length) { - let view = this.viewsPool.pop(); - if (!view) { - view = { - viewport: new Vec4(), - projMat: new Mat4(), - viewMat: new Mat4(), - viewOffMat: new Mat4(), - viewInvMat: new Mat4(), - viewInvOffMat: new Mat4(), - projViewOffMat: new Mat4(), - viewMat3: new Mat3(), - position: new Float32Array(3), - rotation: new Quat() - }; - } + const lengthOld = this.views.list.length; - this.views.push(view); - } - // remove views from list into pool - while (lengthNew < this.views.length) { - this.viewsPool.push(this.views.pop()); - } + // add views + this.views.update(frame, pose.views); // reset position const posePosition = pose.transform.position; @@ -822,28 +824,10 @@ class XrManager extends EventHandler { this._localPosition.set(posePosition.x, posePosition.y, posePosition.z); this._localRotation.set(poseOrientation.x, poseOrientation.y, poseOrientation.z, poseOrientation.w); - const layer = frame.session.renderState.baseLayer; - - for (let i = 0; i < pose.views.length; i++) { - // for each view, calculate matrices - const viewRaw = pose.views[i]; - const view = this.views[i]; - const viewport = layer.getViewport(viewRaw); - - view.viewport.x = viewport.x; - view.viewport.y = viewport.y; - view.viewport.z = viewport.width; - view.viewport.w = viewport.height; - - view.projMat.set(viewRaw.projectionMatrix); - view.viewMat.set(viewRaw.transform.inverse.matrix); - view.viewInvMat.set(viewRaw.transform.matrix); - } - // update the camera fov properties only when we had 0 views - if (lengthOld === 0 && this.views.length > 0) { + if (lengthOld === 0 && this.views.list.length > 0) { const viewProjMat = new Mat4(); - const view = this.views[0]; + const view = this.views.list[0]; viewProjMat.copy(view.projMat); const data = viewProjMat.data; @@ -854,7 +838,6 @@ class XrManager extends EventHandler { const nearClip = data[14] / (data[10] - 1); const horizontalFov = false; - const camera = this._camera.camera; camera.setXrProperties({ aspectRatio, diff --git a/src/framework/xr/xr-view.js b/src/framework/xr/xr-view.js new file mode 100644 index 00000000000..ecb9b688188 --- /dev/null +++ b/src/framework/xr/xr-view.js @@ -0,0 +1,340 @@ +import { Texture } from '../../platform/graphics/texture.js'; +import { Vec4 } from "../../core/math/vec4.js"; +import { Mat3 } from "../../core/math/mat3.js"; +import { Mat4 } from "../../core/math/mat4.js"; + +import { ADDRESS_CLAMP_TO_EDGE, FILTER_LINEAR, PIXELFORMAT_RGB8 } from '../../platform/graphics/constants.js'; + +/** + * Represents XR View which represents a screen (mobile phone context) or an eye (HMD context). + * + * @category XR + */ +class XrView { + /** + * @type {import('./xr-manager.js').XrManager} + * @private + */ + _manager; + + /** + * @type {XRView} + * @private + */ + _xrView; + + /** + * @type {Float32Array} + * @private + */ + _positionData = new Float32Array(3); + + /** + * @type {Vec4} + * @private + */ + _viewport = new Vec4(); + + /** + * @type {Mat4} + * @private + */ + _projMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ + _projViewOffMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ + _viewMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ + _viewOffMat = new Mat4(); + + /** + * @type {Mat3} + * @private + */ + _viewMat3 = new Mat3(); + + /** + * @type {Mat4} + * @private + */ + _viewInvMat = new Mat4(); + + /** + * @type {Mat4} + * @private + */ + _viewInvOffMat = new Mat4(); + + /** + * @type {XRCamera} + * @private + */ + _xrCamera = null; + + /** + * @type {Texture|null} + * @private + */ + _textureColor = null; + + /** + * Create a new XrView instance. + * + * @param {import('./xr-manager.js').XrManager} manager - WebXR Manager. + * @param {XRView} xrView - [XRView](https://developer.mozilla.org/en-US/docs/Web/API/XRView) + * object that is created by WebXR API. + * @hideconstructor + */ + constructor(manager, xrView) { + this._manager = manager; + this._xrView = xrView; + + if (this._manager.views.supportedColor) { + this._xrCamera = this._xrView.camera; + + // color texture + if (this._manager.views.availableColor && this._xrCamera) { + this._textureColor = new Texture(this._manager.app.graphicsDevice, { + format: PIXELFORMAT_RGB8, + mipmaps: false, + addressU: ADDRESS_CLAMP_TO_EDGE, + addressV: ADDRESS_CLAMP_TO_EDGE, + minFilter: FILTER_LINEAR, + magFilter: FILTER_LINEAR, + width: this._xrCamera.width, + height: this._xrCamera.height, + name: `XrView-${this._xrView.eye}-Color` + }); + + this._manager.app.graphicsDevice?.on('devicelost', this._onDeviceLost, this); + } + } + } + + /** + * Texture associated with this view's camera color. Equals to null if camera color is + * not available or not supported. + * + * @type {Texture|null} + */ + get textureColor() { + return this._textureColor; + } + + /** + * An eye with which this view is associated. Can be any of: + * + * - {@link XREYE_NONE}: None - inidcates a monoscopic view (likely mobile phone screen). + * - {@link XREYE_LEFT}: Left - indicates left eye view. + * - {@link XREYE_RIGHT}: Right - indicates a right eye view. + * + * @type {string} + */ + get eye() { + return this._xrView.eye; + } + + /** + * A Vec4 (x, y, width, height) that represents a view's viewport. For monoscopic screen + * it will define fullscreen view, but for stereoscopic views (left/right eye) it will define + * a part of a whole screen that view is occupying. + * + * @type {Vec4} + */ + get viewport() { + return this._viewport; + } + + /** + * @type {Mat4} + * @ignore + */ + get projMat() { + return this._projMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get projViewOffMat() { + return this._projViewOffMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get viewOffMat() { + return this._viewOffMat; + } + + /** + * @type {Mat4} + * @ignore + */ + get viewInvOffMat() { + return this._viewInvOffMat; + } + + /** + * @type {Mat3} + * @ignore + */ + get viewMat3() { + return this._viewMat3; + } + + /** + * @type {Float32Array} + * @ignore + */ + get positionData() { + return this._positionData; + } + + /** + * @param {*} frame - XRFrame from requestAnimationFrame callback. + * @param {XRView} xrView - XRView from WebXR API. + * @ignore + */ + update(frame, xrView) { + this._xrView = xrView; + if (this._manager.views.availableColor) + this._xrCamera = this._xrView.camera; + + const layer = frame.session.renderState.baseLayer; + + // viewport + const viewport = layer.getViewport(this._xrView); + this._viewport.x = viewport.x; + this._viewport.y = viewport.y; + this._viewport.z = viewport.width; + this._viewport.w = viewport.height; + + // matrices + this._projMat.set(this._xrView.projectionMatrix); + this._viewMat.set(this._xrView.transform.inverse.matrix); + this._viewInvMat.set(this._xrView.transform.matrix); + + this._updateTextureColor(); + } + + /** + * @private + */ + _updateTextureColor() { + if (!this._manager.views.availableColor || !this._xrCamera || !this._textureColor) + return; + + const binding = this._manager.webglBinding; + if (!binding) + return; + + const texture = binding.getCameraImage(this._xrCamera); + if (!texture) + return; + + const device = this._manager.app.graphicsDevice; + const gl = device.gl; + + if (!this._frameBufferSource) { + // create frame buffer to read from + this._frameBufferSource = gl.createFramebuffer(); + + // create frame buffer to write to + this._frameBuffer = gl.createFramebuffer(); + } else { + const attachmentBaseConstant = device.isWebGL2 ? gl.COLOR_ATTACHMENT0 : (device.extDrawBuffers?.COLOR_ATTACHMENT0_WEBGL ?? gl.COLOR_ATTACHMENT0); + const width = this._xrCamera.width; + const height = this._xrCamera.height; + + // set frame buffer to read from + device.setFramebuffer(this._frameBufferSource); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + texture, + 0 + ); + + // set frame buffer to write to + device.setFramebuffer(this._frameBuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + attachmentBaseConstant, + gl.TEXTURE_2D, + this._textureColor.impl._glTexture, + 0 + ); + + // bind buffers + gl.bindFramebuffer(gl.READ_FRAMEBUFFER, this._frameBufferSource); + gl.bindFramebuffer(gl.DRAW_FRAMEBUFFER, this._frameBuffer); + + // copy buffers with flip Y + gl.blitFramebuffer(0, height, width, 0, 0, 0, width, height, gl.COLOR_BUFFER_BIT, gl.NEAREST); + } + } + + /** + * @param {Mat4|null} transform - World Transform of a parents GraphNode. + * @ignore + */ + updateTransforms(transform) { + if (transform) { + this._viewInvOffMat.mul2(transform, this._viewInvMat); + this.viewOffMat.copy(this._viewInvOffMat).invert(); + } else { + this._viewInvOffMat.copy(this._viewInvMat); + this.viewOffMat.copy(this._viewMat); + } + + this._viewMat3.setFromMat4(this._viewOffMat); + this._projViewOffMat.mul2(this._projMat, this._viewOffMat); + + this._positionData[0] = this._viewInvOffMat.data[12]; + this._positionData[1] = this._viewInvOffMat.data[13]; + this._positionData[2] = this._viewInvOffMat.data[14]; + } + + _onDeviceLost() { + this._frameBufferSource = null; + this._frameBuffer = null; + } + + /** + * @ignore + */ + destroy() { + if (this._textureColor) { + this._textureColor.destroy(); + this._textureColor = null; + } + + if (this._frameBufferSource) { + const gl = this._manager.app.graphicsDevice.gl; + + gl.deleteFramebuffer(this._frameBufferSource); + this._frameBufferSource = null; + + gl.deleteFramebuffer(this._frameBuffer); + this._frameBuffer = null; + } + } +} + +export { XrView }; diff --git a/src/framework/xr/xr-views.js b/src/framework/xr/xr-views.js new file mode 100644 index 00000000000..5d34b59a768 --- /dev/null +++ b/src/framework/xr/xr-views.js @@ -0,0 +1,186 @@ +import { platform } from '../../core/platform.js'; +import { EventHandler } from "../../core/event-handler.js"; +import { XrView } from "./xr-view.js"; +import { XRTYPE_AR } from "./constants.js"; + +/** + * Provides access to list of {@link XrView}'s. And information about their capabilities, + * such as support and availability of view's camera color texture. + * + * @category XR + */ +class XrViews extends EventHandler { + /** + * @type {import('./xr-manager.js').XrManager} + * @private + */ + _manager; + + /** + * @type {Map} + * @private + */ + _index = new Map(); + + /** + * @type {Map} + * @private + */ + _indexTmp = new Map(); + + /** + * @type {XrView[]} + * @private + */ + _list = []; + + /** + * @type {boolean} + * @private + */ + _supportedColor = platform.browser && !!window.XRCamera && !!window.XRWebGLBinding; + + /** + * @type {boolean} + * @private + */ + _availableColor = false; + + /** + * @param {import('./xr-manager.js').XrManager} manager - WebXR Manager. + * @hideconstructor + */ + constructor(manager) { + super(); + + this._manager = manager; + this._manager.on('start', this._onSessionStart, this); + this._manager.on('end', this._onSessionEnd, this); + } + + /** + * Fired when view has been added. Views are not available straight away on session start + * and are added mid-session. They can be added/removed mid session by underlyng system. + * + * @event XrViews#add + * @param {XrView} view - XrView that has been added. + * @example + * xr.views.on('add', function (view) { + * // view that has been added + * }); + */ + + /** + * Fired when view has been removed. They can be added/removed mid session by underlyng system. + * + * @event XrViews#remove + * @param {XrView} view - XrView that has been removed. + * @example + * xr.views.on('remove', function (view) { + * // view that has been added + * }); + */ + + /** + * An array of {@link XrView}'s of this session. Views are not available straight + * away on session start, and can be added/removed mid-session. So use of add/remove + * events is required for accessing views. + * + * @type {XrView[]} + */ + get list() { + return this._list; + } + + /** + * Check if Camera Color is supported. It might be still unavailable even if requested, + * based on hardware capabilities and granted permissions. + * + * @type {boolean} + */ + get supportedColor() { + return this._supportedColor; + } + + /** + * Check if Camera Color is available. This information becomes available only after + * session has started. + * + * @type {boolean} + */ + get availableColor() { + return this._availableColor; + } + + /** + * @param {*} frame - XRFrame from requestAnimationFrame callback. + * @param {XRView} xrView - XRView from WebXR API. + * @ignore + */ + update(frame, xrViews) { + for (let i = 0; i < xrViews.length; i++) { + this._indexTmp.set(xrViews[i].eye, xrViews[i]); + } + + for (const [eye, xrView] of this._indexTmp) { + let view = this._index.get(eye); + + if (!view) { + // add new view + view = new XrView(this._manager, xrView); + this._index.set(eye, view); + this._list.push(view); + view.update(frame, xrView); + this.fire('add', view); + } else { + // update existing view0 + view.update(frame, xrView); + } + } + + // remove views + for (const [eye, view] of this._index) { + if (this._indexTmp.has(eye)) + continue; + + view.destroy(); + this._index.delete(eye); + const ind = this._list.indexOf(view); + if (ind !== -1) this._list.splice(ind, 1); + this.fire('remove', view); + } + + this._indexTmp.clear(); + } + + /** + * @param {string} eye - An XREYE_* view is associated with. Can be 'none' for monoscope views. + * @returns {XrView|null} View or null if view of such eye is not available. + */ + get(eye) { + return this._index.get(eye) || null; + } + + /** + * @private + */ + _onSessionStart() { + if (this._manager.type !== XRTYPE_AR) + return; + this._availableColor = this._manager.session.enabledFeatures.indexOf('camera-access') !== -1; + } + + /** + * @private + */ + _onSessionEnd() { + for (const view of this._index.values()) { + view.destroy(); + } + this._index.clear(); + this._availableColor = false; + this._list.length = 0; + } +} + +export { XrViews }; diff --git a/src/scene/renderer/forward-renderer.js b/src/scene/renderer/forward-renderer.js index 79191c46b12..8e459a3f20e 100644 --- a/src/scene/renderer/forward-renderer.js +++ b/src/scene/renderer/forward-renderer.js @@ -621,11 +621,11 @@ class ForwardRenderer extends Renderer { drawCallback?.(drawCall, i); - if (camera.xr && camera.xr.session && camera.xr.views.length) { + if (camera.xr && camera.xr.session && camera.xr.views.list.length) { const views = camera.xr.views; - for (let v = 0; v < views.length; v++) { - const view = views[v]; + for (let v = 0; v < views.list.length; v++) { + const view = views.list[v]; device.setViewport(view.viewport.x, view.viewport.y, view.viewport.z, view.viewport.w); @@ -635,7 +635,7 @@ class ForwardRenderer extends Renderer { this.viewInvId.setValue(view.viewInvOffMat.data); this.viewId3.setValue(view.viewMat3.data); this.viewProjId.setValue(view.projViewOffMat.data); - this.viewPosId.setValue(view.position); + this.viewPosId.setValue(view.positionData); if (v === 0) { this.drawInstance(device, drawCall, mesh, style, true); diff --git a/src/scene/renderer/renderer.js b/src/scene/renderer/renderer.js index a5c08aa1cc3..e2219712fed 100644 --- a/src/scene/renderer/renderer.js +++ b/src/scene/renderer/renderer.js @@ -295,31 +295,12 @@ class Renderer { let viewCount = 1; if (camera.xr && camera.xr.session) { - let transform; - const parent = camera._node.parent; - if (parent) - transform = parent.getWorldTransform(); - + const transform = camera._node?.parent?.getWorldTransform() || null; const views = camera.xr.views; - viewCount = views.length; + viewCount = views.list.length; for (let v = 0; v < viewCount; v++) { - const view = views[v]; - - if (parent) { - view.viewInvOffMat.mul2(transform, view.viewInvMat); - view.viewOffMat.copy(view.viewInvOffMat).invert(); - } else { - view.viewInvOffMat.copy(view.viewInvMat); - view.viewOffMat.copy(view.viewMat); - } - - view.viewMat3.setFromMat4(view.viewOffMat); - view.projViewOffMat.mul2(view.projMat, view.viewOffMat); - - view.position[0] = view.viewInvOffMat.data[12]; - view.position[1] = view.viewInvOffMat.data[13]; - view.position[2] = view.viewInvOffMat.data[14]; - + const view = views.list[v]; + view.updateTransforms(transform); camera.frustum.setFromMat4(view.projViewOffMat); } } else { @@ -497,9 +478,9 @@ class Renderer { updateCameraFrustum(camera) { - if (camera.xr && camera.xr.views.length) { + if (camera.xr && camera.xr.views.list.length) { // calculate frustum based on XR view - const view = camera.xr.views[0]; + const view = camera.xr.views.list[0]; viewProjMat.mul2(view.projMat, view.viewOffMat); camera.frustum.setFromMat4(viewProjMat); return;