From 2772594c1358f5615395fa4fd1e15462fb1559ab Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Mon, 6 Nov 2023 14:22:37 +0200 Subject: [PATCH 1/2] webxr planes labels; plane detection example; --- .../src/examples/xr/ar-plane-detection.mjs | 308 ++++++++++++++++++ examples/src/examples/xr/index.mjs | 1 + src/framework/xr/xr-plane-detection.js | 36 +- src/framework/xr/xr-plane.js | 13 + 4 files changed, 340 insertions(+), 18 deletions(-) create mode 100644 examples/src/examples/xr/ar-plane-detection.mjs diff --git a/examples/src/examples/xr/ar-plane-detection.mjs b/examples/src/examples/xr/ar-plane-detection.mjs new file mode 100644 index 00000000000..b5764d7a357 --- /dev/null +++ b/examples/src/examples/xr/ar-plane-detection.mjs @@ -0,0 +1,308 @@ +import * as pc from 'playcanvas'; + +/** + * @typedef {import('../../options.mjs').ExampleOptions} ExampleOptions + * @param {import('../../options.mjs').ExampleOptions} options - The example options. + * @returns {Promise} The example application. + */ +async function example({ canvas }) { + /** + * @param {string} msg - The message. + */ + const message = function (msg) { + /** @type {HTMLDivElement} */ + let el = document.querySelector('.message'); + if (!el) { + el = document.createElement('div'); + el.classList.add('message'); + el.style.position = 'absolute'; + el.style.bottom = '96px'; + el.style.right = '0'; + el.style.padding = '8px 16px'; + el.style.fontFamily = 'Helvetica, Arial, sans-serif'; + el.style.color = '#fff'; + el.style.backgroundColor = 'rgba(0, 0, 0, 0.5)'; + document.body.append(el); + } + el.textContent = msg; + }; + + const assets = { + font: new pc.Asset('font', 'font', { url: assetPath + 'fonts/courier.json' }) + }; + + const app = new pc.Application(canvas, { + mouse: new pc.Mouse(canvas), + touch: new pc.TouchDevice(canvas), + keyboard: new pc.Keyboard(window), + graphicsDeviceOptions: { alpha: true } + }); + + app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); + app.setCanvasResolution(pc.RESOLUTION_AUTO); + + // Ensure canvas is resized when window changes size + const resize = () => app.resizeCanvas(); + window.addEventListener('resize', resize); + app.on('destroy', () => { + window.removeEventListener('resize', resize); + }); + + // use device pixel ratio + app.graphicsDevice.maxPixelRatio = window.devicePixelRatio; + + const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets); + assetListLoader.load(() => { + app.start(); + + // create camera + const camera = new pc.Entity(); + camera.addComponent('camera', { + clearColor: new pc.Color(0, 0, 0, 0), + farClip: 10000 + }); + app.root.addChild(camera); + + const l = new pc.Entity(); + l.addComponent("light", { + type: "spot", + range: 30 + }); + l.translate(0, 10, 0); + camera.addChild(l); + + if (app.xr.supported) { + const activate = function () { + if (app.xr.isAvailable(pc.XRTYPE_AR)) { + camera.camera.startXr(pc.XRTYPE_AR, pc.XRSPACE_LOCALFLOOR, { + planeDetection: true, + callback: function (err) { + if (err) message("WebXR Immersive AR failed to start: " + err.message); + } + }); + } else { + message("Immersive AR is not available"); + } + }; + + app.mouse.on("mousedown", function () { + if (!app.xr.active) + activate(); + }); + + if (app.touch) { + app.touch.on("touchend", function (evt) { + if (!app.xr.active) { + // if not in VR, activate + activate(); + } else { + // otherwise reset camera + camera.camera.endXr(); + } + + evt.event.preventDefault(); + evt.event.stopPropagation(); + }); + } + + // end session by keyboard ESC + app.keyboard.on('keydown', function (evt) { + if (evt.key === pc.KEY_ESCAPE && app.xr.active) { + app.xr.end(); + } + }); + + app.xr.on('start', function () { + message("Immersive AR session has started"); + }); + app.xr.on('end', function () { + message("Immersive AR session has ended"); + }); + app.xr.on('available:' + pc.XRTYPE_AR, function (available) { + if (available) { + if (app.xr.planeDetection.supported) { + message("Touch screen to start AR session and look at the floor or walls"); + } else { + message("AR Plane Detection is not supported"); + } + } else { + message("Immersive AR is unavailable"); + } + }); + + const updateMesh = (xrPlane, entity) => { + let created = false; + let mesh = entity.render.meshInstances[0]?.mesh; + if (!mesh) { + mesh = new pc.Mesh(app.graphicsDevice); + created = true; + } + mesh.clear(true, false); + + let meshWireframe = entity.render.meshInstances[1]?.mesh; + if (created) { + meshWireframe = new pc.Mesh(app.graphicsDevice); + } + meshWireframe.clear(true, false); + + const vertices = new Float32Array((xrPlane.points.length + 1) * 3); + const verticesWireframe = new Float32Array(xrPlane.points.length * 3); + vertices[0] = 0; + vertices[1] = 0; + vertices[2] = 0; + + const indices = new Uint32Array(xrPlane.points.length * 3); + const indicesWireframe = new Uint32Array(xrPlane.points.length); + + for(let i = 0; i < xrPlane.points.length; i++) { + vertices[i * 3 + 3 + 0] = xrPlane.points[i].x; + vertices[i * 3 + 3 + 1] = xrPlane.points[i].y; + vertices[i * 3 + 3 + 2] = xrPlane.points[i].z; + verticesWireframe[i * 3 + 0] = xrPlane.points[i].x; + verticesWireframe[i * 3 + 1] = xrPlane.points[i].y; + verticesWireframe[i * 3 + 2] = xrPlane.points[i].z; + indices[i * 3 + 0] = 0; + indices[i * 3 + 1] = i + 1; + indices[i * 3 + 2] = ((i + 1) % xrPlane.points.length) + 1; + indicesWireframe[i] = i; + } + + mesh.setPositions(vertices); + mesh.setNormals(pc.calculateNormals(vertices, indices)); + mesh.setIndices(indices); + mesh.update(pc.PRIMITIVE_TRIANGLES); + + meshWireframe.setPositions(verticesWireframe); + meshWireframe.setIndices(indicesWireframe); + meshWireframe.update(pc.PRIMITIVE_LINELOOP); + + let meshInstance = entity.render.meshInstances[0]; + if (created) { + meshInstance = new pc.MeshInstance(mesh, material); + } + + let meshInstanceWireframe = entity.render.meshInstances[1]; + if (created) { + meshInstanceWireframe = new pc.MeshInstance(meshWireframe, materialWireframe); + meshInstanceWireframe.renderStyle = pc.RENDERSTYLE_WIREFRAME; + } + + if (created) entity.render.meshInstances = [ meshInstance, meshInstanceWireframe ]; + } + + const entities = new Map(); + + const material = new pc.StandardMaterial(); + material.blendType = pc.BLEND_PREMULTIPLIED; + material.opacity = 0.5; + + const materialWireframe = new pc.StandardMaterial(); + materialWireframe.emissive = new pc.Color(1, 1, 1); + + app.xr.planeDetection.on('add', (xrPlane) => { + // entity + const entity = new pc.Entity(); + entity.addComponent("render"); + app.root.addChild(entity); + entities.set(xrPlane, entity); + + updateMesh(xrPlane, entity); + + // label + const label = new pc.Entity(); + label.setLocalPosition(0, 0, 0); + label.addComponent("element", { + pivot: new pc.Vec2(0.5, 0.5), + fontAsset: assets.font.id, + fontSize: 0.05, + text: xrPlane.label || '-', + width: 1, + height: .1, + color: new pc.Color(1, 0, 0), + type: pc.ELEMENTTYPE_TEXT + }); + entity.addChild(label); + label.setLocalPosition(0, -.05, 0); + entity.label = label; + + // transform + entity.setPosition(xrPlane.getPosition()); + entity.setRotation(xrPlane.getRotation()); + + xrPlane.on('change', () => { + updateMesh(xrPlane, entity); + }); + }); + + // when XrPlane is removed, destroy related entity + app.xr.planeDetection.on('remove', (xrPlane) => { + const entity = entities.get(xrPlane); + if (entity) { + entity.destroy(); + entities.delete(xrPlane); + } + }); + + const vec3A = new pc.Vec3(); + const vec3B = new pc.Vec3(); + const vec3C = new pc.Vec3(); + const transform = new pc.Mat4(); + + app.on('update', () => { + if (app.xr.active && app.xr.planeDetection.supported) { + // iterate through each XrMesh + for(let i = 0; i < app.xr.planeDetection.planes.length; i++) { + const plane = app.xr.planeDetection.planes[i]; + + const entity = entities.get(plane); + if (entity) { + // update entity transforms based on XrPlane + entity.setPosition(plane.getPosition()); + entity.setRotation(plane.getRotation()); + + // make sure label is looking at the camera + entity.label.setLocalPosition(0, -.05, 0); + entity.label.lookAt(camera.getPosition()); + entity.label.rotateLocal(0, 180, 0); + entity.label.translateLocal(0, 0, .05); + } + + // render XrPlane gizmo axes + transform.setTRS(plane.getPosition(), plane.getRotation(), pc.Vec3.ONE); + vec3A.set(.2, 0, 0); + vec3B.set(0, .2, 0); + vec3C.set(0, 0, .2); + transform.transformPoint(vec3A, vec3A); + transform.transformPoint(vec3B, vec3B); + transform.transformPoint(vec3C, vec3C); + app.drawLine(plane.getPosition(), vec3A, pc.Color.RED, false); + app.drawLine(plane.getPosition(), vec3B, pc.Color.GREEN, false); + app.drawLine(plane.getPosition(), vec3C, pc.Color.BLUE, false); + + vec3A.copy(plane.points[0]); + transform.transformPoint(vec3A, vec3A); + } + } + }); + + if (!app.xr.isAvailable(pc.XRTYPE_AR)) { + message("Immersive AR is not available"); + } else if (!app.xr.planeDetection.supported) { + message("AR Plane Detection is not supported"); + } else { + message("Touch screen to start AR session and look at the floor or walls"); + } + } else { + message("WebXR is not supported"); + } + }); + return app; +} + +class ArPlanesDetectionExample { + static CATEGORY = 'XR'; + static NAME = 'AR Plane Detection'; + static example = example; +} + +export { ArPlanesDetectionExample }; diff --git a/examples/src/examples/xr/index.mjs b/examples/src/examples/xr/index.mjs index c40b96d7c91..0580abfeded 100644 --- a/examples/src/examples/xr/index.mjs +++ b/examples/src/examples/xr/index.mjs @@ -1,5 +1,6 @@ export * from "./ar-basic.mjs"; export * from "./ar-hit-test.mjs"; +export * from "./ar-plane-detection.mjs"; export * from "./vr-basic.mjs"; export * from './vr-controllers.mjs'; export * from "./vr-hands.mjs"; diff --git a/src/framework/xr/xr-plane-detection.js b/src/framework/xr/xr-plane-detection.js index 2ffb608d8a8..a19f5717e4f 100644 --- a/src/framework/xr/xr-plane-detection.js +++ b/src/framework/xr/xr-plane-detection.js @@ -47,10 +47,10 @@ class XrPlaneDetection extends EventHandler { _planesIndex = new Map(); /** - * @type {XrPlane[]|null} + * @type {XrPlane[]} * @private */ - _planes = null; + _planes = []; /** * Create a new XrPlaneDetection instance. @@ -64,6 +64,7 @@ class XrPlaneDetection extends EventHandler { this._manager = manager; if (this._supported) { + this._manager.on('start', this._onSessionStart, this); this._manager.on('end', this._onSessionEnd, this); } } @@ -102,6 +103,15 @@ class XrPlaneDetection extends EventHandler { * }); */ + /** @private */ + _onSessionStart() { + const available = this._supported && this._manager.session.enabledFeatures.indexOf('plane-detection') !== -1; + if (available) { + this._available = true; + this.fire('available'); + } + } + /** @private */ _onSessionEnd() { if (this._planes) { @@ -111,7 +121,7 @@ class XrPlaneDetection extends EventHandler { } this._planesIndex.clear(); - this._planes = null; + this._planes.length = 0; if (this._available) { this._available = false; @@ -124,20 +134,10 @@ class XrPlaneDetection extends EventHandler { * @ignore */ update(frame) { - let detectedPlanes; - - if (!this._available) { - try { - detectedPlanes = frame.detectedPlanes; - this._planes = []; - this._available = true; - this.fire('available'); - } catch (ex) { - return; - } - } else { - detectedPlanes = frame.detectedPlanes; - } + if (!this._supported || !this._available) + return; + + const detectedPlanes = frame.detectedPlanes; // iterate through indexed planes for (const [xrPlane, plane] of this._planesIndex) { @@ -194,7 +194,7 @@ class XrPlaneDetection extends EventHandler { * Array of {@link XrPlane} instances that contain individual plane information, or null if * plane detection is not available. * - * @type {XrPlane[]|null} + * @type {XrPlane[]} */ get planes() { return this._planes; diff --git a/src/framework/xr/xr-plane.js b/src/framework/xr/xr-plane.js index ae0bc0bf4a2..53dd2899664 100644 --- a/src/framework/xr/xr-plane.js +++ b/src/framework/xr/xr-plane.js @@ -94,6 +94,9 @@ class XrPlane extends EventHandler { /** @ignore */ destroy() { + if (!this._xrPlane) + return; + this._xrPlane = null; this.fire('remove'); } @@ -184,6 +187,16 @@ class XrPlane extends EventHandler { get points() { return this._xrPlane.polygon; } + + /** + * Semantic Label of a plane that is provided by underlying system. + * Current list includes (but not limited to): https://github.com/immersive-web/semantic-labels/blob/master/labels.json + * + * @type {string} + */ + get label() { + return this._xrPlane.semanticLabel || ''; + } } export { XrPlane }; From d6d553b72275dd859df655246f1a4599727b23dc Mon Sep 17 00:00:00 2001 From: mrmaxm Date: Tue, 7 Nov 2023 13:25:20 +0200 Subject: [PATCH 2/2] fire missing 'remove' event on planeDetection when plane was removed --- examples/src/examples/xr/ar-plane-detection.mjs | 3 +++ src/framework/xr/xr-plane-detection.js | 7 +++---- src/framework/xr/xr-plane.js | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/src/examples/xr/ar-plane-detection.mjs b/examples/src/examples/xr/ar-plane-detection.mjs index b5764d7a357..8dd229fa5ac 100644 --- a/examples/src/examples/xr/ar-plane-detection.mjs +++ b/examples/src/examples/xr/ar-plane-detection.mjs @@ -114,6 +114,9 @@ async function example({ canvas }) { app.xr.on('start', function () { message("Immersive AR session has started"); + + // trigger manual scanning on session start + // app.xr.initiateRoomCapture((err) => { }); }); app.xr.on('end', function () { message("Immersive AR session has ended"); diff --git a/src/framework/xr/xr-plane-detection.js b/src/framework/xr/xr-plane-detection.js index a19f5717e4f..356b7a483c3 100644 --- a/src/framework/xr/xr-plane-detection.js +++ b/src/framework/xr/xr-plane-detection.js @@ -114,10 +114,9 @@ class XrPlaneDetection extends EventHandler { /** @private */ _onSessionEnd() { - if (this._planes) { - for (let i = 0; i < this._planes.length; i++) { - this._planes[i].destroy(); - } + for (let i = 0; i < this._planes.length; i++) { + this._planes[i].destroy(); + this.fire('remove', this._planes[i]); } this._planesIndex.clear(); diff --git a/src/framework/xr/xr-plane.js b/src/framework/xr/xr-plane.js index 53dd2899664..537336bbc40 100644 --- a/src/framework/xr/xr-plane.js +++ b/src/framework/xr/xr-plane.js @@ -94,8 +94,7 @@ class XrPlane extends EventHandler { /** @ignore */ destroy() { - if (!this._xrPlane) - return; + if (!this._xrPlane) return; this._xrPlane = null; this.fire('remove'); }