diff --git a/diorama.js b/diorama.js index 07d5716e18..d99556e906 100644 --- a/diorama.js +++ b/diorama.js @@ -662,7 +662,7 @@ sideScene.add(glyphMesh); sideScene.add(outlineMesh); sideScene.add(labelMesh); sideScene.add(textObject); -const _addPreviewLights = scene => { +/* const _addPreviewLights = scene => { const ambientLight = new THREE.AmbientLight(0xffffff, 2); scene.add(ambientLight); @@ -671,7 +671,19 @@ const _addPreviewLights = scene => { directionalLight.updateMatrixWorld(); scene.add(directionalLight); }; -_addPreviewLights(sideScene); +_addPreviewLights(sideScene); */ +const autoLights = (() => { + const ambientLight = new THREE.AmbientLight(0xffffff, 2); + + const directionalLight = new THREE.DirectionalLight(0xffffff, 2); + directionalLight.position.set(1, 2, 3); + directionalLight.updateMatrixWorld(); + + return [ + ambientLight, + directionalLight, + ]; +})(); /* let sideSceneCompiled = false; const _ensureSideSceneCompiled = () => { if (!sideSceneCompiled) { @@ -705,6 +717,7 @@ const createPlayerDiorama = ({ canvas = null, objects = [], target = null, + lights = true, label = null, outline = false, lightningBackground = false, @@ -716,6 +729,8 @@ const createPlayerDiorama = ({ const {devicePixelRatio: pixelRatio} = window; + // const renderer = getRenderer(); + let locallyOwnedCanvas; if (canvas) { locallyOwnedCanvas = null; @@ -807,6 +822,11 @@ const createPlayerDiorama = ({ for (const object of objects) { scene.add(object); } + if (lights) { + for (const autoLight of autoLights) { + scene.add(autoLight); + } + } }; // push old state @@ -828,6 +848,11 @@ const createPlayerDiorama = ({ } } } + if (lights) { + for (const autoLight of autoLights) { + autoLight.parent.remove(autoLight); + } + } }; const oldRenderTarget = renderer.getRenderTarget(); const oldViewport = renderer.getViewport(localVector4D); @@ -972,13 +997,13 @@ const createPlayerDiorama = ({ const compile = () => { diorama.triggerLoad(); postProcessing.addEventListener('update', recompile); - } */ - /* if (player.avatar) { + } + if (player.avatar) { compile(); } else { function avatarchange() { if (player.avatar) { - // compile(); + compile(); player.removeEventListener('avatarchange', avatarchange); } } diff --git a/io-manager.js b/io-manager.js index 76e2dc25ee..8b211cacdf 100644 --- a/io-manager.js +++ b/io-manager.js @@ -684,17 +684,21 @@ const _updateMouseMovement = e => { const _getMouseRaycaster = (e, raycaster) => { const {clientX, clientY} = e; const renderer = getRenderer(); - renderer.getSize(localVector2D2); - localVector2D.set( - (clientX / localVector2D2.x) * 2 - 1, - -(clientY / localVector2D2.y) * 2 + 1 - ); - if ( - localVector2D.x >= -1 && localVector2D.x <= 1 && - localVector2D.y >= -1 && localVector2D.y <= 1 - ) { - raycaster.setFromCamera(localVector2D, camera); - return raycaster; + if (renderer) { + renderer.getSize(localVector2D2); + localVector2D.set( + (clientX / localVector2D2.x) * 2 - 1, + -(clientY / localVector2D2.y) * 2 + 1 + ); + if ( + localVector2D.x >= -1 && localVector2D.x <= 1 && + localVector2D.y >= -1 && localVector2D.y <= 1 + ) { + raycaster.setFromCamera(localVector2D, camera); + return raycaster; + } else { + return null; + } } else { return null; } @@ -804,7 +808,9 @@ ioManager.mousedown = e => { } else { if ((changedButtons & 1) && (e.buttons & 1)) { // left const raycaster = _getMouseRaycaster(e, localRaycaster); - transformControls.handleMouseDown(raycaster); + if (raycaster) { + transformControls.handleMouseDown(raycaster); + } } if ((changedButtons & 1) && (e.buttons & 2)) { // right game.menuDragdownRight(); diff --git a/pic-main.js b/pic-main.js new file mode 100644 index 0000000000..1f853f1e4c --- /dev/null +++ b/pic-main.js @@ -0,0 +1,40 @@ +import {bindCanvas} from './renderer.js'; +import {genPic} from './pic.js'; + +const defaultUrl = `/avatars/scillia_drophunter_v15_vian.vrm`; +const rendererSize = 2048; +const width = 500; +const height = 500; + +const formEl = document.getElementById('form'); +const urlEl = document.getElementById('url'); +const canvasEl = document.getElementById('canvas'); +const videoEl = document.getElementById('video'); +formEl.addEventListener('submit', async e => { + e.preventDefault(); + + const url = urlEl.value; + + const _bindRendererCanvas = () => { + const canvas = document.createElement('canvas'); + canvas.width = rendererSize; + canvas.height = rendererSize; + bindCanvas(canvas); + }; + _bindRendererCanvas(); + + const _initLocalCanvas = () => { + canvasEl.width = width; + canvasEl.height = height; + }; + _initLocalCanvas(); + + await genPic({ + url, + width, + height, + canvas: canvasEl, + video: videoEl, + }); +}); +urlEl.value = defaultUrl; \ No newline at end of file diff --git a/pic.html b/pic.html new file mode 100644 index 0000000000..ade62273fe --- /dev/null +++ b/pic.html @@ -0,0 +1,20 @@ + + +Pic | Webaverse + + + +
+ + +
+
+ + +
+ + + + + + diff --git a/pic.js b/pic.js new file mode 100644 index 0000000000..9922c61b8e --- /dev/null +++ b/pic.js @@ -0,0 +1,270 @@ +import * as THREE from 'three'; +import metaversefile from './metaversefile-api.js'; +// import {getExt, makePromise, parseQuery, fitCameraToBoundingBox} from './util.js'; +import Avatar from './avatars/avatars.js'; +import * as audioManager from './audio-manager.js'; +import npcManager from './npc-manager.js'; +import dioramaManager from './diorama.js'; +import {getRenderer, scene} from './renderer.js'; + +// import GIF from './gif.js'; +import * as WebMWriter from 'webm-writer'; +// const defaultWidth = 512; +// const defaultHeight = 512; +const FPS = 60; +// const videoQuality = 0.99999; +const videoQuality = 0.95; + +const localVector = new THREE.Vector3(); +const localVector2 = new THREE.Vector3(); +const localVector3 = new THREE.Vector3(); +const localMatrix = new THREE.Matrix4(); + +// I can take all of you motherfuckers on at once! + +/* const _makeRenderer = (width, height) => { + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('webgl2', { + alpha: true, + antialias: true, + desynchronized: true, + }); + const renderer = new THREE.WebGLRenderer({ + // alpha: true, + // antialias: true, + canvas, + context, + }); + // renderer.setSize(width, height); + renderer.setClearColor(0xff0000, 1); + + const scene = new THREE.Scene(); + scene.autoUpdate = false; + + const camera = new THREE.PerspectiveCamera(60, width/height, 0.1, 100); + + const directionalLight = new THREE.DirectionalLight(0xFFFFFF, 1); + directionalLight.position.set(2, 2, -2); + scene.add(directionalLight); + const directionalLight2 = new THREE.DirectionalLight(0xFFFFFF, 1); + directionalLight2.position.set(-2, 2, 2); + scene.add(directionalLight2); + + return {renderer, scene, camera}; +}; */ +const _makeLights = () => { + // const ambientLight = new THREE.AmbientLight(0xFFFFFF, 50); + // directionalLight.position.set(1, 1.5, -2); + // directionalLight.updateMatrixWorld(); + + const directionalLight = new THREE.DirectionalLight(0xFFFFFF, 5); + directionalLight.position.set(1, 1.5, -2); + directionalLight.updateMatrixWorld(); + + /* const directionalLight2 = new THREE.DirectionalLight(0xFFFFFF, 1.5); + directionalLight2.position.set(-1, 1.5, -2); + directionalLight2.updateMatrixWorld(); */ + + return [ + // ambientLight, + directionalLight, + // directionalLight2, + ]; +}; + +export const genPic = async ({ + url, + width, + height, + canvas, + video, +}) => { + await Avatar.waitForLoad(); + await audioManager.waitForLoad(); + + console.log('gen pic', { + url, + width, + height, + canvas, + video, + }); + + const animations = metaversefile.useAvatarAnimations(); + const idleAnimation = animations.find(a => a.name === 'idle.fbx'); + const idleAnimationDuration = idleAnimation.duration; + + // load app + const app = await metaversefile.createAppAsync({ + start_url: url, + }); + + const position = new THREE.Vector3(0, 1.5, 0); + const quaternion = new THREE.Quaternion(); + const scale = new THREE.Vector3(1, 1, 1); + const player = await npcManager.createNpc({ + name: 'npc', + avatarApp: app, + position, + quaternion, + scale, + }); + + const _setTransform = () => { + player.position.y = player.avatar.height; + player.updateMatrixWorld(); + }; + _setTransform(); + + const _initializeAnimation = () => { + player.avatar.setTopEnabled(false); + player.avatar.setHandEnabled(0, false); + player.avatar.setHandEnabled(1, false); + player.avatar.setBottomEnabled(false); + player.avatar.inputs.hmd.position.y = player.avatar.height; + player.avatar.inputs.hmd.quaternion.setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI); + player.avatar.inputs.hmd.updateMatrixWorld(); + player.addAction({ + type: 'emote', + emotion: 'angry', + }); + }; + const _animate = (timestamp, timeDiff) => { + // console.log('got position', player.position.y); + player.updateAvatar(timestamp, timeDiff); + }; + const _lookAt = (camera, boundingBox) => { + /* boundingBox.getCenter(camera.position); + // const size = boundingBox.getSize(localVector); + + camera.position.y = 0; + camera.position.z += 1; + camera.updateMatrixWorld(); + + fitCameraToBoundingBox(camera, boundingBox); */ + + camera.position.copy(player.position) + .add(localVector.set(0.3, 0, -0.5).applyQuaternion(player.quaternion)); + camera.quaternion.setFromRotationMatrix( + localMatrix.lookAt( + camera.position, + player.position, + localVector3.set(0, 1, 0) + ) + ); + camera.updateMatrixWorld(); + }; + + // rendering + const localLights = _makeLights(); + const objects = localLights.concat([ + player.avatar.model, + ]); + const diorama = dioramaManager.createPlayerDiorama({ + canvas, + target: player, + objects, + lights: false, + // label: true, + outline: true, + grassBackground: true, + // glyphBackground: true, + }); + // diorama.enabled = false; + + const videoWriter = new WebMWriter({ + quality: 1, + fileWriter: null, + fd: null, + frameDuration: null, + frameRate: FPS, + }); + const writeCanvas = canvas; + writeCanvas.width = width; + writeCanvas.height = height; + writeCanvas.style.width = `${width/window.devicePixelRatio}px`; + writeCanvas.style.height = `${height/window.devicePixelRatio}px`; + const writeCtx = writeCanvas.getContext('2d'); + + const framePromises = []; + const _pushFrame = async () => { + writeCtx.drawImage(canvas, 0, 0); + + const p = new Promise((resolve, reject) => { + writeCanvas.toBlob(blob => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onloadend = function() { + const dataUrl = reader.result; + resolve(dataUrl); + }; + }, 'image/webp', videoQuality); + }); + framePromises.push(p); + }; + const _render = async () => { + const boundingBox = new THREE.Box3().setFromObject(app); + + _initializeAnimation(); + // _lookAt(camera, boundingBox); + + const _renderFrames = async () => { + let now = 0; + const timeDiff = 1000/FPS; + for (let i = 0; i < FPS*2; i++) { + // _lookAt(camera, boundingBox); + _animate(now, timeDiff); + // app.updateMatrixWorld(); + } + + let index = 0; + const framesPerFrame = FPS; + while (now < idleAnimationDuration*1000) { + // _lookAt(camera, boundingBox); + _animate(now, timeDiff); + + diorama.update(now, timeDiff); + // renderer.render(scene, camera); + // renderer.getContext().flush(); + + _pushFrame(); + now += timeDiff; + + if ((index % framesPerFrame) === framesPerFrame-1) { + await new Promise((accept, reject) => { + requestAnimationFrame(() => { + accept(); + }); + }); + } + index++; + } + + const frameDataUrls = await Promise.all(framePromises); + framePromises.length = 0; + let dataUrl; + while ((dataUrl = frameDataUrls.shift()) !== undefined) { + videoWriter.addFrame({ + toDataURL() { + return dataUrl; + }, + }); + } + }; + await _renderFrames(); + }; + await _render(); + + const blob = await videoWriter.complete(); + await new Promise((accept, reject) => { + video.oncanplaythrough = accept; + video.onerror = reject; + video.src = URL.createObjectURL(blob); + }); + video.style.width = `${width/window.devicePixelRatio}px`; + video.style.height = `${height/window.devicePixelRatio}px`; + video.controls = true; + video.loop = true; +}; \ No newline at end of file diff --git a/renderer.js b/renderer.js index 6c1b387797..da728b3fc5 100644 --- a/renderer.js +++ b/renderer.js @@ -33,10 +33,15 @@ function bindCanvas(c) { rendererExtensionFragDepth: true, logarithmicDepthBuffer: true, }); + + const { + width, + height, + pixelRatio, + } = _getCanvasDimensions(); + renderer.setSize(width, height); + renderer.setPixelRatio(pixelRatio); - const rect = renderer.domElement.getBoundingClientRect(); - renderer.setSize(rect.width, rect.height); - renderer.setPixelRatio(window.devicePixelRatio); renderer.autoClear = false; renderer.sortObjects = false; renderer.physicallyCorrectLights = true; @@ -44,30 +49,22 @@ function bindCanvas(c) { // renderer.gammaFactor = 2.2; renderer.shadowMap.enabled = true; renderer.shadowMap.type = THREE.PCFSoftShadowMap; - if (!canvas) { - canvas = renderer.domElement; - } - if (!context) { - context = renderer.getContext(); - } - // context.enable(context.SAMPLE_ALPHA_TO_COVERAGE); renderer.xr.enabled = true; // initialize post-processing - { - const size = renderer.getSize(new THREE.Vector2()); - const pixelRatio = renderer.getPixelRatio(); - const encoding = THREE.sRGBEncoding; - const renderTarget = new THREE.WebGLMultisampleRenderTarget(size.x * pixelRatio, size.y * pixelRatio, { - minFilter: THREE.LinearFilter, - magFilter: THREE.LinearFilter, - format: THREE.RGBAFormat, - encoding, - }); - renderTarget.samples = context.MAX_SAMPLES; - composer = new EffectComposer(renderer, renderTarget); - } + const renderTarget = new THREE.WebGLMultisampleRenderTarget(width * pixelRatio, height * pixelRatio, { + minFilter: THREE.LinearFilter, + magFilter: THREE.LinearFilter, + format: THREE.RGBAFormat, + encoding: THREE.sRGBEncoding, + }); + renderTarget.samples = context.MAX_SAMPLES; + composer = new EffectComposer(renderer, renderTarget); + + // initialize camera + _setCameraSize(width, height, pixelRatio); + // resolve promise waitPromise.accept(); } @@ -84,7 +81,6 @@ function getComposer() { return composer; } - const scene = new THREE.Object3D(); scene.name = 'scene'; const sceneHighPriority = new THREE.Object3D(); @@ -105,7 +101,7 @@ rootScene.add(sceneLowPriority); // const orthographicScene = new THREE.Scene(); // const avatarScene = new THREE.Scene(); -const camera = new THREE.PerspectiveCamera(minFov, window.innerWidth / window.innerHeight, 0.1, 1000); +const camera = new THREE.PerspectiveCamera(minFov, 1, 0.1, 1000); camera.position.set(0, 1.6, 0); camera.rotation.order = 'YXZ'; camera.name = 'sceneCamera'; @@ -124,37 +120,67 @@ scene.add(dolly); const orthographicCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0.01, 100); // scene.add(orthographicCamera); -window.addEventListener('resize', e => { +const _getCanvasDimensions = () => { + let width, height, pixelRatio; + width = window.innerWidth; + height = window.innerHeight; + pixelRatio = window.devicePixelRatio; + + return { + width, + height, + pixelRatio, + }; +}; + +const _setSizes = () => { + const { + width, + height, + pixelRatio, + } = _getCanvasDimensions(); + _setRendererSize(width, height, pixelRatio); + _setComposerSize(width, height, pixelRatio); + _setCameraSize(width, height, pixelRatio); +}; + +const _setRendererSize = (width, height, pixelRatio) => { + // pause XR since it gets in the way of resize const renderer = getRenderer(); if (renderer) { if (renderer.xr.getSession()) { renderer.xr.isPresenting = false; } - const containerElement = getContainerElement(); - const {width, height} = containerElement.getBoundingClientRect(); - const pixelRatio = window.devicePixelRatio; + const { + width, + height, + pixelRatio, + } = _getCanvasDimensions(); renderer.setSize(width, height); renderer.setPixelRatio(pixelRatio); - // renderer2.setSize(window.innerWidth, window.innerHeight); - - const aspect = width / height; - camera.aspect = aspect; - camera.updateProjectionMatrix(); - // avatarCamera.aspect = aspect; - // avatarCamera.updateProjectionMatrix(); - + // resume XR if (renderer.xr.getSession()) { renderer.xr.isPresenting = true; } - - const composer = getComposer(); - if (composer) { - composer.setSize(width, height); - composer.setPixelRatio(pixelRatio); - } } +}; +const _setComposerSize = (width, height, pixelRatio) => { + const composer = getComposer(); + if (composer) { + composer.setSize(width, height); + composer.setPixelRatio(pixelRatio); + } +}; +const _setCameraSize = (width, height, pixelRatio) => { + const aspect = width / height; + camera.aspect = aspect; + camera.updateProjectionMatrix(); +}; + +window.addEventListener('resize', e => { + _setSizes(); }); /* addDefaultLights(scene, { diff --git a/src/CharacterHups.jsx b/src/CharacterHups.jsx index b797262706..e14795089e 100644 --- a/src/CharacterHups.jsx +++ b/src/CharacterHups.jsx @@ -13,7 +13,9 @@ import {chatTextSpeed} from '../constants.js'; const defaultHupSize = 256; const pixelRatio = window.devicePixelRatio; -function CharacterHup(props) { +const chatDioramas = new WeakMap(); + +const CharacterHup = function(props) { const {hup, index, hups, setHups} = props; const canvasRef = useRef(); @@ -26,16 +28,30 @@ function CharacterHup(props) { if (canvasRef.current) { const canvas = canvasRef.current; const player = hup.parent.player; - const diorama = dioramaManager.createPlayerDiorama(player, { - canvas, - grassBackground: true, - }); + let diorama = chatDioramas.get(player); + if (diorama) { + // console.log('got diorama', diorama); + diorama.resetCanvases(); + diorama.addCanvas(canvas); + } else { + diorama = dioramaManager.createPlayerDiorama({ + canvas, + target: player, + objects: [ + player.avatar.model, + ], + grassBackground: true, + }); + chatDioramas.set(player, diorama); + // console.log('no diorama'); + } return () => { + chatDioramas.delete(player); diorama.destroy(); }; } - }, [canvasRef.current]); + }, [canvasRef]); useEffect(() => { if (hupRef.current) { const hupEl = hupRef.current; @@ -53,7 +69,7 @@ function CharacterHup(props) { hupEl.removeEventListener('transitionend', transitionend); }; } - }, [hupRef.current, localOpen, hups, hups.length]); + }, [hupRef, localOpen, hups, hups.length]); useEffect(() => { setFullText(hup.fullText); }, []); diff --git a/src/tabs/character.jsx b/src/tabs/character.jsx index d62a97af5a..7b0f489c7d 100644 --- a/src/tabs/character.jsx +++ b/src/tabs/character.jsx @@ -5,7 +5,7 @@ import {Tab} from '../components/tab'; import metaversefile from '../../metaversefile-api.js'; import {defaultPlayerName} from '../../constants.js'; -export const Character = ({open, game, wearActions, panelsRef, setOpen, toggleOpen, previewCanvasRef}) => { +export const Character = ({open, game, wearActions, panelsRef, setOpen, toggleOpen, dioramaCanvasRef}) => { const sideSize = 400; return ( @@ -22,7 +22,7 @@ export const Character = ({open, game, wearActions, panelsRef, setOpen, toggleOp } panels={[ (
- +

{defaultPlayerName}

diff --git a/webaverse.js b/webaverse.js index 2b1a4892a9..8f87edb209 100644 --- a/webaverse.js +++ b/webaverse.js @@ -622,20 +622,6 @@ const _startHacks = () => { }); })(); } - } else if (e.which === 221) { // ] - const localPlayer = metaversefileApi.useLocalPlayer(); - if (localPlayer.avatar) { - if (!playerDiorama) { - playerDiorama = dioramaManager.createPlayerDiorama(localPlayer, { - label: true, - outline: true, - lightningBackground: true, - }); - } else { - playerDiorama.destroy(); - playerDiorama = null; - } - } } else if (e.which === 46) { // . emoteIndex = -1; _updateEmote();