From fb30e556fda775e09793dbc534a6ea091e81430c Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Tue, 10 Oct 2023 17:02:50 -0600 Subject: [PATCH 1/8] add library --- src/components/Selector.jsx | 20 ++++-- src/context/SceneContext.jsx | 11 +++ src/library/VRMRigMapMixamo.js | 57 ++++++++++++++++ src/library/animationManager.js | 101 +++++++++++++++++++++------- src/library/loadMixamoAnimation.js | 99 +++++++++++++++++++++++++++ src/library/loadMixamoAnimation2.js | 97 ++++++++++++++++++++++++++ 6 files changed, 354 insertions(+), 31 deletions(-) create mode 100644 src/library/VRMRigMapMixamo.js create mode 100644 src/library/loadMixamoAnimation.js create mode 100644 src/library/loadMixamoAnimation2.js diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index 78ead7f5..105e3eda 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -1,7 +1,7 @@ import React, { useContext, useEffect, useState } from "react" import * as THREE from "three" import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader" -import { MToonMaterial, VRMLoaderPlugin } from "@pixiv/three-vrm" +import { MToonMaterial, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" import cancel from "../../public/ui/selector/cancel.png" import { addModelData, disposeVRM } from "../library/utils" import { computeBoundsTree, disposeBoundsTree, acceleratedRaycast, SAH } from 'three-mesh-bvh'; @@ -42,7 +42,8 @@ export default function Selector({confirmDialog, templateInfo, animationManager, removeOption, saveUserSelection, setIsChangingWholeAvatar, - debugMode + debugMode, + vrmHelperRoot } = useContext(SceneContext) const { playSound @@ -274,7 +275,7 @@ export default function Selector({confirmDialog, templateInfo, animationManager, // load options first const loadOptions = (options, filterRestrictions = true, useTemplateBaseDirectory = true, saveUserSel = true) => { - //const loadOptions = (options, filterRestrictions = true) => { + //const loadOptions = (options, filterRestrictions = true) => { for (const option of options) { updateCurrentTraitMap(option.trait.trait, option.key) } @@ -306,8 +307,10 @@ export default function Selector({confirmDialog, templateInfo, animationManager, //create a gltf loader for the 3d models const gltfLoader = new GLTFLoader(loadingManager) + gltfLoader.crossOrigin = 'anonymous'; + console.log(vrmHelperRoot); gltfLoader.register((parser) => { - return new VRMLoaderPlugin(parser) + return new VRMLoaderPlugin(parser, {autoUpdateHumanBones: true}) }) // and a texture loaders for all the textures @@ -510,17 +513,19 @@ export default function Selector({confirmDialog, templateInfo, animationManager, if (getAsArray(templateInfo.lipSyncTraits).indexOf(traitData.trait) !== -1) setLipSync(new LipSync(vrm)); - renameVRMBones(vrm) + //renameVRMBones(vrm) if (getAsArray(templateInfo.blinkerTraits).indexOf(traitData.trait) !== -1) blinkManager.addBlinker(vrm) lookatManager.addVRM(vrm) - // animation setup section - // play animations on this vrm TODO, letscreate a single animation manager per traitInfo, as model may change since it is now a trait option + //animation setup section + //play animations on this vrm TODO, letscreate a single animation manager per traitInfo, as model may change since it is now a trait option animationManager.startAnimation(vrm) + //animationManager.loadMixamo(vrm) + // mesh target setup section if (item.meshTargets){ getAsArray(item.meshTargets).map((target) => { @@ -653,6 +658,7 @@ export default function Selector({confirmDialog, templateInfo, animationManager, if(vrm) { const m = vrm.scene; + console.log(vrm); m.visible = false; // add the now model to the current scene model.add(m) diff --git a/src/context/SceneContext.jsx b/src/context/SceneContext.jsx index 5e9856cd..0725264e 100644 --- a/src/context/SceneContext.jsx +++ b/src/context/SceneContext.jsx @@ -12,6 +12,9 @@ import { local } from "../library/store" export const SceneContext = createContext() export const SceneProvider = (props) => { + + const [vrmHelperRoot, setVrmHelperRoot] = useState(null); + const initializeScene = () => { const scene = new THREE.Scene() const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); @@ -22,10 +25,16 @@ export const SceneProvider = (props) => { directionalLight.position.set(0, 1, 1); scene.add(directionalLight); + const helperRoot = new THREE.Group(); + helperRoot.renderOrder = 10000; + scene.add( helperRoot ); + setVrmHelperRoot(helperRoot); + return scene; } const [scene, setScene] = useState(initializeScene) + const [currentTraitName, setCurrentTraitName] = useState(null) const [currentOptions, setCurrentOptions] = useState([]) @@ -193,6 +202,8 @@ export const SceneProvider = (props) => { return ( { } class AnimationControl { - constructor(animationManager, scene, animations, curIdx, lastIdx){ - this.mixer = new AnimationMixer(scene); + constructor(animationManager, scene, vrm, animations, curIdx, lastIdx){ + this.mixer = new THREE.AnimationMixer(scene); this.actions = []; this.to = null; this.from = null; + this.vrm = vrm; this.animationManager = null; this.animationManager = animationManager; + console.log(animations); animations[0].tracks.map((track, index) => { if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ animations[0].tracks.splice(index, 1) @@ -30,6 +34,8 @@ class AnimationControl { for (let i =0; i < animations.length;i++){ this.actions.push(this.mixer.clipAction(animations[i])); } + this.actions[0].play(); + console.log(this.actions); this.to = this.actions[curIdx] @@ -47,6 +53,19 @@ class AnimationControl { this.actions[curIdx].time = animationManager.getToActionTime(); this.actions[curIdx].play(); } + update(weightIn,weightOut){ + if (this.from != null) { + this.from.weight = weightOut; + } + if (this.to != null) { + this.to.weight = weightIn; + } + + this.mixer.update(1/30); + if (this.vrm != null){ + this.vrm.update(1/30); + } + } reset() { this.mixer.setTime(0); @@ -66,10 +85,10 @@ class AnimationControl { export class AnimationManager{ constructor (offset){ this.lastAnimID = null; - this.curAnimID = null; this.mainControl = null; this.animationControl = null; this.animations = null; + this.mixamoModel = null; this.weightIn = NaN; // note: can't set null, because of check `null < 1` will result `true`. this.weightOut = NaN; this.offset = null; @@ -78,7 +97,7 @@ export class AnimationManager{ this.animationControls = []; this.started = false; if (offset){ - this.offset = new Vector3( + this.offset = new THREE.Vector3( offset[0], offset[1], offset[2] @@ -88,16 +107,31 @@ export class AnimationManager{ this.update(); }, 1000/30); } + async loadAnimations(path){ + console.log(path) const loader = path.endsWith('.fbx') ? fbxLoader : gltfLoader; - const anim = await loader.loadAsync(path); - // offset hips - this.animations = anim.animations; - if (this.offset) - this.offsetHips(); + const animationModel = await loader.loadAsync(path); + + // if we have mixamo animations store the model + const clip = THREE.AnimationClip.findByName( animationModel.animations, 'mixamo.com' ); + if (clip != null){ + console.log("mixamo!") + this.mixamoModel = animationModel; + } + // if no mixamo animation is present, just save the animations + else{ + this.animations = animationModel.animations; + // offset hips + if (this.offset) + this.offsetHips(); + } + + - this.mainControl = new AnimationControl(this, anim, anim.animations, this.curAnimID, this.lastAnimID) + + this.mainControl = new AnimationControl(this, animationModel, null, animationModel.animations, this.curAnimID, this.lastAnimID) this.animationControls.push(this.mainControl) } @@ -129,21 +163,47 @@ export class AnimationManager{ } }); } + // loadMixamo(vrm){ + // console.log("load animmm") + + // const currentMixer = new THREE.AnimationMixer( vrm.scene ); + + // loadMixamoAnimation( './3d/animations/CapoeiraMixamo.fbx', vrm ).then( ( clip ) => { + // console.log(clip); + // // Apply the loaded animation to mixer and play + // currentMixer.clipAction( clip ).play(); + // currentMixer.timeScale = 1; + + // } ); + // } startAnimation(vrm){ - //return - if (!this.animations) { + let animations = null; + if (this.mixamoModel != null){ + console.log("has mixamo model"); + animations = [getMixamoAnimation(this.mixamoModel,vrm)] + if (this.animations == null) + this.animations = animations; + } + else{ + animations = this.animations; + } + + console.log(animations); + //const animation = + if (!animations) { console.warn("no animations were preloaded, ignoring"); return } - const animationControl = new AnimationControl(this, vrm.scene, this.animations, this.curAnimID, this.lastAnimID) + const animationControl = new AnimationControl(this, vrm.scene, vrm, animations, this.curAnimID, this.lastAnimID) this.animationControls.push(animationControl); addModelData(vrm , {animationControl}); - + console.log("an") if (this.started === false){ + console.log("start initial animation") this.started = true; - this.animRandomizer(this.animations[this.curAnimID].duration); + this.animRandomizer(animations[this.curAnimID].duration); } } @@ -203,14 +263,7 @@ export class AnimationManager{ update(){ if (this.mainControl) { this.animationControls.forEach(animControl => { - if (animControl.from != null) { - animControl.from.weight = this.weightOut; - } - if (animControl.to != null) { - animControl.to.weight = this.weightIn; - } - - animControl.mixer.update(1/30); + animControl.update(this.weightIn,this.weightOut); }); if (this.weightIn < 1) { diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js new file mode 100644 index 00000000..78975fe8 --- /dev/null +++ b/src/library/loadMixamoAnimation.js @@ -0,0 +1,99 @@ +import * as THREE from 'three'; +import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; + +/** + * Load Mixamo animation, convert for three-vrm use, and return it. + * + * @param {string} url A url of mixamo animation data + * @param {VRM} vrm A target VRM + * @returns {Promise} The converted AnimationClip + */ +export function getMixamoAnimation( asset, vrm ) { + + const clip = THREE.AnimationClip.findByName( asset.animations, 'mixamo.com' ); // extract the AnimationClip + + const tracks = []; // KeyframeTracks compatible with VRM will be added here + + const restRotationInverse = new THREE.Quaternion(); + const parentRestWorldRotation = new THREE.Quaternion(); + const _quatA = new THREE.Quaternion(); + const _vec3 = new THREE.Vector3(); + + // Adjust with reference to hips height. + const motionHipsHeight = asset.getObjectByName( 'mixamorigHips' ).position.y; + const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode( 'hips' ).getWorldPosition( _vec3 ).y; + const vrmRootY = vrm.scene.getWorldPosition( _vec3 ).y; + const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); + const hipsPositionScale = vrmHipsHeight / motionHipsHeight; + + clip.tracks.forEach( ( track ) => { + + // Convert each tracks for VRM use, and push to `tracks` + const trackSplitted = track.name.split( '.' ); + const mixamoRigName = trackSplitted[ 0 ]; + const vrmBoneName = VRMRigMapMixamo[ mixamoRigName ]; + const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; + //const vrmNodeName = vrmBoneName; + // console.log("name", vrmNodeName, vrmBoneName); + const mixamoRigNode = asset.getObjectByName( mixamoRigName ); + + if ( vrmNodeName != null ) { + + const propertyName = trackSplitted[ 1 ]; + + // Store rotations of rest-pose. + mixamoRigNode.getWorldQuaternion( restRotationInverse ).invert(); + mixamoRigNode.parent.getWorldQuaternion( parentRestWorldRotation ); + + if ( track instanceof THREE.QuaternionKeyframeTrack ) { + + // Retarget rotation of mixamoRig to NormalizedBone. + for ( let i = 0; i < track.values.length; i += 4 ) { + + const flatQuaternion = track.values.slice( i, i + 4 ); + + _quatA.fromArray( flatQuaternion ); + + // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆 + _quatA + .premultiply( parentRestWorldRotation ) + .multiply( restRotationInverse ); + + _quatA.toArray( flatQuaternion ); + + flatQuaternion.forEach( ( v, index ) => { + + track.values[ index + i ] = v; + + } ); + + } + + tracks.push( + new THREE.QuaternionKeyframeTrack( + `${vrmNodeName}.${propertyName}`, + track.times, + track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 2 === 0 ? - v : v ) ), + ), + ); + + } else if ( track instanceof THREE.VectorKeyframeTrack ) { + + const value = track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 3 !== 1 ? - v : v ) * hipsPositionScale ); + tracks.push( new THREE.VectorKeyframeTrack( `${vrmNodeName}.${propertyName}`, track.times, value ) ); + + } + + } + + } ); + + return new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); + // const loader = new FBXLoader(); // A loader which loads FBX + // return loader.loadAsync( url ).then( ( asset ) => { + + + + // } ); + +} \ No newline at end of file diff --git a/src/library/loadMixamoAnimation2.js b/src/library/loadMixamoAnimation2.js new file mode 100644 index 00000000..0f95fb6e --- /dev/null +++ b/src/library/loadMixamoAnimation2.js @@ -0,0 +1,97 @@ +import * as THREE from 'three'; +import { FBXLoader } from 'three/addons/loaders/FBXLoader.js'; +import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; + +/** + * Load Mixamo animation, convert for three-vrm use, and return it. + * + * @param {string} url A url of mixamo animation data + * @param {VRM} vrm A target VRM + * @returns {Promise} The converted AnimationClip + */ +export function loadMixamoAnimation( url, vrm ) { + + const loader = new FBXLoader(); // A loader which loads FBX + return loader.loadAsync( url ).then( ( asset ) => { + + const clip = THREE.AnimationClip.findByName( asset.animations, 'mixamo.com' ); // extract the AnimationClip + + const tracks = []; // KeyframeTracks compatible with VRM will be added here + + const restRotationInverse = new THREE.Quaternion(); + const parentRestWorldRotation = new THREE.Quaternion(); + const _quatA = new THREE.Quaternion(); + const _vec3 = new THREE.Vector3(); + + // Adjust with reference to hips height. + const motionHipsHeight = asset.getObjectByName( 'mixamorigHips' ).position.y; + const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode( 'hips' ).getWorldPosition( _vec3 ).y; + const vrmRootY = vrm.scene.getWorldPosition( _vec3 ).y; + const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); + const hipsPositionScale = vrmHipsHeight / motionHipsHeight; + + clip.tracks.forEach( ( track ) => { + + // Convert each tracks for VRM use, and push to `tracks` + const trackSplitted = track.name.split( '.' ); + const mixamoRigName = trackSplitted[ 0 ]; + const vrmBoneName = VRMRigMapMixamo[ mixamoRigName ]; + const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; + const mixamoRigNode = asset.getObjectByName( mixamoRigName ); + + if ( vrmNodeName != null ) { + + const propertyName = trackSplitted[ 1 ]; + + // Store rotations of rest-pose. + mixamoRigNode.getWorldQuaternion( restRotationInverse ).invert(); + mixamoRigNode.parent.getWorldQuaternion( parentRestWorldRotation ); + + if ( track instanceof THREE.QuaternionKeyframeTrack ) { + + // Retarget rotation of mixamoRig to NormalizedBone. + for ( let i = 0; i < track.values.length; i += 4 ) { + + const flatQuaternion = track.values.slice( i, i + 4 ); + + _quatA.fromArray( flatQuaternion ); + + // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆 + _quatA + .premultiply( parentRestWorldRotation ) + .multiply( restRotationInverse ); + + _quatA.toArray( flatQuaternion ); + + flatQuaternion.forEach( ( v, index ) => { + + track.values[ index + i ] = v; + + } ); + + } + + tracks.push( + new THREE.QuaternionKeyframeTrack( + `${vrmNodeName}.${propertyName}`, + track.times, + track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 2 === 0 ? - v : v ) ), + ), + ); + + } else if ( track instanceof THREE.VectorKeyframeTrack ) { + + const value = track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 3 !== 1 ? - v : v ) * hipsPositionScale ); + tracks.push( new THREE.VectorKeyframeTrack( `${vrmNodeName}.${propertyName}`, track.times, value ) ); + + } + + } + + } ); + + return new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); + + } ); + +} \ No newline at end of file From 00ac9cb2d745f3b6ffa56a678c68537cd6477a96 Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Wed, 11 Oct 2023 14:01:08 -0600 Subject: [PATCH 2/8] add mixamo animation suppoer with vrm update --- src/components/Selector.jsx | 2 -- src/library/animationManager.js | 28 ++++++++++------------------ src/library/loadMixamoAnimation.js | 17 ++++++++++------- src/library/loadMixamoAnimation2.js | 2 +- 4 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index 105e3eda..3819c080 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -524,8 +524,6 @@ export default function Selector({confirmDialog, templateInfo, animationManager, //play animations on this vrm TODO, letscreate a single animation manager per traitInfo, as model may change since it is now a trait option animationManager.startAnimation(vrm) - //animationManager.loadMixamo(vrm) - // mesh target setup section if (item.meshTargets){ getAsArray(item.meshTargets).map((target) => { diff --git a/src/library/animationManager.js b/src/library/animationManager.js index f388e73c..263c7e7f 100644 --- a/src/library/animationManager.js +++ b/src/library/animationManager.js @@ -23,7 +23,7 @@ class AnimationControl { this.vrm = vrm; this.animationManager = null; this.animationManager = animationManager; - console.log(animations); + console.log("animations", animations); animations[0].tracks.map((track, index) => { if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ animations[0].tracks.splice(index, 1) @@ -88,7 +88,7 @@ export class AnimationManager{ this.mainControl = null; this.animationControl = null; this.animations = null; - this.mixamoModel = null; + this.weightIn = NaN; // note: can't set null, because of check `null < 1` will result `true`. this.weightOut = NaN; this.offset = null; @@ -96,6 +96,10 @@ export class AnimationManager{ this.curAnimID = 0; this.animationControls = []; this.started = false; + + this.mixamoModel = null; + this.mixamoAnimations = null; + if (offset){ this.offset = new THREE.Vector3( offset[0], @@ -109,15 +113,14 @@ export class AnimationManager{ } async loadAnimations(path){ - console.log(path) const loader = path.endsWith('.fbx') ? fbxLoader : gltfLoader; const animationModel = await loader.loadAsync(path); - // if we have mixamo animations store the model const clip = THREE.AnimationClip.findByName( animationModel.animations, 'mixamo.com' ); if (clip != null){ console.log("mixamo!") - this.mixamoModel = animationModel; + this.mixamoModel = animationModel.clone(); + this.mixamoAnimations = animationModel.animations; } // if no mixamo animation is present, just save the animations else{ @@ -163,25 +166,14 @@ export class AnimationManager{ } }); } - // loadMixamo(vrm){ - // console.log("load animmm") - // const currentMixer = new THREE.AnimationMixer( vrm.scene ); - - // loadMixamoAnimation( './3d/animations/CapoeiraMixamo.fbx', vrm ).then( ( clip ) => { - // console.log(clip); - // // Apply the loaded animation to mixer and play - // currentMixer.clipAction( clip ).play(); - // currentMixer.timeScale = 1; - - // } ); - // } startAnimation(vrm){ let animations = null; if (this.mixamoModel != null){ + console.log("mixamoModel", this.mixamoModel) console.log("has mixamo model"); - animations = [getMixamoAnimation(this.mixamoModel,vrm)] + animations = [getMixamoAnimation(this.mixamoAnimations, this.mixamoModel.clone() ,vrm)] if (this.animations == null) this.animations = animations; } diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js index 78975fe8..80a1309a 100644 --- a/src/library/loadMixamoAnimation.js +++ b/src/library/loadMixamoAnimation.js @@ -8,9 +8,11 @@ import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; * @param {VRM} vrm A target VRM * @returns {Promise} The converted AnimationClip */ -export function getMixamoAnimation( asset, vrm ) { - - const clip = THREE.AnimationClip.findByName( asset.animations, 'mixamo.com' ); // extract the AnimationClip +export function getMixamoAnimation( animations, model, vrm ) { + console.log("animations", animations); + console.log("model", model); + console.log("vrm", vrm) + const clip = THREE.AnimationClip.findByName( animations, 'mixamo.com' ); // extract the AnimationClip const tracks = []; // KeyframeTracks compatible with VRM will be added here @@ -20,7 +22,7 @@ export function getMixamoAnimation( asset, vrm ) { const _vec3 = new THREE.Vector3(); // Adjust with reference to hips height. - const motionHipsHeight = asset.getObjectByName( 'mixamorigHips' ).position.y; + const motionHipsHeight = model.getObjectByName( 'mixamorigHips' ).position.y; const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode( 'hips' ).getWorldPosition( _vec3 ).y; const vrmRootY = vrm.scene.getWorldPosition( _vec3 ).y; const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); @@ -35,7 +37,7 @@ export function getMixamoAnimation( asset, vrm ) { const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; //const vrmNodeName = vrmBoneName; // console.log("name", vrmNodeName, vrmBoneName); - const mixamoRigNode = asset.getObjectByName( mixamoRigName ); + const mixamoRigNode = model.getObjectByName( mixamoRigName ); if ( vrmNodeName != null ) { @@ -87,8 +89,9 @@ export function getMixamoAnimation( asset, vrm ) { } } ); - - return new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); + const animClip = new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); + console.log("clip", animClip); + return animClip; // const loader = new FBXLoader(); // A loader which loads FBX // return loader.loadAsync( url ).then( ( asset ) => { diff --git a/src/library/loadMixamoAnimation2.js b/src/library/loadMixamoAnimation2.js index 0f95fb6e..425b6e79 100644 --- a/src/library/loadMixamoAnimation2.js +++ b/src/library/loadMixamoAnimation2.js @@ -89,7 +89,7 @@ export function loadMixamoAnimation( url, vrm ) { } } ); - + return new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); } ); From 0450bd01b05ba95ea7422900053ba798592b5f6b Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Wed, 11 Oct 2023 14:30:22 -0600 Subject: [PATCH 3/8] finish mixamo integration --- src/components/Selector.jsx | 2 +- src/library/animationManager.js | 7 +------ src/library/loadMixamoAnimation.js | 7 ++----- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index 3819c080..e69af7f1 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -513,7 +513,7 @@ export default function Selector({confirmDialog, templateInfo, animationManager, if (getAsArray(templateInfo.lipSyncTraits).indexOf(traitData.trait) !== -1) setLipSync(new LipSync(vrm)); - //renameVRMBones(vrm) + renameVRMBones(vrm) if (getAsArray(templateInfo.blinkerTraits).indexOf(traitData.trait) !== -1) blinkManager.addBlinker(vrm) diff --git a/src/library/animationManager.js b/src/library/animationManager.js index 263c7e7f..98136a22 100644 --- a/src/library/animationManager.js +++ b/src/library/animationManager.js @@ -3,7 +3,6 @@ import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader" import { FBXLoader } from "three/examples/jsm/loaders/FBXLoader" import { addModelData } from "./utils"; import { getMixamoAnimation } from './loadMixamoAnimation'; -import { loadMixamoAnimation } from './loadMixamoAnimation2'; // make a class that hold all the informarion const fbxLoader = new FBXLoader(); @@ -62,9 +61,6 @@ class AnimationControl { } this.mixer.update(1/30); - if (this.vrm != null){ - this.vrm.update(1/30); - } } reset() { @@ -118,9 +114,8 @@ export class AnimationManager{ // if we have mixamo animations store the model const clip = THREE.AnimationClip.findByName( animationModel.animations, 'mixamo.com' ); if (clip != null){ - console.log("mixamo!") this.mixamoModel = animationModel.clone(); - this.mixamoAnimations = animationModel.animations; + this.mixamoAnimations = animationModel.animations; } // if no mixamo animation is present, just save the animations else{ diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js index 80a1309a..f0179909 100644 --- a/src/library/loadMixamoAnimation.js +++ b/src/library/loadMixamoAnimation.js @@ -9,9 +9,6 @@ import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; * @returns {Promise} The converted AnimationClip */ export function getMixamoAnimation( animations, model, vrm ) { - console.log("animations", animations); - console.log("model", model); - console.log("vrm", vrm) const clip = THREE.AnimationClip.findByName( animations, 'mixamo.com' ); // extract the AnimationClip const tracks = []; // KeyframeTracks compatible with VRM will be added here @@ -34,8 +31,8 @@ export function getMixamoAnimation( animations, model, vrm ) { const trackSplitted = track.name.split( '.' ); const mixamoRigName = trackSplitted[ 0 ]; const vrmBoneName = VRMRigMapMixamo[ mixamoRigName ]; - const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; - //const vrmNodeName = vrmBoneName; + //const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; + const vrmNodeName = vrmBoneName; // console.log("name", vrmNodeName, vrmBoneName); const mixamoRigNode = model.getObjectByName( mixamoRigName ); From 1d656015d717d46b5ec18b91cd144e33d18d8310 Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Wed, 11 Oct 2023 14:41:08 -0600 Subject: [PATCH 4/8] cleanup logs --- src/components/Selector.jsx | 6 +----- src/library/animationManager.js | 8 -------- src/library/loadMixamoAnimation.js | 7 ------- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index e69af7f1..86cb40f7 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -308,8 +308,8 @@ export default function Selector({confirmDialog, templateInfo, animationManager, //create a gltf loader for the 3d models const gltfLoader = new GLTFLoader(loadingManager) gltfLoader.crossOrigin = 'anonymous'; - console.log(vrmHelperRoot); gltfLoader.register((parser) => { + //return new VRMLoaderPlugin(parser, {autoUpdateHumanBones: true, helperRoot:vrmHelperRoot}) return new VRMLoaderPlugin(parser, {autoUpdateHumanBones: true}) }) @@ -331,9 +331,6 @@ export default function Selector({confirmDialog, templateInfo, animationManager, loadingManager.onError = function (url){ console.log(resultData); console.warn("error loading " + url) - // setLoadPercentage(0) - // resolve(resultData); - // setIsLoading(false) } loadingManager.onProgress = function(url, loaded, total){ setLoadPercentage(Math.round(loaded/total * 100 )) @@ -656,7 +653,6 @@ export default function Selector({confirmDialog, templateInfo, animationManager, if(vrm) { const m = vrm.scene; - console.log(vrm); m.visible = false; // add the now model to the current scene model.add(m) diff --git a/src/library/animationManager.js b/src/library/animationManager.js index 98136a22..4becf4d0 100644 --- a/src/library/animationManager.js +++ b/src/library/animationManager.js @@ -22,7 +22,6 @@ class AnimationControl { this.vrm = vrm; this.animationManager = null; this.animationManager = animationManager; - console.log("animations", animations); animations[0].tracks.map((track, index) => { if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ animations[0].tracks.splice(index, 1) @@ -34,7 +33,6 @@ class AnimationControl { this.actions.push(this.mixer.clipAction(animations[i])); } this.actions[0].play(); - console.log(this.actions); this.to = this.actions[curIdx] @@ -166,8 +164,6 @@ export class AnimationManager{ startAnimation(vrm){ let animations = null; if (this.mixamoModel != null){ - console.log("mixamoModel", this.mixamoModel) - console.log("has mixamo model"); animations = [getMixamoAnimation(this.mixamoAnimations, this.mixamoModel.clone() ,vrm)] if (this.animations == null) this.animations = animations; @@ -175,8 +171,6 @@ export class AnimationManager{ else{ animations = this.animations; } - - console.log(animations); //const animation = if (!animations) { console.warn("no animations were preloaded, ignoring"); @@ -186,9 +180,7 @@ export class AnimationManager{ this.animationControls.push(animationControl); addModelData(vrm , {animationControl}); - console.log("an") if (this.started === false){ - console.log("start initial animation") this.started = true; this.animRandomizer(animations[this.curAnimID].duration); } diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js index f0179909..fdda03ec 100644 --- a/src/library/loadMixamoAnimation.js +++ b/src/library/loadMixamoAnimation.js @@ -87,13 +87,6 @@ export function getMixamoAnimation( animations, model, vrm ) { } ); const animClip = new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); - console.log("clip", animClip); return animClip; - // const loader = new FBXLoader(); // A loader which loads FBX - // return loader.loadAsync( url ).then( ( asset ) => { - - - - // } ); } \ No newline at end of file From adada7ec132de7b353c1231fa59d78002d663d57 Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Thu, 12 Oct 2023 00:56:38 -0600 Subject: [PATCH 5/8] add drag and drop component --- src/components/FileDropComponent.jsx | 49 +++++++++++++++++++++ src/components/FileDropComponent.module.css | 13 ++++++ src/pages/Appearance.jsx | 9 ++++ 3 files changed, 71 insertions(+) create mode 100644 src/components/FileDropComponent.jsx create mode 100644 src/components/FileDropComponent.module.css diff --git a/src/components/FileDropComponent.jsx b/src/components/FileDropComponent.jsx new file mode 100644 index 00000000..4a4c0b17 --- /dev/null +++ b/src/components/FileDropComponent.jsx @@ -0,0 +1,49 @@ +// FileDropComponent.js +import React, { useEffect, useState } from 'react'; +import styles from './FileDropComponent.module.css'; + +export default function FileDropComponent ({onFileDrop}){ + const [isDragging, setIsDragging] = useState(false); + + useEffect(() => { + const handleDrop = (event) => { + event.preventDefault(); + setIsDragging(false); + console.log(onFileDrop); + const file = event.dataTransfer.files[0]; + console.log('Dropped file:', file); + if (onFileDrop) { + onFileDrop(file); + } + }; + + const handleDragOver = (event) => { + event.preventDefault(); + setIsDragging(true); + }; + + // Attach event listeners to the window + window.addEventListener('drop', handleDrop); + window.addEventListener('dragover', handleDragOver); + + // Clean up event listeners on component unmount + return () => { + window.removeEventListener('drop', handleDrop); + window.removeEventListener('dragover', handleDragOver); + }; + }, []); + + const handleDragLeave = () => { + setIsDragging(false); + }; + + return ( +
+
+ ); +}; + diff --git a/src/components/FileDropComponent.module.css b/src/components/FileDropComponent.module.css new file mode 100644 index 00000000..26db7856 --- /dev/null +++ b/src/components/FileDropComponent.module.css @@ -0,0 +1,13 @@ +.dropArea { + height: 100vh; + width: 100vw; + border: '2px dashed #aaa'; + background-size: cover; + text-align: 'center'; + display: fixed; + flex-direction: column; + align-items: center; + overflow: hidden; + position: absolute; + z-index: 10000; +} \ No newline at end of file diff --git a/src/pages/Appearance.jsx b/src/pages/Appearance.jsx index 8691a159..0ed72b4e 100644 --- a/src/pages/Appearance.jsx +++ b/src/pages/Appearance.jsx @@ -7,6 +7,7 @@ import CustomButton from "../components/custom-button" import { LanguageContext } from "../context/LanguageContext" import { SoundContext } from "../context/SoundContext" import { AudioContext } from "../context/AudioContext" +import FileDropComponent from "../components/FileDropComponent" function Appearance({ animationManager, @@ -75,12 +76,20 @@ function Appearance({ // Translate hook const { t } = useContext(LanguageContext) + const handleFileDrop = (file) => { + // Handle the dropped file as needed + console.log('Handling dropped file:', file); + }; + return (
{t("pageTitles.chooseAppearance")}
+ Date: Thu, 12 Oct 2023 09:36:06 -0600 Subject: [PATCH 6/8] add drag and drop animations from mixamo --- src/App.jsx | 2 +- src/components/FileDropComponent.jsx | 2 - src/components/Selector.jsx | 2 +- src/library/animationManager.js | 55 ++++++++++++++++++---------- src/pages/Appearance.jsx | 9 ++++- 5 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/App.jsx b/src/App.jsx index 2b34dabc..6ec153d3 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -92,7 +92,7 @@ async function fetchScene() { async function fetchAnimation(templateInfo) { // create an animation manager for all the traits that will be loaded const newAnimationManager = new AnimationManager(templateInfo.offset) - await newAnimationManager.loadAnimations(templateInfo.animationPath) + await newAnimationManager.loadAnimations(templateInfo.animationPath, templateInfo.animationPath.endsWith('.fbx')) return newAnimationManager } diff --git a/src/components/FileDropComponent.jsx b/src/components/FileDropComponent.jsx index 4a4c0b17..bb9792d0 100644 --- a/src/components/FileDropComponent.jsx +++ b/src/components/FileDropComponent.jsx @@ -9,9 +9,7 @@ export default function FileDropComponent ({onFileDrop}){ const handleDrop = (event) => { event.preventDefault(); setIsDragging(false); - console.log(onFileDrop); const file = event.dataTransfer.files[0]; - console.log('Dropped file:', file); if (onFileDrop) { onFileDrop(file); } diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index 86cb40f7..5a9aa0eb 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -329,7 +329,7 @@ export default function Selector({confirmDialog, templateInfo, animationManager, setIsLoading(false) }; loadingManager.onError = function (url){ - console.log(resultData); + console.log("currentTraits", resultData); console.warn("error loading " + url) } loadingManager.onProgress = function(url, loaded, total){ diff --git a/src/library/animationManager.js b/src/library/animationManager.js index 4becf4d0..66990aa9 100644 --- a/src/library/animationManager.js +++ b/src/library/animationManager.js @@ -22,17 +22,8 @@ class AnimationControl { this.vrm = vrm; this.animationManager = null; this.animationManager = animationManager; - animations[0].tracks.map((track, index) => { - if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ - animations[0].tracks.splice(index, 1) - } - }) - // animations[0].tracks.splice(9, 2); - this.actions = []; - for (let i =0; i < animations.length;i++){ - this.actions.push(this.mixer.clipAction(animations[i])); - } - this.actions[0].play(); + + this.setAnimations(animations); this.to = this.actions[curIdx] @@ -50,6 +41,26 @@ class AnimationControl { this.actions[curIdx].time = animationManager.getToActionTime(); this.actions[curIdx].play(); } + setAnimations(animations, mixamoModel){ + this.mixer.stopAllAction(); + if (mixamoModel != null){ + if (this.vrm != null) + animations = [getMixamoAnimation(animations, mixamoModel , this.vrm)] + // modify animations + } + animations[0].tracks.map((track, index) => { + if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ + animations[0].tracks.splice(index, 1) + } + }) + + this.actions = []; + for (let i =0; i < animations.length;i++){ + this.actions.push(this.mixer.clipAction(animations[i])); + } + this.actions[0].play(); + } + update(weightIn,weightOut){ if (this.from != null) { this.from.weight = weightOut; @@ -106,8 +117,8 @@ export class AnimationManager{ }, 1000/30); } - async loadAnimations(path){ - const loader = path.endsWith('.fbx') ? fbxLoader : gltfLoader; + async loadAnimations(path, isfbx = true){ + const loader = isfbx ? fbxLoader : gltfLoader; const animationModel = await loader.loadAsync(path); // if we have mixamo animations store the model const clip = THREE.AnimationClip.findByName( animationModel.animations, 'mixamo.com' ); @@ -117,18 +128,22 @@ export class AnimationManager{ } // if no mixamo animation is present, just save the animations else{ + this.mixamoModel = null this.animations = animationModel.animations; - // offset hips if (this.offset) this.offsetHips(); } - - - - - this.mainControl = new AnimationControl(this, animationModel, null, animationModel.animations, this.curAnimID, this.lastAnimID) - this.animationControls.push(this.mainControl) + if (this.mainControl == null){ + this.mainControl = new AnimationControl(this, animationModel, null, animationModel.animations, this.curAnimID, this.lastAnimID) + this.animationControls.push(this.mainControl) + } + else{ + //cons + this.animationControls.forEach(animationControl => { + animationControl.setAnimations(animationModel.animations, this.mixamoModel) + }); + } } diff --git a/src/pages/Appearance.jsx b/src/pages/Appearance.jsx index 0ed72b4e..c72c6727 100644 --- a/src/pages/Appearance.jsx +++ b/src/pages/Appearance.jsx @@ -77,8 +77,13 @@ function Appearance({ const { t } = useContext(LanguageContext) const handleFileDrop = (file) => { - // Handle the dropped file as needed - console.log('Handling dropped file:', file); + // Check if the file has the .fbx extension + if (file && file.name.toLowerCase().endsWith('.fbx')) { + console.log('Dropped .fbx file:', file); + const path = URL.createObjectURL(file); + animationManager.loadAnimations(path, true); + // Handle the dropped .fbx file + } }; return ( From b4cbec6696adf11b8bb971d9337681fcbf563089 Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Thu, 12 Oct 2023 13:05:52 -0600 Subject: [PATCH 7/8] remove test code --- src/library/loadMixamoAnimation2.js | 97 ----------------------------- 1 file changed, 97 deletions(-) delete mode 100644 src/library/loadMixamoAnimation2.js diff --git a/src/library/loadMixamoAnimation2.js b/src/library/loadMixamoAnimation2.js deleted file mode 100644 index 425b6e79..00000000 --- a/src/library/loadMixamoAnimation2.js +++ /dev/null @@ -1,97 +0,0 @@ -import * as THREE from 'three'; -import { FBXLoader } from 'three/addons/loaders/FBXLoader.js'; -import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; - -/** - * Load Mixamo animation, convert for three-vrm use, and return it. - * - * @param {string} url A url of mixamo animation data - * @param {VRM} vrm A target VRM - * @returns {Promise} The converted AnimationClip - */ -export function loadMixamoAnimation( url, vrm ) { - - const loader = new FBXLoader(); // A loader which loads FBX - return loader.loadAsync( url ).then( ( asset ) => { - - const clip = THREE.AnimationClip.findByName( asset.animations, 'mixamo.com' ); // extract the AnimationClip - - const tracks = []; // KeyframeTracks compatible with VRM will be added here - - const restRotationInverse = new THREE.Quaternion(); - const parentRestWorldRotation = new THREE.Quaternion(); - const _quatA = new THREE.Quaternion(); - const _vec3 = new THREE.Vector3(); - - // Adjust with reference to hips height. - const motionHipsHeight = asset.getObjectByName( 'mixamorigHips' ).position.y; - const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode( 'hips' ).getWorldPosition( _vec3 ).y; - const vrmRootY = vrm.scene.getWorldPosition( _vec3 ).y; - const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); - const hipsPositionScale = vrmHipsHeight / motionHipsHeight; - - clip.tracks.forEach( ( track ) => { - - // Convert each tracks for VRM use, and push to `tracks` - const trackSplitted = track.name.split( '.' ); - const mixamoRigName = trackSplitted[ 0 ]; - const vrmBoneName = VRMRigMapMixamo[ mixamoRigName ]; - const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; - const mixamoRigNode = asset.getObjectByName( mixamoRigName ); - - if ( vrmNodeName != null ) { - - const propertyName = trackSplitted[ 1 ]; - - // Store rotations of rest-pose. - mixamoRigNode.getWorldQuaternion( restRotationInverse ).invert(); - mixamoRigNode.parent.getWorldQuaternion( parentRestWorldRotation ); - - if ( track instanceof THREE.QuaternionKeyframeTrack ) { - - // Retarget rotation of mixamoRig to NormalizedBone. - for ( let i = 0; i < track.values.length; i += 4 ) { - - const flatQuaternion = track.values.slice( i, i + 4 ); - - _quatA.fromArray( flatQuaternion ); - - // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆 - _quatA - .premultiply( parentRestWorldRotation ) - .multiply( restRotationInverse ); - - _quatA.toArray( flatQuaternion ); - - flatQuaternion.forEach( ( v, index ) => { - - track.values[ index + i ] = v; - - } ); - - } - - tracks.push( - new THREE.QuaternionKeyframeTrack( - `${vrmNodeName}.${propertyName}`, - track.times, - track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 2 === 0 ? - v : v ) ), - ), - ); - - } else if ( track instanceof THREE.VectorKeyframeTrack ) { - - const value = track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 3 !== 1 ? - v : v ) * hipsPositionScale ); - tracks.push( new THREE.VectorKeyframeTrack( `${vrmNodeName}.${propertyName}`, track.times, value ) ); - - } - - } - - } ); - - return new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); - - } ); - -} \ No newline at end of file From 5a67b32d2abfedd01e997bcd7a72156ce9226fec Mon Sep 17 00:00:00 2001 From: memelotsqui Date: Fri, 13 Oct 2023 15:15:33 -0600 Subject: [PATCH 8/8] fix double inverses --- src/library/loadMixamoAnimation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js index fdda03ec..1b51e703 100644 --- a/src/library/loadMixamoAnimation.js +++ b/src/library/loadMixamoAnimation.js @@ -25,8 +25,8 @@ export function getMixamoAnimation( animations, model, vrm ) { const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); const hipsPositionScale = vrmHipsHeight / motionHipsHeight; - clip.tracks.forEach( ( track ) => { - + clip.tracks.forEach( ( origTrack ) => { + const track = origTrack.clone(); // Convert each tracks for VRM use, and push to `tracks` const trackSplitted = track.name.split( '.' ); const mixamoRigName = trackSplitted[ 0 ];