diff --git a/src/App.jsx b/src/App.jsx index 2b34dabc..6ec153d3 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -92,7 +92,7 @@ async function fetchScene() { async function fetchAnimation(templateInfo) { // create an animation manager for all the traits that will be loaded const newAnimationManager = new AnimationManager(templateInfo.offset) - await newAnimationManager.loadAnimations(templateInfo.animationPath) + await newAnimationManager.loadAnimations(templateInfo.animationPath, templateInfo.animationPath.endsWith('.fbx')) return newAnimationManager } diff --git a/src/components/FileDropComponent.jsx b/src/components/FileDropComponent.jsx new file mode 100644 index 00000000..bb9792d0 --- /dev/null +++ b/src/components/FileDropComponent.jsx @@ -0,0 +1,47 @@ +// FileDropComponent.js +import React, { useEffect, useState } from 'react'; +import styles from './FileDropComponent.module.css'; + +export default function FileDropComponent ({onFileDrop}){ + const [isDragging, setIsDragging] = useState(false); + + useEffect(() => { + const handleDrop = (event) => { + event.preventDefault(); + setIsDragging(false); + const file = event.dataTransfer.files[0]; + if (onFileDrop) { + onFileDrop(file); + } + }; + + const handleDragOver = (event) => { + event.preventDefault(); + setIsDragging(true); + }; + + // Attach event listeners to the window + window.addEventListener('drop', handleDrop); + window.addEventListener('dragover', handleDragOver); + + // Clean up event listeners on component unmount + return () => { + window.removeEventListener('drop', handleDrop); + window.removeEventListener('dragover', handleDragOver); + }; + }, []); + + const handleDragLeave = () => { + setIsDragging(false); + }; + + return ( +
+
+ ); +}; + diff --git a/src/components/FileDropComponent.module.css b/src/components/FileDropComponent.module.css new file mode 100644 index 00000000..26db7856 --- /dev/null +++ b/src/components/FileDropComponent.module.css @@ -0,0 +1,13 @@ +.dropArea { + height: 100vh; + width: 100vw; + border: '2px dashed #aaa'; + background-size: cover; + text-align: 'center'; + display: fixed; + flex-direction: column; + align-items: center; + overflow: hidden; + position: absolute; + z-index: 10000; +} \ No newline at end of file diff --git a/src/components/Selector.jsx b/src/components/Selector.jsx index 78ead7f5..5a9aa0eb 100644 --- a/src/components/Selector.jsx +++ b/src/components/Selector.jsx @@ -1,7 +1,7 @@ import React, { useContext, useEffect, useState } from "react" import * as THREE from "three" import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader" -import { MToonMaterial, VRMLoaderPlugin } from "@pixiv/three-vrm" +import { MToonMaterial, VRMLoaderPlugin, VRMUtils } from "@pixiv/three-vrm" import cancel from "../../public/ui/selector/cancel.png" import { addModelData, disposeVRM } from "../library/utils" import { computeBoundsTree, disposeBoundsTree, acceleratedRaycast, SAH } from 'three-mesh-bvh'; @@ -42,7 +42,8 @@ export default function Selector({confirmDialog, templateInfo, animationManager, removeOption, saveUserSelection, setIsChangingWholeAvatar, - debugMode + debugMode, + vrmHelperRoot } = useContext(SceneContext) const { playSound @@ -274,7 +275,7 @@ export default function Selector({confirmDialog, templateInfo, animationManager, // load options first const loadOptions = (options, filterRestrictions = true, useTemplateBaseDirectory = true, saveUserSel = true) => { - //const loadOptions = (options, filterRestrictions = true) => { + //const loadOptions = (options, filterRestrictions = true) => { for (const option of options) { updateCurrentTraitMap(option.trait.trait, option.key) } @@ -306,8 +307,10 @@ export default function Selector({confirmDialog, templateInfo, animationManager, //create a gltf loader for the 3d models const gltfLoader = new GLTFLoader(loadingManager) + gltfLoader.crossOrigin = 'anonymous'; gltfLoader.register((parser) => { - return new VRMLoaderPlugin(parser) + //return new VRMLoaderPlugin(parser, {autoUpdateHumanBones: true, helperRoot:vrmHelperRoot}) + return new VRMLoaderPlugin(parser, {autoUpdateHumanBones: true}) }) // and a texture loaders for all the textures @@ -326,11 +329,8 @@ export default function Selector({confirmDialog, templateInfo, animationManager, setIsLoading(false) }; loadingManager.onError = function (url){ - console.log(resultData); + console.log("currentTraits", resultData); console.warn("error loading " + url) - // setLoadPercentage(0) - // resolve(resultData); - // setIsLoading(false) } loadingManager.onProgress = function(url, loaded, total){ setLoadPercentage(Math.round(loaded/total * 100 )) @@ -517,8 +517,8 @@ export default function Selector({confirmDialog, templateInfo, animationManager, lookatManager.addVRM(vrm) - // animation setup section - // play animations on this vrm TODO, letscreate a single animation manager per traitInfo, as model may change since it is now a trait option + //animation setup section + //play animations on this vrm TODO, letscreate a single animation manager per traitInfo, as model may change since it is now a trait option animationManager.startAnimation(vrm) // mesh target setup section diff --git a/src/context/SceneContext.jsx b/src/context/SceneContext.jsx index 5e9856cd..0725264e 100644 --- a/src/context/SceneContext.jsx +++ b/src/context/SceneContext.jsx @@ -12,6 +12,9 @@ import { local } from "../library/store" export const SceneContext = createContext() export const SceneProvider = (props) => { + + const [vrmHelperRoot, setVrmHelperRoot] = useState(null); + const initializeScene = () => { const scene = new THREE.Scene() const ambientLight = new THREE.AmbientLight(0xffffff, 0.5); @@ -22,10 +25,16 @@ export const SceneProvider = (props) => { directionalLight.position.set(0, 1, 1); scene.add(directionalLight); + const helperRoot = new THREE.Group(); + helperRoot.renderOrder = 10000; + scene.add( helperRoot ); + setVrmHelperRoot(helperRoot); + return scene; } const [scene, setScene] = useState(initializeScene) + const [currentTraitName, setCurrentTraitName] = useState(null) const [currentOptions, setCurrentOptions] = useState([]) @@ -193,6 +202,8 @@ export const SceneProvider = (props) => { return ( { } class AnimationControl { - constructor(animationManager, scene, animations, curIdx, lastIdx){ - this.mixer = new AnimationMixer(scene); + constructor(animationManager, scene, vrm, animations, curIdx, lastIdx){ + this.mixer = new THREE.AnimationMixer(scene); this.actions = []; this.to = null; this.from = null; + this.vrm = vrm; this.animationManager = null; this.animationManager = animationManager; - animations[0].tracks.map((track, index) => { - if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ - animations[0].tracks.splice(index, 1) - } - }) - // animations[0].tracks.splice(9, 2); - this.actions = []; - for (let i =0; i < animations.length;i++){ - this.actions.push(this.mixer.clipAction(animations[i])); - } + + this.setAnimations(animations); this.to = this.actions[curIdx] @@ -47,6 +41,36 @@ class AnimationControl { this.actions[curIdx].time = animationManager.getToActionTime(); this.actions[curIdx].play(); } + setAnimations(animations, mixamoModel){ + this.mixer.stopAllAction(); + if (mixamoModel != null){ + if (this.vrm != null) + animations = [getMixamoAnimation(animations, mixamoModel , this.vrm)] + // modify animations + } + animations[0].tracks.map((track, index) => { + if(track.name === "neck.quaternion" || track.name === "spine.quaternion"){ + animations[0].tracks.splice(index, 1) + } + }) + + this.actions = []; + for (let i =0; i < animations.length;i++){ + this.actions.push(this.mixer.clipAction(animations[i])); + } + this.actions[0].play(); + } + + update(weightIn,weightOut){ + if (this.from != null) { + this.from.weight = weightOut; + } + if (this.to != null) { + this.to.weight = weightIn; + } + + this.mixer.update(1/30); + } reset() { this.mixer.setTime(0); @@ -66,10 +90,10 @@ class AnimationControl { export class AnimationManager{ constructor (offset){ this.lastAnimID = null; - this.curAnimID = null; this.mainControl = null; this.animationControl = null; this.animations = null; + this.weightIn = NaN; // note: can't set null, because of check `null < 1` will result `true`. this.weightOut = NaN; this.offset = null; @@ -77,8 +101,12 @@ export class AnimationManager{ this.curAnimID = 0; this.animationControls = []; this.started = false; + + this.mixamoModel = null; + this.mixamoAnimations = null; + if (offset){ - this.offset = new Vector3( + this.offset = new THREE.Vector3( offset[0], offset[1], offset[2] @@ -88,17 +116,34 @@ export class AnimationManager{ this.update(); }, 1000/30); } - async loadAnimations(path){ - const loader = path.endsWith('.fbx') ? fbxLoader : gltfLoader; - const anim = await loader.loadAsync(path); - // offset hips - this.animations = anim.animations; - if (this.offset) - this.offsetHips(); - - - this.mainControl = new AnimationControl(this, anim, anim.animations, this.curAnimID, this.lastAnimID) - this.animationControls.push(this.mainControl) + + async loadAnimations(path, isfbx = true){ + const loader = isfbx ? fbxLoader : gltfLoader; + const animationModel = await loader.loadAsync(path); + // if we have mixamo animations store the model + const clip = THREE.AnimationClip.findByName( animationModel.animations, 'mixamo.com' ); + if (clip != null){ + this.mixamoModel = animationModel.clone(); + this.mixamoAnimations = animationModel.animations; + } + // if no mixamo animation is present, just save the animations + else{ + this.mixamoModel = null + this.animations = animationModel.animations; + if (this.offset) + this.offsetHips(); + } + + if (this.mainControl == null){ + this.mainControl = new AnimationControl(this, animationModel, null, animationModel.animations, this.curAnimID, this.lastAnimID) + this.animationControls.push(this.mainControl) + } + else{ + //cons + this.animationControls.forEach(animationControl => { + animationControl.setAnimations(animationModel.animations, this.mixamoModel) + }); + } } @@ -130,20 +175,29 @@ export class AnimationManager{ }); } + startAnimation(vrm){ - //return - if (!this.animations) { + let animations = null; + if (this.mixamoModel != null){ + animations = [getMixamoAnimation(this.mixamoAnimations, this.mixamoModel.clone() ,vrm)] + if (this.animations == null) + this.animations = animations; + } + else{ + animations = this.animations; + } + //const animation = + if (!animations) { console.warn("no animations were preloaded, ignoring"); return } - const animationControl = new AnimationControl(this, vrm.scene, this.animations, this.curAnimID, this.lastAnimID) + const animationControl = new AnimationControl(this, vrm.scene, vrm, animations, this.curAnimID, this.lastAnimID) this.animationControls.push(animationControl); addModelData(vrm , {animationControl}); - if (this.started === false){ this.started = true; - this.animRandomizer(this.animations[this.curAnimID].duration); + this.animRandomizer(animations[this.curAnimID].duration); } } @@ -203,14 +257,7 @@ export class AnimationManager{ update(){ if (this.mainControl) { this.animationControls.forEach(animControl => { - if (animControl.from != null) { - animControl.from.weight = this.weightOut; - } - if (animControl.to != null) { - animControl.to.weight = this.weightIn; - } - - animControl.mixer.update(1/30); + animControl.update(this.weightIn,this.weightOut); }); if (this.weightIn < 1) { diff --git a/src/library/loadMixamoAnimation.js b/src/library/loadMixamoAnimation.js new file mode 100644 index 00000000..1b51e703 --- /dev/null +++ b/src/library/loadMixamoAnimation.js @@ -0,0 +1,92 @@ +import * as THREE from 'three'; +import { VRMRigMapMixamo } from './VRMRigMapMixamo.js'; + +/** + * Load Mixamo animation, convert for three-vrm use, and return it. + * + * @param {string} url A url of mixamo animation data + * @param {VRM} vrm A target VRM + * @returns {Promise} The converted AnimationClip + */ +export function getMixamoAnimation( animations, model, vrm ) { + const clip = THREE.AnimationClip.findByName( animations, 'mixamo.com' ); // extract the AnimationClip + + const tracks = []; // KeyframeTracks compatible with VRM will be added here + + const restRotationInverse = new THREE.Quaternion(); + const parentRestWorldRotation = new THREE.Quaternion(); + const _quatA = new THREE.Quaternion(); + const _vec3 = new THREE.Vector3(); + + // Adjust with reference to hips height. + const motionHipsHeight = model.getObjectByName( 'mixamorigHips' ).position.y; + const vrmHipsY = vrm.humanoid?.getNormalizedBoneNode( 'hips' ).getWorldPosition( _vec3 ).y; + const vrmRootY = vrm.scene.getWorldPosition( _vec3 ).y; + const vrmHipsHeight = Math.abs( vrmHipsY - vrmRootY ); + const hipsPositionScale = vrmHipsHeight / motionHipsHeight; + + clip.tracks.forEach( ( origTrack ) => { + const track = origTrack.clone(); + // Convert each tracks for VRM use, and push to `tracks` + const trackSplitted = track.name.split( '.' ); + const mixamoRigName = trackSplitted[ 0 ]; + const vrmBoneName = VRMRigMapMixamo[ mixamoRigName ]; + //const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode( vrmBoneName )?.name; + const vrmNodeName = vrmBoneName; + // console.log("name", vrmNodeName, vrmBoneName); + const mixamoRigNode = model.getObjectByName( mixamoRigName ); + + if ( vrmNodeName != null ) { + + const propertyName = trackSplitted[ 1 ]; + + // Store rotations of rest-pose. + mixamoRigNode.getWorldQuaternion( restRotationInverse ).invert(); + mixamoRigNode.parent.getWorldQuaternion( parentRestWorldRotation ); + + if ( track instanceof THREE.QuaternionKeyframeTrack ) { + + // Retarget rotation of mixamoRig to NormalizedBone. + for ( let i = 0; i < track.values.length; i += 4 ) { + + const flatQuaternion = track.values.slice( i, i + 4 ); + + _quatA.fromArray( flatQuaternion ); + + // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆 + _quatA + .premultiply( parentRestWorldRotation ) + .multiply( restRotationInverse ); + + _quatA.toArray( flatQuaternion ); + + flatQuaternion.forEach( ( v, index ) => { + + track.values[ index + i ] = v; + + } ); + + } + + tracks.push( + new THREE.QuaternionKeyframeTrack( + `${vrmNodeName}.${propertyName}`, + track.times, + track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 2 === 0 ? - v : v ) ), + ), + ); + + } else if ( track instanceof THREE.VectorKeyframeTrack ) { + + const value = track.values.map( ( v, i ) => ( vrm.meta?.metaVersion === '0' && i % 3 !== 1 ? - v : v ) * hipsPositionScale ); + tracks.push( new THREE.VectorKeyframeTrack( `${vrmNodeName}.${propertyName}`, track.times, value ) ); + + } + + } + + } ); + const animClip = new THREE.AnimationClip( 'vrmAnimation', clip.duration, tracks ); + return animClip; + +} \ No newline at end of file diff --git a/src/pages/Appearance.jsx b/src/pages/Appearance.jsx index 8691a159..c72c6727 100644 --- a/src/pages/Appearance.jsx +++ b/src/pages/Appearance.jsx @@ -7,6 +7,7 @@ import CustomButton from "../components/custom-button" import { LanguageContext } from "../context/LanguageContext" import { SoundContext } from "../context/SoundContext" import { AudioContext } from "../context/AudioContext" +import FileDropComponent from "../components/FileDropComponent" function Appearance({ animationManager, @@ -75,12 +76,25 @@ function Appearance({ // Translate hook const { t } = useContext(LanguageContext) + const handleFileDrop = (file) => { + // Check if the file has the .fbx extension + if (file && file.name.toLowerCase().endsWith('.fbx')) { + console.log('Dropped .fbx file:', file); + const path = URL.createObjectURL(file); + animationManager.loadAnimations(path, true); + // Handle the dropped .fbx file + } + }; + return (
{t("pageTitles.chooseAppearance")}
+