Grunt voice and mouth movement

avatars/avatars.js

@@ -948,6 +948,8 @@ class Avatar {
     this.startEyeTargetQuaternion = new THREE.Quaternion();
     this.lastNeedsEyeTarget = false;
     this.lastEyeTargetTime = -Infinity;
+
+    this.manuallySetMouth=false;
   }
   static bindAvatar(object) {
     const model = object.scene;
@@ -1649,16 +1651,16 @@ class Avatar {
           if (aIndex !== -1) {
             morphTargetInfluences[aIndex] = volumeValue;
           }
-          if (eIndex !== -1) {
+          if (eIndex !== -1 && !this.manuallySetMouth) {
             morphTargetInfluences[eIndex] = volumeValue * this.vowels[1];
           }
-          if (iIndex !== -1) {
+          if (iIndex !== -1 && !this.manuallySetMouth) {
             morphTargetInfluences[iIndex] = volumeValue * this.vowels[2];
           }
-          if (oIndex !== -1) {
+          if (oIndex !== -1 && !this.manuallySetMouth) {
             morphTargetInfluences[oIndex] = volumeValue * this.vowels[3];
           }
-          if (uIndex !== -1) {
+          if (uIndex !== -1 && !this.manuallySetMouth) {
             morphTargetInfluences[uIndex] = volumeValue * this.vowels[4];
           }
           /* } else { // fake speech
@@ -1883,7 +1885,7 @@ class Avatar {
       }
       this.debugMesh.visible = debug.enabled;
     }
-	}
+  }
 
   isAudioEnabled() {
     return !!this.microphoneWorker;
@@ -1916,7 +1918,9 @@ class Avatar {
         emitBuffer: true,
       });
       this.microphoneWorker.addEventListener('volume', e => {
-        this.volume = this.volume*0.8 + e.data*0.2;
+        if(!this.manuallySetMouth){
+          this.volume = this.volume*0.8 + e.data*0.2;
+        }
       });
       this.microphoneWorker.addEventListener('buffer', e => {
         this.audioRecognizer.send(e.data);

character-behavior.js

@@ -0,0 +1,84 @@
+class CharacterBehavior {
+    constructor(player) {
+        this.player = player;
+        this.mouthMovementState=null;
+        this.mouthMovementStartTime=0;
+        this.mouthMovementAttackTime=0;
+        this.mouthMovementDecayTime=0;
+        this.mouthMovementSustainTime=0;
+        this.mouthMovementReleaseTime=0;
+    }
+
+    update(timestamp, timeDiffS) {
+        if (!this.player.avatar) {
+            return;
+        }
+        //#################################### manually set mouth movement ##########################################
+        const _handleMouthMovementAttack=()=>{
+            this.player.avatar.volume = ((timestamp/1000 - this.mouthMovementStartTime) / this.mouthMovementAttackTime)/12;
+            if(timestamp/1000 - this.mouthMovementStartTime >= this.mouthMovementAttackTime){
+                this.mouthMovementState = 'decay';
+                this.mouthMovementStartTime = timestamp/1000;
+            }
+        }
+        const _handleMouthMovementDecay=()=>{
+            this.player.avatar.volume = (1 - ((timestamp/1000 - this.mouthMovementStartTime) / this.mouthMovementDecayTime) * 0.8)/12;
+            if(timestamp/1000 - this.mouthMovementStartTime >= this.mouthMovementDecayTime){
+                this.mouthMovementState='sustain';
+                this.mouthMovementStartTime = timestamp/1000;
+            }
+        }
+        const _handleMouthMovementSustain=()=>{
+            if(timestamp/1000 - this.mouthMovementStartTime >= this.mouthMovementSustainTime){
+                this.mouthMovementState='release';
+                this.mouthMovementStartTime = timestamp/1000;
+            } 
+        }
+        const _handleMouthMovementRelease=()=>{
+            this.player.avatar.volume = (0.2 - ((timestamp/1000 - this.mouthMovementStartTime) / this.mouthMovementReleaseTime) * 0.2)/12;
+            if(timestamp/1000 - this.mouthMovementStartTime >= this.mouthMovementReleaseTime){
+                this.mouthMovementState=null;
+                this.manuallySetMouth=false;
+                this.mouthMovementStartTime = -1;
+            }
+        }
+        const _handleMouthMovementNull=()=>{
+            this.mouthMovementState = this.manuallySetMouth ? 'attack' : null;
+            this.mouthMovementStartTime = timestamp/1000;
+        }
+        switch (this.mouthMovementState) {
+            case 'attack': {
+                _handleMouthMovementAttack();
+                break;
+            }
+            case 'decay': {
+                _handleMouthMovementDecay();
+                break;
+            }
+            case 'sustain': {
+                _handleMouthMovementSustain();
+                break;
+            }
+            case 'release': {
+                _handleMouthMovementRelease();
+                break;
+            }
+            case null: {
+                _handleMouthMovementNull();
+                break;
+            }
+        }
+    }
+    setMouthMoving(attack, decay, sustain, release){
+        this.mouthMovementState=null;
+        this.manuallySetMouth=true;
+        this.mouthMovementAttackTime=attack;
+        this.mouthMovementDecayTime=decay;
+        this.mouthMovementSustainTime=sustain;
+        this.mouthMovementReleaseTime=release;
+    }
+  }
+
+  export {
+    CharacterBehavior,
+  };

character-controller.js

@@ -30,6 +30,7 @@ import {AppManager} from './app-manager.js';
 import {CharacterPhysics} from './character-physics.js';
 import {CharacterHups} from './character-hups.js';
 import {CharacterSfx} from './character-sfx.js';
+import {CharacterBehavior} from './character-behavior.js';
 import {CharacterFx} from './character-fx.js';
 import {VoicePack, VoicePackVoicer} from './voice-output/voice-pack-voicer.js';
 import {VoiceEndpoint, VoiceEndpointVoicer} from './voice-output/voice-endpoint-voicer.js';
@@ -914,6 +915,8 @@ class LocalPlayer extends UninterpolatedPlayer {
     this.characterHups = new CharacterHups(this);
     this.characterSfx = new CharacterSfx(this);
     this.characterFx = new CharacterFx(this);
+    this.characterBehavior = new CharacterBehavior(this);
+
   }
   async setAvatarUrl(u) {
     const localAvatarEpoch = ++this.avatarEpoch;
@@ -1083,6 +1086,7 @@ class LocalPlayer extends UninterpolatedPlayer {
       const timeDiffS = timeDiff / 1000;
       this.characterSfx.update(timestamp, timeDiffS);
       this.characterFx.update(timestamp, timeDiffS);
+      this.characterBehavior.update(timestamp, timeDiffS);
 
       this.updateInterpolation(timeDiff);
 

character-sfx.js

@@ -15,6 +15,7 @@ import {
 } from './constants.js';
 import {
   mod,
+  selectVoice,
   // loadJson,
   // loadAudioBuffer,
 } from './util.js';
@@ -73,6 +74,10 @@ class CharacterSfx {
     this.preQ=new THREE.Quaternion();
     this.arr = [0, 0, 0, 0];
 
+    this.runStep = 0;
+    this.lastRunningTime = 0;
+
+    this.willGasp = false;
 
 
     this.oldNarutoRunSound = null;
@@ -97,6 +102,11 @@ class CharacterSfx {
       if (this.player.avatar.jumpState && !this.lastJumpState) {
         const audioSpec = soundFiles.jump[Math.floor(Math.random() * soundFiles.jump.length)];
         sounds.playSound(audioSpec);
+
+        // play jump grunt 
+        if(this.player.hasAction('jump') && this.player.getAction('jump').trigger=='jump'){
+          this.playGrunt('jump'); 
+        }
       } else if (this.lastJumpState && !this.player.avatar.jumpState) {
         const audioSpec = soundFiles.land[Math.floor(Math.random() * soundFiles.land.length)];
         sounds.playSound(audioSpec);
@@ -166,6 +176,11 @@ class CharacterSfx {
 
                 const audioSpec = candidateAudios[Math.floor(Math.random() * candidateAudios.length)];
                 sounds.playSound(audioSpec);
+                // set runStep
+                if(isRunning){
+                  this.lastRunningTime = timeSeconds;
+                  this.runStep++;
+                }
               }
             }
             this.lastStepped[0] = leftStepIndices[i];
@@ -179,6 +194,11 @@ class CharacterSfx {
 
                 const audioSpec = candidateAudios[Math.floor(Math.random() * candidateAudios.length)];
                 sounds.playSound(audioSpec);
+                // set runStep
+                if(isRunning){
+                  this.lastRunningTime = timeSeconds;
+                  this.runStep++;
+                }
               }
             }
             this.lastStepped[1] = rightStepIndices[i];
@@ -189,6 +209,7 @@ class CharacterSfx {
 
         this.lastWalkTime = timeSeconds;
       }
+
     };
     _handleStep();
 
@@ -212,6 +233,7 @@ class CharacterSfx {
         if(this.narutoRunStartTime===0){
           this.narutoRunStartTime=timeSeconds; 
           sounds.playSound(soundFiles.sonicBoom[0]);
+          this.playGrunt('narutoRun');
         }
         else {
           if(this.arr.reduce((a,b)=>a+b) >= Math.PI/3){
@@ -225,21 +247,24 @@ class CharacterSfx {
           }
 
           if(timeSeconds - this.narutoRunTrailSoundStartTime>soundFiles.sonicBoom[2].duration-0.2 || this.narutoRunTrailSoundStartTime==0){
-            if(!this.player.getAction('sit')){
-              const localSound = sounds.playSound(soundFiles.sonicBoom[2]);
-              this.oldNarutoRunSound = localSound;
-              localSound.addEventListener('ended', () => {
-                if (this.oldNarutoRunSound === localSound) {
-                  this.oldNarutoRunSound = null;
-                }
-              });
-
-              this.narutoRunTrailSoundStartTime = timeSeconds;
-            }
 
+            const localSound = sounds.playSound(soundFiles.sonicBoom[2]);
+            this.oldNarutoRunSound = localSound;
+            localSound.addEventListener('ended', () => {
+              if (this.oldNarutoRunSound === localSound) {
+                this.oldNarutoRunSound = null;
+              }
+            });
+
+            this.narutoRunTrailSoundStartTime = timeSeconds;
           }
         }
 
+        // if naruto run play more than 2 sec, set willGasp
+        if(timeSeconds -  this.narutoRunStartTime > 2){
+          this.willGasp = true;
+        }
+
       }
       if(!this.player.avatar.narutoRunState && this.narutoRunStartTime!=0 ){
         this.narutoRunStartTime=0;
@@ -259,6 +284,20 @@ class CharacterSfx {
 
     };
     _handleNarutoRun();
+
+
+    const _handleGasp = () =>{
+      if(timeSeconds - this.lastRunningTime > 2 && !this.player.avatar.narutoRunState && timeSeconds - this.narutoRunFinishTime > 0.5){
+        this.runStep = 0;
+      }
+      // if running step is more than 10
+      if(currentSpeed < 0.1 && timeSeconds - this.lastWalkTime > 0.01 && (this.runStep > 10 || this.willGasp) && !this.player.avatar.narutoRunState){
+        this.playGrunt('gasp');
+        this.runStep = 0;
+        this.willGasp = false;
+      }
+    }
+    _handleGasp();
 
     const _handleFood = () => {
       const useAction = this.player.getAction('use');
@@ -271,6 +310,8 @@ class CharacterSfx {
           if (eatFrameIndex !== 0 && eatFrameIndex !== this.lastEatFrameIndex) {
             const audioSpec = soundFiles.chomp[Math.floor(Math.random() * soundFiles.chomp.length)];
             sounds.playSound(audioSpec);
+            // control mouth movement
+            this.player.characterBehavior.setMouthMoving(0.04,0.04,0.1,0.02);
           }
 
           this.lastEatFrameIndex = eatFrameIndex;
@@ -285,6 +326,8 @@ class CharacterSfx {
           if (drinkFrameIndex !== 0 && drinkFrameIndex !== this.lastDrinkFrameIndex) {
             const audioSpec = soundFiles.gulp[Math.floor(Math.random() * soundFiles.gulp.length)];
             sounds.playSound(audioSpec);
+            // control mouth movement
+            this.player.characterBehavior.setMouthMoving(0.1,0.1,0.1,0.1);
           }
 
           this.lastDrinkFrameIndex = drinkFrameIndex;
@@ -308,6 +351,76 @@ class CharacterSfx {
     };
     _handleFood();
   }
+  playGrunt(type, index){
+
+    let voiceFiles, offset, duration;
+    switch (type) {
+      case 'pain': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /pain/i.test(f.name));
+        break;
+      }
+      case 'scream': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /scream/i.test(f.name));
+        break;
+      }
+      case 'attack': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /attack/i.test(f.name));
+        break;
+      }
+      case 'angry': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /angry/i.test(f.name));
+        break;
+      }
+      case 'gasp': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /gasp/i.test(f.name));
+        break;
+      }
+      case 'jump': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /jump/i.test(f.name));
+        break;
+      }
+      case 'narutoRun': {
+        voiceFiles = this.player.voicePack.actionVoices.filter(f => /nr/i.test(f.name));
+        break;
+      }
+    }
+
+    if(index===undefined){
+      let voice = selectVoice(voiceFiles);
+      duration = voice.duration;
+      offset = voice.offset;
+    }
+    else{
+      duration = voiceFiles[index].duration;
+      offset = voiceFiles[index].offset;
+    } 
+
+    const audioContext = Avatar.getAudioContext();
+    const audioBufferSourceNode = audioContext.createBufferSource();
+    audioBufferSourceNode.buffer = this.player.voicePack.audioBuffer;
+
+    // control mouth movement with audio volume
+    if (!this.player.avatar.isAudioEnabled()) {
+      this.player.avatar.setAudioEnabled(true);
+    }
+    audioBufferSourceNode.connect(this.player.avatar.getAudioInput());
+
+    // if the oldGrunt are still playing
+    if(this.oldGrunt){
+      this.oldGrunt.stop();
+      this.oldGrunt = null;
+    }
+
+    this.oldGrunt=audioBufferSourceNode;
+    // clean the oldGrunt if voice end
+    audioBufferSourceNode.addEventListener('ended', () => {
+      if (this.oldGrunt === audioBufferSourceNode) {
+        this.oldGrunt = null;
+      }
+    });
+
+    audioBufferSourceNode.start(0, offset, duration);
+  }
   destroy() {
     // nothing
   }