New code

config/kss.yaml

@@ -5,7 +5,7 @@ model:
   gmm: 10
 ---
 data:
-  path: 'KSS'
+  path: ''
   extension: '*.wav'
 ---
 audio:

inference.py

@@ -12,7 +12,6 @@
 from utils.hparams import HParam
 from model.model import MelNet
 
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('-c', '--config', type=str, required=True,
@@ -45,9 +44,13 @@
     spectrogram = plot_spectrogram_to_numpy(generated[0].cpu().detach().numpy())
     plt.imsave(os.path.join('temp', args.name + '.png'), spectrogram.transpose((1, 2, 0)))
 
-    waveform = Reconstruct(hp).inverse(generated[0]).unsqueeze(-1)
+    waveform, wavespec = Reconstruct(hp).inverse(generated[0])
+    wavespec = plot_spectrogram_to_numpy(wavespec.cpu().detach().numpy())
+    plt.imsave(os.path.join('temp', 'Final ' + args.name + '.png'), wavespec.transpose((1, 2, 0)))
+
+    waveform = waveform.unsqueeze(-1)
     waveform = waveform.cpu().detach().numpy()
-    waveform *= 32768
+    waveform *= 32768 / waveform.max()
     waveform = waveform.astype(np.int16)
     audio = audiosegment.from_numpy_array(
         waveform,

model/model.py

@@ -54,14 +54,18 @@ def forward(self, x, tier_num):
         return self.tiers[tier_num](x)
 
     def sample(self, condition):
-        x = torch.zeros(1, self.n_mels // self.f_div, self.args.timestep // self.t_div).cuda()
+        x = None
         seq = torch.from_numpy(text_to_sequence(condition)).long().unsqueeze(0)
         input_lengths = torch.LongTensor([seq[0].shape[0]]).cuda()
 
         ## Tier 1 ##
         tqdm.write('Tier 1')
-        for t in tqdm(range(x.size(2))):
-            for m in tqdm(range(x.size(1))):
+        for t in tqdm(range(self.args.timestep // self.t_div)):
+            if x is None:
+                x = torch.zeros((1, self.n_mels // self.f_div, 1)).cuda()
+            else:
+                x = torch.cat([x, torch.zeros((1, self.n_mels // self.f_div, 1)).cuda()], dim=-1)
+            for m in tqdm(range(self.n_mels // self.f_div)):
                 torch.cuda.synchronize()
                 if self.infer_hp.conditional:
                     mu, std, pi, _ = self.tiers[1](x, seq, input_lengths)

utils/reconstruct.py

@@ -9,9 +9,11 @@ def __init__(self, hp):
         self.hp = hp
         self.window = torch.hann_window(window_length=hp.audio.win_length).cuda()
         self.mel_basis = librosa.filters.mel(
-            sr=hp.audio.sr, n_fft=hp.audio.n_fft, n_mels=hp.audio.n_mels)
-        self.mel_basis = \
-            torch.from_numpy(self.mel_basis).float().cuda() # [n_mels, n_fft//2+1]
+            sr=hp.audio.sr,
+            n_fft=hp.audio.n_fft,
+            n_mels=hp.audio.n_mels
+        )
+        self.mel_basis = torch.from_numpy(self.mel_basis).cuda() # [n_mels, n_fft//2+1]
         self.criterion = torch.nn.MSELoss()
 
     def get_mel(self, x):
@@ -30,6 +32,11 @@ def post_spec(self, x):
         x = (x - 1) * -self.hp.audio.min_level_db + self.hp.audio.ref_level_db
         x = torch.pow(10, x / 10)
         return x
+
+    def pre_spec(self, x):
+        x = torch.log10(x) * 10
+        x = (x - self.hp.audio.ref_level_db) / -self.hp.audio.min_level_db + 1
+        return x
 
     def inverse(self, melspectrogram, iters=1000):
         x = torch.normal(0, 1e-6, size=((melspectrogram.size(1) - 1) * self.hp.audio.hop_length, )).cuda().requires_grad_()
@@ -48,4 +55,4 @@ def closure():
                 optimizer.step(closure=closure)
                 pbar.set_postfix(loss=self.criterion(self.get_mel(x), melspectrogram).item())
 
-        return x
+        return x, self.pre_spec(self.get_mel(x))

utils/utils.py

@@ -10,11 +10,10 @@ def get_commit_hash():
 def read_wav_np(wavpath, sample_rate):
     file_format = wavpath.split('.')[-1]
     audio = audiosegment.from_file(wavpath).resample(sample_rate_Hz=sample_rate)
-    data = audio.raw_data
-    wav = np.frombuffer(data, dtype=np.int16)
+    wav = audio.to_numpy_array()
 
     if len(wav.shape) == 2:
-        wav = wav[:, 0]
+        wav = wav.T.flatten()
 
     if wav.dtype == np.int16:
         wav = wav / 32768.0