Initial Commit

.gitignore

@@ -30,3 +30,7 @@
 *.exe
 *.out
 *.app
+
+# Builds
+Builds
+JuceLibraryCode

README.md

@@ -1,2 +1,22 @@
 # RiffusionVST
- A VST plugin for Riffusion
+ A VST plugin for Riffusion based on JUCE
+
+## Notes on Use
+This is an experimental VST plugin for interfacing with Riffusion (https://github.com/riffusion/riffusion). You must be able to run Riffusion locally using mklingen's special branch (https://github.com/mklingen/riffusion-inference/). Then, this will generate a VST3 plugin for you, and a standalone app.
+
+As of writing, this DOES NOT WORK with standard riffusion servers, you must be using mklingen's special branch.
+
+![screenshot in trackiton waveform](screenshot.png)
+> The above is a screenshot of the plugin running in trackiton waveform.
+
+## Install
+1. Download mklingen's special branch, and run the vst server after the lengthy install steps, getting torch setup, conda, etc. https://github.com/mklingen/riffusion-inference
+2. Download and install [juce](https://juce.com/get-juce/download), including Projucer.
+3. Build this plugin in visual studio after using projucer.
+4. A plugin will be generated in the `Builds` folder.
+5. Add the plugin to your favorite DAW.
+6. Run the Riffusion server locally (or, if you have some powerful build machine somewhere, run it there).
+7. Point the plugin at the IP address of your riffusion server with port 3000 (if running locally, you won't have to do anything).
+
+## Use
+Once everything is running, you can first press "RECORD" to record up to 5 seconds of audio into the VSTs input. You can then hit "generate" to generate audio based on the recording, then "play" to loop the generated audio. To record it, pipe the output of the VST to a send line in your DAW.

RiffusionVST.jucer

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<JUCERPROJECT id="cwipx0" name="RiffusionVST" projectType="audioplug" useAppConfig="0"
+              addUsingNamespaceToJuceHeader="0" displaySplashScreen="1" jucerFormatVersion="1">
+  <MAINGROUP id="KRu5jy" name="RiffusionVST">
+    <GROUP id="{F37BF7D7-566A-BDFC-3FED-B5E3242E74E6}" name="Source">
+      <FILE id="gN5x0k" name="PluginProcessor.cpp" compile="1" resource="0"
+            file="Source/PluginProcessor.cpp"/>
+      <FILE id="GuqqDu" name="PluginProcessor.h" compile="0" resource="0"
+            file="Source/PluginProcessor.h"/>
+      <FILE id="xEBWlW" name="PluginEditor.cpp" compile="1" resource="0"
+            file="Source/PluginEditor.cpp"/>
+      <FILE id="veG1SK" name="PluginEditor.h" compile="0" resource="0" file="Source/PluginEditor.h"/>
+    </GROUP>
+  </MAINGROUP>
+  <JUCEOPTIONS JUCE_STRICT_REFCOUNTEDPOINTER="1" JUCE_VST3_CAN_REPLACE_VST2="0"/>
+  <EXPORTFORMATS>
+    <VS2022 targetFolder="Builds/VisualStudio2022">
+      <CONFIGURATIONS>
+        <CONFIGURATION isDebug="1" name="Debug" targetName="RiffusionVST"/>
+        <CONFIGURATION isDebug="0" name="Release" targetName="RiffusionVST"/>
+      </CONFIGURATIONS>
+      <MODULEPATHS>
+        <MODULEPATH id="juce_audio_basics" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_audio_devices" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_audio_formats" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_audio_plugin_client" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_audio_processors" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_audio_utils" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_core" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_data_structures" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_events" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_graphics" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_gui_basics" path="../../../Desktop/JUCE/modules"/>
+        <MODULEPATH id="juce_gui_extra" path="../../../Desktop/JUCE/modules"/>
+      </MODULEPATHS>
+    </VS2022>
+  </EXPORTFORMATS>
+  <MODULES>
+    <MODULE id="juce_audio_basics" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_audio_devices" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_audio_formats" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_audio_plugin_client" showAllCode="1" useLocalCopy="0"
+            useGlobalPath="0"/>
+    <MODULE id="juce_audio_processors" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_audio_utils" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_core" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_data_structures" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_events" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_graphics" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_gui_basics" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+    <MODULE id="juce_gui_extra" showAllCode="1" useLocalCopy="0" useGlobalPath="0"/>
+  </MODULES>
+</JUCERPROJECT>

Source/PluginEditor.cpp

@@ -0,0 +1,194 @@
+/*
+  ==============================================================================
+
+    This file contains the basic framework code for a JUCE plugin editor.
+
+  ==============================================================================
+*/
+
+#include "PluginProcessor.h"
+#include "PluginEditor.h"
+
+//==============================================================================
+RiffusionVSTAudioProcessorEditor::RiffusionVSTAudioProcessorEditor (RiffusionVSTAudioProcessor& p)
+    : AudioProcessorEditor (&p), audioProcessor (p)
+{
+    // Make sure that before the constructor has finished, you've set the
+    // editor's size to whatever you need it to be.
+    setSize (400, 500);
+    serverIp.setText("http://127.0.0.1:3000");
+    prompt1Text.setText("prompt 1");
+    prompt2Text.setText("prompt 2");
+    generateButton.setButtonText("Generate");
+    generateButton.onClick = [this]() {
+        onGenerateClicked();
+    };
+
+    recordButton.setButtonText("Record");
+    recordButton.onClick = [this]() {
+        onRecordClicked();
+    };
+
+    playbackButton.setButtonText("Play");
+    playbackButton.onClick = [this]() {
+        onPlayClicked();
+    };
+
+    alphaSlider.setTextValueSuffix(" Blend");
+    alphaSlider.setValue(0.5);
+    alphaSlider.setRange(0.0, 1.0, 0.1);
+    strengthSlider.setTextValueSuffix(" Prompt Strength");
+    strengthSlider.setValue(7.0);
+    strengthSlider.setRange(0.0, 25.0, 0.1);
+    denoisingSlider.setTextValueSuffix(" Denoising");
+    denoisingSlider.setValue(0.7);
+    denoisingSlider.setRange(0.0, 1.0, 0.05);
+    seedText.setText("seed");
+    itersSlider.setTextValueSuffix(" Iters");
+    itersSlider.setRange(1, 100, 1.0);
+    itersSlider.setValue(50);
+    messageText.setText("");
+    messageText.setColour(juce::Colour(255, 255, 255));
+    messageText.setJustification(juce::Justification::centred);
+
+    addAndMakeVisible(&serverIp);
+    addAndMakeVisible(&prompt1Text);
+    addAndMakeVisible(&prompt2Text);
+    addAndMakeVisible(&alphaSlider);
+    addAndMakeVisible(&strengthSlider);
+    addAndMakeVisible(&denoisingSlider);
+    addAndMakeVisible(&itersSlider);
+    addAndMakeVisible(&seedText);
+    addAndMakeVisible(&recordButton);
+    addAndMakeVisible(&generateButton);
+    addAndMakeVisible(&playbackButton);
+    addAndMakeVisible(&messageText);
+}
+
+void RiffusionVSTAudioProcessorEditor::onGenerateClicked() {
+    if (state != RecordingState::Generating) {
+        state = RecordingState::Generating;
+        recordButton.setEnabled(false);
+        playbackButton.setEnabled(false);
+        generateButton.setButtonText("Stop");
+        RiffusionVSTAudioProcessor::ProcessParams params;
+        params.alpha = alphaSlider.getValue();
+        params.denoising = denoisingSlider.getValue();
+        params.guidance = strengthSlider.getValue();
+        params.numInferenceSteps = static_cast<int>(itersSlider.getValue());
+        params.promptA = prompt1Text.getText().toStdString();
+        params.promptB = prompt2Text.getText().toStdString();
+        params.serverAddress = serverIp.getText().toStdString();
+        params.seed = static_cast<int>(std::hash<std::string>()(seedText.getText().toStdString()));
+        audioProcessor.startGenerating(params);
+    }
+    else {
+        state = RecordingState::Idle;
+        recordButton.setEnabled(true);
+        playbackButton.setEnabled(true);
+        generateButton.setButtonText("Generate");
+    }
+}
+
+void RiffusionVSTAudioProcessorEditor::onRecordClicked() {
+    if (state == RecordingState::Recording) {
+        state = RecordingState::Idle;
+        recordButton.setButtonText("Record");
+        playbackButton.setEnabled(true);
+        generateButton.setEnabled(true);
+        audioProcessor.stopRecording();
+    } else {
+        state = RecordingState::Recording;
+        recordButton.setButtonText("Stop");
+        playbackButton.setEnabled(false);
+        generateButton.setEnabled(false);
+        audioProcessor.startRecording();
+    }
+}
+
+void RiffusionVSTAudioProcessorEditor::onPlayClicked() {
+    if (state != RecordingState::Playing) {
+        state = RecordingState::Playing;
+        recordButton.setEnabled(false);
+        generateButton.setEnabled(false);
+        playbackButton.setButtonText("Stop");
+        audioProcessor.startPlaying();
+    }
+    else {
+        state = RecordingState::Idle;
+        recordButton.setEnabled(true);
+        generateButton.setEnabled(true);
+        playbackButton.setButtonText("Play");
+        audioProcessor.stopPlaying();
+    }
+}
+
+RiffusionVSTAudioProcessorEditor::~RiffusionVSTAudioProcessorEditor()
+{
+}
+
+//==============================================================================
+void RiffusionVSTAudioProcessorEditor::paint (juce::Graphics& g)
+{
+    // (Our component is opaque, so we must completely fill the background with a solid colour)
+    g.fillAll (getLookAndFeel().findColour (juce::ResizableWindow::backgroundColourId));
+    g.setColour(juce::Colours::white);
+    g.setFont(19.0f);
+    g.drawText("Riffusion VST", 0, 0, 300, 250, 30, juce::Justification::left);
+    g.setFont(15.0f);
+    g.drawFittedText("Server IP: ", serverIp.getPosition().x - 120, serverIp.getPosition().y, 100, 30, juce::Justification::right, 1);
+    messageText.setText(audioProcessor.message);
+
+    if (!audioProcessor.getIsRecording() && state == RecordingState::Recording) {
+        state = RecordingState::Idle;
+        recordButton.setButtonText("Record");
+        playbackButton.setEnabled(true);
+        generateButton.setEnabled(true);
+    }
+    else if (!audioProcessor.getIsPlaying() && state == RecordingState::Playing) {
+        state = RecordingState::Idle;
+        recordButton.setEnabled(true);
+        generateButton.setEnabled(true);
+        playbackButton.setButtonText("Play");
+    }
+    else if (!audioProcessor.getIsGenerating() && state == RecordingState::Generating) {
+        state = RecordingState::Idle;
+        recordButton.setEnabled(true);
+        playbackButton.setEnabled(true);
+        generateButton.setEnabled(true);
+        generateButton.setButtonText("Generate");
+    }
+}
+
+void RiffusionVSTAudioProcessorEditor::resized()
+{
+    constexpr int xPadding = 15;
+    constexpr int yPadding = 15;
+    constexpr int elementHeight = 30;
+    constexpr  int minWidth = 125;
+    const int w = getWidth();
+    const int h = getHeight();
+    const int r = w - xPadding / 2;
+    constexpr  int l = xPadding / 2;
+    constexpr  int row_padding = 4;
+    auto row = [=](int r_idx) { return yPadding + r_idx * (elementHeight + row_padding); };
+    int curr_row_idx = 0;
+    auto next_row = [&curr_row_idx, &row]() {
+        int rowPx = row(curr_row_idx);
+        curr_row_idx++; 
+        return rowPx;
+    };
+    // sets the position and size of the slider with arguments (x, y, width, height)
+    serverIp.setBounds(w - xPadding - minWidth, next_row(), minWidth, elementHeight);
+    seedText.setBounds(l, next_row(), r, elementHeight);
+    prompt1Text.setBounds(l, next_row(), r, elementHeight);
+    prompt2Text.setBounds(l, next_row(), r, elementHeight);
+    alphaSlider.setBounds(l, next_row(), r, elementHeight);
+    strengthSlider.setBounds(l, next_row(), r, elementHeight);
+    denoisingSlider.setBounds(l, next_row(), r, elementHeight);
+    itersSlider.setBounds(l, next_row(), r, elementHeight);
+    recordButton.setBounds(l, next_row(), r, elementHeight);
+    generateButton.setBounds(l, next_row(), r, elementHeight);
+    playbackButton.setBounds(l, next_row(), r, elementHeight);
+    messageText.setBoundingBox(juce::Parallelogram(juce::Rectangle<float>(l, next_row(), r, elementHeight)));
+}

Source/PluginEditor.h

@@ -0,0 +1,60 @@
+/*
+  ==============================================================================
+
+    This file contains the basic framework code for a JUCE plugin editor.
+
+  ==============================================================================
+*/
+
+#pragma once
+
+#include <JuceHeader.h>
+#include "PluginProcessor.h"
+
+//==============================================================================
+/**
+*/
+class RiffusionVSTAudioProcessorEditor  : public juce::AudioProcessorEditor
+{
+public:
+    RiffusionVSTAudioProcessorEditor (RiffusionVSTAudioProcessor&);
+    ~RiffusionVSTAudioProcessorEditor() override;
+
+    //==============================================================================
+    void paint (juce::Graphics&) override;
+    void resized() override;
+
+private:
+    // This reference is provided as a quick way for your editor to
+    // access the processor object that created it.
+    RiffusionVSTAudioProcessor& audioProcessor;
+
+    enum class RecordingState
+    {
+        Idle,
+        Recording,
+        Generating,
+        Playing
+    };
+
+    void onGenerateClicked();
+    void onRecordClicked();
+    void onPlayClicked();
+
+    juce::TextEditor serverIp;
+    juce::TextEditor prompt1Text;
+    juce::TextEditor prompt2Text;
+    juce::TextEditor seedText;
+    juce::Slider alphaSlider;
+    juce::Slider strengthSlider;
+    juce::Slider denoisingSlider;
+    juce::Slider itersSlider;
+    juce::TextButton recordButton;
+    juce::TextButton generateButton;
+    juce::TextButton playbackButton;
+    juce::DrawableText messageText;
+    RecordingState state = RecordingState::Idle;
+
+
+    JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (RiffusionVSTAudioProcessorEditor)
+};