Merge pull request #144 from fieldsJacksonG/master

Update spectator view to the latest HoloToolkit (1.5.7)
microsoft · Jun 3, 2017 · 0e08a97 · 0e08a97
2 parents 91e7423 + 5c9e832
commit 0e08a97
Show file tree

Hide file tree

Showing 473 changed files with 60,455 additions and 55,305 deletions.
diff --git a/SpectatorView/Calibration/Calibration/CalibrationApp.cpp b/SpectatorView/Calibration/Calibration/CalibrationApp.cpp
@@ -100,7 +100,7 @@ void CalibrationApp::Initialize(HWND window, int width, int height)
     frameProvider = new DeckLinkManager(true, true);
 #endif
 #if USE_OPENCV
-    frameProvider = new OpenCVFrameProvider();
+    frameProvider = new OpenCVFrameProvider(false);
 #endif
 
     frameProvider->Initialize(srv, nullptr);

diff --git a/SpectatorView/Compositor/CompositorDLL/CompositorInterface.cpp b/SpectatorView/Compositor/CompositorDLL/CompositorInterface.cpp
@@ -50,7 +50,7 @@ bool CompositorInterface::Initialize(ID3D11Device* device, ID3D11ShaderResourceV
 
     _device = device;
 
-    hologramQueue = new HologramQueue(device);
+    hologramQueue = new HologramQueue();
 
     return SUCCEEDED(frameProvider->Initialize(colorSRV, outputTexture));
 }
@@ -124,7 +124,7 @@ void CompositorInterface::StopFrameProvider()
 #endif
 }
 
-LONGLONG CompositorInterface::GetColorTime()
+LONGLONG CompositorInterface::GetTimestamp()
 {
     if (frameProvider != nullptr)
     {

diff --git a/SpectatorView/Compositor/CompositorDLL/CompositorInterface.h b/SpectatorView/Compositor/CompositorDLL/CompositorInterface.h
@@ -57,7 +57,8 @@ class CompositorInterface
     DLLEXPORT void Update();
     DLLEXPORT void StopFrameProvider();
 
-    DLLEXPORT LONGLONG GetColorTime();
+    DLLEXPORT LONGLONG GetTimestamp();
+
     DLLEXPORT LONGLONG GetColorDuration();
 
     DLLEXPORT void TakePicture(ID3D11Device* device, int width, int height, int bpp, 

diff --git a/SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.cpp b/SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.cpp
@@ -79,9 +79,10 @@ DeckLinkDevice::~DeckLinkDevice()
     DeleteCriticalSection(&m_outputCriticalSection);
     DeleteCriticalSection(&m_frameAccessCriticalSection);
 
-    delete[] cachedBuffer;
-    delete[] stagingBuffer;
+    delete[] thirdCachedBuffer;
+    delete[] secondCachedBuffer;
     delete[] latestBuffer;
+    delete[] stagingBuffer;
     delete[] outputBuffer;
     delete[] outputBufferRaw;
 }
@@ -148,7 +149,8 @@ bool DeckLinkDevice::Init(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* o
     BSTR                            deviceNameBSTR = NULL;
 
     ZeroMemory(rawBuffer, FRAME_BUFSIZE_RAW);
-    ZeroMemory(cachedBuffer, FRAME_BUFSIZE);
+    ZeroMemory(thirdCachedBuffer, FRAME_BUFSIZE);
+    ZeroMemory(secondCachedBuffer, FRAME_BUFSIZE);
     ZeroMemory(latestBuffer, FRAME_BUFSIZE);
     ZeroMemory(outputBuffer, FRAME_BUFSIZE);
     ZeroMemory(outputBufferRaw, FRAME_BUFSIZE_RAW);
@@ -372,13 +374,16 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
     {
         if (frame->GetBytes((void**)&rawBuffer) == S_OK)
         {
+            // Always return the latest buffer when using the CPU.
             if (_useCPU)
             {
-                DirectXHelper::ConvertYUVtoBGRA(rawBuffer, cachedBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
+                DirectXHelper::ConvertYUVtoBGRA(rawBuffer, latestBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
             }
             else
             {
-                memcpy(cachedBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE_RAW);
+                memcpy(latestBuffer, rawBuffer, FRAME_BUFSIZE_RAW);
             }
         }
     }
@@ -391,11 +396,14 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
                 //TODO: Remove this block if R and B components are swapped in color feed.
                 memcpy(stagingBuffer, localFrameBuffer, FRAME_BUFSIZE);
                 DirectXHelper::ConvertBGRAtoRGBA(stagingBuffer, FRAME_WIDTH, FRAME_HEIGHT, true);
-                memcpy(cachedBuffer, stagingBuffer, FRAME_BUFSIZE);
+                // Do not cache frames when using the CPU
+                memcpy(latestBuffer, stagingBuffer, FRAME_BUFSIZE);
             }
             else
             {
-                memcpy(cachedBuffer, localFrameBuffer, FRAME_BUFSIZE);
+                memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
+                memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
+                memcpy(latestBuffer, localFrameBuffer, FRAME_BUFSIZE);
             }
         }
     }
@@ -404,7 +412,9 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
     frame->GetStreamTime(&t, &frameDuration, QPC_MULTIPLIER);
 
     // Get frame time.
-    cachedTimeStamp = time.QuadPart;
+    thirdTimeStamp = secondTimeStamp;
+    secondTimeStamp = latestTimeStamp;
+    latestTimeStamp = time.QuadPart;
 
     dirtyFrame = false;
 
@@ -445,13 +455,20 @@ HRESULT DeckLinkDevice::VideoInputFrameArrived(/* in */ IDeckLinkVideoInputFrame
 void DeckLinkDevice::Update()
 {
     if (_colorSRV != nullptr &&
-        cachedBuffer != nullptr &&
         device != nullptr)
     {
         if (!dirtyFrame)
         {
             dirtyFrame = true;
-            DirectXHelper::UpdateSRV(device, _colorSRV, cachedBuffer, FRAME_WIDTH * FRAME_BPP);
+            if (_useCPU && latestBuffer != nullptr)
+            {
+                // Do not cache when using CPU.
+                DirectXHelper::UpdateSRV(device, _colorSRV, latestBuffer, FRAME_WIDTH * FRAME_BPP);
+            }
+            else if (!_useCPU && thirdCachedBuffer != nullptr)
+            {
+                DirectXHelper::UpdateSRV(device, _colorSRV, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
+            }
 
             EnterCriticalSection(&m_frameAccessCriticalSection);
             isVideoFrameReady = true;

diff --git a/SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.h b/SpectatorView/Compositor/CompositorDLL/DeckLinkDevice.h
@@ -57,18 +57,20 @@ class DeckLinkDevice : public IDeckLinkInputCallback
     CRITICAL_SECTION          m_outputCriticalSection;
 
     BYTE* localFrameBuffer;
-    BYTE* rawBuffer =         new BYTE[FRAME_BUFSIZE_RAW];
+    BYTE* rawBuffer =           new BYTE[FRAME_BUFSIZE_RAW];
 
-    BYTE* cachedBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* stagingBuffer =     new BYTE[FRAME_BUFSIZE];
-    BYTE* latestBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* outputBuffer =      new BYTE[FRAME_BUFSIZE];
-    BYTE* outputBufferRaw =   new BYTE[FRAME_BUFSIZE_RAW];
+    BYTE* thirdCachedBuffer =   new BYTE[FRAME_BUFSIZE];
+    BYTE* secondCachedBuffer =  new BYTE[FRAME_BUFSIZE];
+    BYTE* latestBuffer =        new BYTE[FRAME_BUFSIZE];
+    BYTE* stagingBuffer =       new BYTE[FRAME_BUFSIZE];
+    BYTE* outputBuffer =        new BYTE[FRAME_BUFSIZE];
+    BYTE* outputBufferRaw =     new BYTE[FRAME_BUFSIZE_RAW];
 
     BMDTimeValue frameDuration = 0;
 
     LONGLONG latestTimeStamp = 0;
-    LONGLONG cachedTimeStamp = 0;
+    LONGLONG secondTimeStamp = 0;
+    LONGLONG thirdTimeStamp = 0;
 
     bool dirtyFrame = true;
     bool isVideoFrameReady = false;
@@ -104,9 +106,9 @@ class DeckLinkDevice : public IDeckLinkInputCallback
     virtual HRESULT  STDMETHODCALLTYPE    VideoInputFormatChanged (/* in */ BMDVideoInputFormatChangedEvents notificationEvents, /* in */ IDeckLinkDisplayMode *newDisplayMode, /* in */ BMDDetectedVideoInputFormatFlags detectedSignalFlags);
     virtual HRESULT  STDMETHODCALLTYPE    VideoInputFrameArrived (/* in */ IDeckLinkVideoInputFrame* frame, /* in */ IDeckLinkAudioInputPacket* audioPacket);
 
-    LONGLONG GetTimeStamp()
+    LONGLONG GetTimestamp()
     {
-        return cachedTimeStamp;
+        return thirdTimeStamp;
     }
 
     LONGLONG GetDurationHNS()

diff --git a/SpectatorView/Compositor/CompositorDLL/DeckLinkManager.cpp b/SpectatorView/Compositor/CompositorDLL/DeckLinkManager.cpp
@@ -123,7 +123,7 @@ LONGLONG DeckLinkManager::GetTimestamp()
 {
     if (IsEnabled())
     {
-        return deckLinkDevice->GetTimeStamp();
+        return deckLinkDevice->GetTimestamp();
     }
 
     return 0;

diff --git a/SpectatorView/Compositor/CompositorDLL/DeckLinkManager.h b/SpectatorView/Compositor/CompositorDLL/DeckLinkManager.h
@@ -16,7 +16,10 @@ class DeckLinkManager : public IFrameProvider
 
     HRESULT Initialize(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* outputTexture);
 
+    // 3 frames are caches for reliable hologram stability:
+    // Get the timestamp of the earliest (and currently rendered) cached frame.
     LONGLONG GetTimestamp();
+
     LONGLONG GetDurationHNS();
 
     void Update();

diff --git a/SpectatorView/Compositor/CompositorDLL/ElgatoFrameProvider.h b/SpectatorView/Compositor/CompositorDLL/ElgatoFrameProvider.h
@@ -106,8 +106,6 @@ class ElgatoFrameProvider : public IFrameProvider
     IBaseFilter *pNullF = NULL;
     ElgatoSampleCallback *frameCallback = NULL;
     IElgatoVideoCaptureFilter6 *filter = NULL;
-
-    BYTE* cachedBuffer = new BYTE[FRAME_BUFSIZE];
 };
 
 #endif
diff --git a/SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.cpp b/SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.cpp
@@ -20,7 +20,9 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
     // Get frame time.
     LARGE_INTEGER t;
     QueryPerformanceCounter(&t);
-    cachedTimestamp = t.QuadPart;
+    thirdTimeStamp = secondTimeStamp;
+    secondTimeStamp = latestTimeStamp;
+    latestTimeStamp = t.QuadPart;
 
     int copyLength = length;
     if (copyLength > FRAME_BUFSIZE)
@@ -29,7 +31,10 @@ STDMETHODIMP ElgatoSampleCallback::BufferCB(double time, BYTE *pBuffer, long len
         copyLength = FRAME_BUFSIZE;
     }
 
-    memcpy(cachedBytes, pBuffer, copyLength);
+    memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
+    memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
+    memcpy(latestBuffer, pBuffer, copyLength);
+
     EnterCriticalSection(&frameAccessCriticalSection);
     isVideoFrameReady = true;
     LeaveCriticalSection(&frameAccessCriticalSection);
@@ -42,13 +47,14 @@ void ElgatoSampleCallback::UpdateSRV(ID3D11ShaderResourceView* srv, bool useCPU)
     if (useCPU)
     {
         BYTE* stagingBytes = new BYTE[FRAME_BUFSIZE];
-        DirectXHelper::ConvertYUVtoBGRA(cachedBytes, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
+        // Do not cache when using the CPU
+        DirectXHelper::ConvertYUVtoBGRA(latestBuffer, stagingBytes, FRAME_WIDTH, FRAME_HEIGHT, true);
         DirectXHelper::UpdateSRV(_device, srv, stagingBytes, FRAME_WIDTH * FRAME_BPP);
         delete[] stagingBytes;
     }
     else
     {
-        DirectXHelper::UpdateSRV(_device, srv, cachedBytes, FRAME_WIDTH * FRAME_BPP);
+        DirectXHelper::UpdateSRV(_device, srv, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
     }
 }
 

diff --git a/SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.h b/SpectatorView/Compositor/CompositorDLL/ElgatoSampleCallback.h
@@ -56,7 +56,7 @@ class ElgatoSampleCallback : public ISampleGrabberCB
 
     LONGLONG GetTimestamp()
     {
-        return cachedTimestamp;
+        return thirdTimeStamp;
     }
 
     bool IsVideoFrameReady();
@@ -65,8 +65,13 @@ class ElgatoSampleCallback : public ISampleGrabberCB
     ULONG m_cRef = 0;
 
     ID3D11Device* _device;
-    BYTE* cachedBytes = new BYTE[FRAME_BUFSIZE];
-    LONGLONG cachedTimestamp = -1;
+    BYTE* thirdCachedBuffer = new BYTE[FRAME_BUFSIZE];
+    BYTE* secondCachedBuffer = new BYTE[FRAME_BUFSIZE];
+    BYTE* latestBuffer = new BYTE[FRAME_BUFSIZE];
+
+    LONGLONG latestTimeStamp = 0;
+    LONGLONG secondTimeStamp = 0;
+    LONGLONG thirdTimeStamp = 0;
 
     CRITICAL_SECTION frameAccessCriticalSection;
     bool isVideoFrameReady = false;

diff --git a/SpectatorView/Compositor/CompositorDLL/HologramQueue.cpp b/SpectatorView/Compositor/CompositorDLL/HologramQueue.cpp
@@ -6,16 +6,12 @@
 
 // These methods must all be called from the same thread.
 // Otherwise, a lock will need to be added which will substantially slow down the render thread.
-HologramQueue::HologramQueue(ID3D11Device* device)
+HologramQueue::HologramQueue()
 {
     for (int i = 0; i < MAX_QUEUE_SIZE; i++)
     {
         m_holographicFrameQueue[i].timeStamp = INVALID_TIMESTAMP;
         m_holographicFrameQueue[i].m_id = i;
-        if (m_holographicFrameQueue[i].holoTexture == nullptr)
-        {
-            m_holographicFrameQueue[i].holoTexture = DirectXHelper::CreateTexture(device, hologramQueueFrameData, HOLOGRAM_WIDTH, HOLOGRAM_HEIGHT, 4);
-        }
     }
 }
 
@@ -38,12 +34,6 @@ FrameMessage* HologramQueue::FindClosestFrame(LONGLONG timeStamp, LONGLONG frame
     for (int i = 0; i < MAX_QUEUE_SIZE; i++)
     {
         LONGLONG frameTime = m_holographicFrameQueue[i].timeStamp;
-
-        // Absolute value of timestamps in case QPC is currently reporting negative values.
-        if (timeStamp < 0) { timeStamp *= -1; }
-        if (frameTime < 0) { frameTime *= -1; }
-        if (frameOffset < 0) { frameOffset *= -1; }
-
         LONGLONG delta = timeStamp - frameTime - frameOffset;
 
         if (delta >= 0 && delta < smallestDelta)

diff --git a/SpectatorView/Compositor/CompositorDLL/HologramQueue.h b/SpectatorView/Compositor/CompositorDLL/HologramQueue.h
@@ -9,7 +9,7 @@
 #include <memory>
 #include <array>
 
-#define MAX_QUEUE_SIZE 30
+#define MAX_QUEUE_SIZE 90
 
 #define INVALID_TIMESTAMP -1
 
@@ -20,8 +20,7 @@ typedef struct
     friend class HologramQueue;
 public:
     LONGLONG timeStamp = INVALID_TIMESTAMP;
-
-    ID3D11Texture2D* holoTexture;
+    float rotX, rotY, rotZ, rotW, posX, posY, posZ;
 
     int GetId() const { return m_id; }
 private:
@@ -34,7 +33,7 @@ static BYTE* hologramQueueFrameData = new BYTE[HOLOGRAM_BUFSIZE];
 class HologramQueue
 {
 public:
-    HologramQueue(ID3D11Device* device);
+    HologramQueue();
 
     FrameMessage* GetNextFrame(LONGLONG timeStamp);
     FrameMessage* FindClosestFrame(LONGLONG timeStamp, LONGLONG frameOffset);

diff --git a/SpectatorView/Compositor/CompositorDLL/IFrameProvider.h b/SpectatorView/Compositor/CompositorDLL/IFrameProvider.h
@@ -9,7 +9,9 @@ class IFrameProvider
 public:
     // Set up the FrameProvider to start delivering frames.
     virtual HRESULT Initialize(ID3D11ShaderResourceView* colorSRV, ID3D11Texture2D* outputTexture) = 0;
-    // Get the timestamp of the latest frame.
+
+    // 4 frames are caches for reliable hologram stability:
+    // Get the timestamp of the currently rendered cached frame.
     virtual LONGLONG GetTimestamp() = 0;
 
     virtual LONGLONG GetDurationHNS() = 0;

diff --git a/SpectatorView/Compositor/CompositorDLL/OpenCVFrameProvider.cpp b/SpectatorView/Compositor/CompositorDLL/OpenCVFrameProvider.cpp
@@ -6,7 +6,8 @@
 
 #if USE_OPENCV
 
-OpenCVFrameProvider::OpenCVFrameProvider()
+OpenCVFrameProvider::OpenCVFrameProvider(bool cacheFrames) :
+    _cacheFrames(cacheFrames)
 {
 }
 
@@ -65,7 +66,9 @@ void OpenCVFrameProvider::Update()
 
     if (videoCapture->grab())
     {
-        cachedTimestamp = time.QuadPart;
+        thirdTimeStamp = secondTimeStamp;
+        secondTimeStamp = latestTimeStamp;
+        latestTimeStamp = time.QuadPart;
 
         concurrency::create_task([=]
         {
@@ -94,7 +97,9 @@ void OpenCVFrameProvider::Update()
             EnterCriticalSection(&lock);
             dirtyFrame = false;
 
-            memcpy(cachedFrame, tmpData, FRAME_BUFSIZE);
+            memcpy(thirdCachedBuffer, secondCachedBuffer, FRAME_BUFSIZE);
+            memcpy(secondCachedBuffer, latestBuffer, FRAME_BUFSIZE);
+            memcpy(latestBuffer, tmpData, FRAME_BUFSIZE);
 
             LeaveCriticalSection(&lock);
         });
@@ -104,7 +109,14 @@ void OpenCVFrameProvider::Update()
             EnterCriticalSection(&lock);
             if (!dirtyFrame)
             {
-                DirectXHelper::UpdateSRV(_device, _colorSRV, cachedFrame, FRAME_WIDTH * FRAME_BPP);
+                if (!_cacheFrames && latestBuffer != nullptr)
+                {
+                    DirectXHelper::UpdateSRV(_device, _colorSRV, latestBuffer, FRAME_WIDTH * FRAME_BPP);
+                }
+                else if (_cacheFrames && thirdCachedBuffer != nullptr)
+                {
+                    DirectXHelper::UpdateSRV(_device, _colorSRV, thirdCachedBuffer, FRAME_WIDTH * FRAME_BPP);
+                }
                 dirtyFrame = true;
             }
             LeaveCriticalSection(&lock);
@@ -131,7 +143,7 @@ bool OpenCVFrameProvider::IsVideoFrameReady()
 
 LONGLONG OpenCVFrameProvider::GetTimestamp()
 {
-    return cachedTimestamp;
+    return thirdTimeStamp;
 }
 
 LONGLONG OpenCVFrameProvider::GetDurationHNS()