From 6356b28e9cb9b6e18d732a674bb8002169714e57 Mon Sep 17 00:00:00 2001
From: Antoine Martin <totaam@xpra.org>
Date: Tue, 2 Feb 2016 19:26:50 +0000
Subject: [PATCH] on some cards (or drivers? or kernels?) we can fail to
 initialize the context without getting an error code back, which can cause
 hard crashes later

git-svn-id: https://xpra.org/svn/Xpra/trunk@11815 3bb7dfac-3a0b-4e04-842a-767bc560f471
---
 src/xpra/codecs/nvenc4/encoder.pyx | 3 +++
 src/xpra/codecs/nvenc5/encoder.pyx | 3 +++
 src/xpra/codecs/nvenc6/encoder.pyx | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/src/xpra/codecs/nvenc4/encoder.pyx b/src/xpra/codecs/nvenc4/encoder.pyx
index daaf4ffe8d..84084434d2 100644
--- a/src/xpra/codecs/nvenc4/encoder.pyx
+++ b/src/xpra/codecs/nvenc4/encoder.pyx
@@ -2301,6 +2301,7 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
@@ -2309,6 +2310,8 @@ cdef class Encoder:
             log(msg)
             raise TransientCodecException(msg)
         raiseNVENC(r, "opening session")
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         context_counter.increase()
         context_gen_counter.increase()
         log("success, encoder context=%#x (%s context%s in use)", <unsigned long> self.context, context_counter, engs(context_counter))
diff --git a/src/xpra/codecs/nvenc5/encoder.pyx b/src/xpra/codecs/nvenc5/encoder.pyx
index b031ddcd8c..24e91c73e3 100644
--- a/src/xpra/codecs/nvenc5/encoder.pyx
+++ b/src/xpra/codecs/nvenc5/encoder.pyx
@@ -2337,6 +2337,7 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
@@ -2344,6 +2345,8 @@ cdef class Encoder:
             msg = "NV_ENC_ERR_UNSUPPORTED_DEVICE: could not open encode session (out of resources / no more codec contexts?)"
             log(msg)
             raise TransientCodecException(msg)
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         raiseNVENC(r, "opening session")
         context_counter.increase()
         context_gen_counter.increase()
diff --git a/src/xpra/codecs/nvenc6/encoder.pyx b/src/xpra/codecs/nvenc6/encoder.pyx
index e7fc0ede60..9efc911485 100644
--- a/src/xpra/codecs/nvenc6/encoder.pyx
+++ b/src/xpra/codecs/nvenc6/encoder.pyx
@@ -2378,6 +2378,7 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
@@ -2385,6 +2386,8 @@ cdef class Encoder:
             msg = "NV_ENC_ERR_UNSUPPORTED_DEVICE: could not open encode session (out of resources / no more codec contexts?)"
             log(msg)
             raise TransientCodecException(msg)
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         raiseNVENC(r, "opening session")
         context_counter.increase()
         context_gen_counter.increase()