on some cards (or drivers? or kernels?) we can fail to initialize the…

… context without getting an error code back, which can cause hard crashes later git-svn-id: https://xpra.org/svn/Xpra/trunk@11815 3bb7dfac-3a0b-4e04-842a-767bc560f471
Xpra-org · Feb 2, 2016 · 6356b28 · 6356b28
1 parent 6c784cd
commit 6356b28
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 0 deletions.
diff --git a/src/xpra/codecs/nvenc4/encoder.pyx b/src/xpra/codecs/nvenc4/encoder.pyx
@@ -2301,6 +2301,7 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
@@ -2309,6 +2310,8 @@ cdef class Encoder:
             log(msg)
             raise TransientCodecException(msg)
         raiseNVENC(r, "opening session")
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         context_counter.increase()
         context_gen_counter.increase()
         log("success, encoder context=%#x (%s context%s in use)", <unsigned long> self.context, context_counter, engs(context_counter))

diff --git a/src/xpra/codecs/nvenc5/encoder.pyx b/src/xpra/codecs/nvenc5/encoder.pyx
@@ -2337,13 +2337,16 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
             last_context_failure = time.time()
             msg = "NV_ENC_ERR_UNSUPPORTED_DEVICE: could not open encode session (out of resources / no more codec contexts?)"
             log(msg)
             raise TransientCodecException(msg)
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         raiseNVENC(r, "opening session")
         context_counter.increase()
         context_gen_counter.increase()

diff --git a/src/xpra/codecs/nvenc6/encoder.pyx b/src/xpra/codecs/nvenc6/encoder.pyx
@@ -2378,13 +2378,16 @@ cdef class Encoder:
         pstr = cstr[:sizeof(NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS)]
         if DEBUG_API:
             log("calling nvEncOpenEncodeSessionEx @ %#x", <unsigned long> self.functionList.nvEncOpenEncodeSessionEx)
+        self.context = NULL
         with nogil:
             r = self.functionList.nvEncOpenEncodeSessionEx(&params, &self.context)
         if r==NV_ENC_ERR_UNSUPPORTED_DEVICE:
             last_context_failure = time.time()
             msg = "NV_ENC_ERR_UNSUPPORTED_DEVICE: could not open encode session (out of resources / no more codec contexts?)"
             log(msg)
             raise TransientCodecException(msg)
+        if self.context==NULL:
+            raise Exception("cannot open encoding session, context is NULL")
         raiseNVENC(r, "opening session")
         context_counter.increase()
         context_gen_counter.increase()