Skip to content

Commit

Permalink
#3706 add 'av1' via gstreamer
Browse files Browse the repository at this point in the history
  • Loading branch information
@totaam
totaam committed Dec 15, 2022
1 parent d3e4cc7 commit c952c11
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 31 deletions.
35 changes: 21 additions & 14 deletions xpra/codecs/codec_checks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,19 @@
#this test data was generated using a 24x16 blank image as input
TEST_COMPRESSED_DATA = {
"h264": {
"YUV420P" : unhex("000000016764000aacb317cbc2000003000200000300651e244cd00000000168e970312c8b0000010605ffff56dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d2d3220746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffe841fc0a667f891ea1728763fecb5e1"),
"YUV422P" : unhex("00000001677a000abcb317cbc2000003000200000300651e244cd00000000168e970312c8b0000010605ffff56dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d2d3220746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffe841fc0a667f891ec3d121e72aecb5f"),
"YUV444P" : unhex("0000000167f4000a919662f89e1000000300100000030328f12266800000000168e970311121100000010605ffff55dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d3420746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffeeb1fc0a667f75e658f9a9fccb1f341ffff"),
"YUV420P" : (24, 16, unhex("000000016764000aacb317cbc2000003000200000300651e244cd00000000168e970312c8b0000010605ffff56dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d2d3220746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffe841fc0a667f891ea1728763fecb5e1")),
"YUV422P" : (24, 16, unhex("00000001677a000abcb317cbc2000003000200000300651e244cd00000000168e970312c8b0000010605ffff56dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d2d3220746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffe841fc0a667f891ec3d121e72aecb5f")),
"YUV444P" : (24, 16, unhex("0000000167f4000a919662f89e1000000300100000030328f12266800000000168e970311121100000010605ffff55dc45e9bde6d948b7962cd820d923eeef78323634202d20636f726520313432202d20482e3236342f4d5045472d342041564320636f646563202d20436f70796c65667420323030332d32303134202d20687474703a2f2f7777772e766964656f6c616e2e6f72672f783236342e68746d6c202d206f7074696f6e733a2063616261633d31207265663d35206465626c6f636b3d313a303a3020616e616c7973653d3078333a3078313133206d653d756d68207375626d653d38207073793d31207073795f72643d312e30303a302e3030206d697865645f7265663d31206d655f72616e67653d3136206368726f6d615f6d653d31207472656c6c69733d31203878386463743d312063716d3d3020646561647a6f6e653d32312c313120666173745f70736b69703d31206368726f6d615f71705f6f66667365743d3420746872656164733d31206c6f6f6b61686561645f746872656164733d3120736c696365645f746872656164733d30206e723d3020646563696d6174653d3120696e7465726c616365643d3020626c757261795f636f6d7061743d3020636f6e73747261696e65645f696e7472613d3020626672616d65733d3020776569676874703d32206b6579696e743d393939393939206b6579696e745f6d696e3d353030303030207363656e656375743d343020696e7472615f726566726573683d302072633d637266206d62747265653d30206372663d33382e322071636f6d703d302e36302071706d696e3d302071706d61783d3639207170737465703d342069705f726174696f3d312e34302061713d313a312e3030008000000165888404bffeeb1fc0a667f75e658f9a9fccb1f341ffff")),
},
"vp8" : {
"YUV420P" : unhex("1003009d012a1800100000070885858899848800281013ad501fc01fd01050122780feffbb029ffffa2546bd18c06f7ffe8951fffe8951af46301bdfffa22a00"),
"YUV420P" : (24, 16, unhex("1003009d012a1800100000070885858899848800281013ad501fc01fd01050122780feffbb029ffffa2546bd18c06f7ffe8951fffe8951af46301bdfffa22a00")),
},
"vp9" : {
"YUV420P" : unhex("8249834200017000f60038241c18000000200000047ffffffba9da00059fffffff753b413bffffffeea7680000"),
"YUV444P" : unhex("a249834200002e001ec007048383000000040000223fffffeea76800c7ffffffeea7680677ffffff753b40081000"),
"YUV420P" : (24, 16, unhex("8249834200017000f60038241c18000000200000047ffffffba9da00059fffffff753b413bffffffeea7680000")),
"YUV444P" : (24, 16, unhex("a249834200002e001ec007048383000000040000223fffffeea76800c7ffffffeea7680677ffffff753b40081000")),
},
"av1" : {
"YUV420P" : (64, 64, unhex("12000a0b00000002affff036be40103219110100010000004b17c531ecb5321932af9b2fab54ee58012c")),
},
}

Expand Down Expand Up @@ -120,25 +123,29 @@ def testdecoder(decoder_module, full):
return tuple(codecs)

def testdecoding(decoder_module, encoding, full):
W = 24
H = 16
test_data_set = TEST_COMPRESSED_DATA.get(encoding)
if not test_data_set:
log("%s: no test data for %s", decoder_module.get_type(), encoding)
return
for cs in decoder_module.get_input_colorspaces(encoding):
e = decoder_module.Decoder()
try:
e.init_context(encoding, W, H, cs)
test_data = test_data_set.get(cs)
if test_data:
w, h, data = test_data
else:
w = 64
h = 32
data = None
e.init_context(encoding, w, h, cs)
if data:
log("%s: testing %s / %s with %s bytes of data",
decoder_module.get_type(), encoding, cs, len(test_data))
image = e.decompress_image(test_data)
image = e.decompress_image(data)
assert image is not None, "failed to decode test data for encoding '%s' with colorspace '%s'" % (encoding, cs)
assert image.get_width()==W, "expected image of width %s but got %s" % (W, image.get_width())
assert image.get_height()==H, "expected image of height %s but got %s" % (H, image.get_height())
log(f" test passed for {W}x{H} {encoding} - {cs}")
assert image.get_width()==w, "expected image of width %s but got %s" % (w, image.get_width())
assert image.get_height()==h, "expected image of height %s but got %s" % (h, image.get_height())
log(f" test passed for {w}x{h} {encoding} - {cs}")
if full:
log("%s: testing %s / %s with junk data", decoder_module.get_type(), encoding, cs)
#test failures:
Expand Down Expand Up @@ -172,7 +179,7 @@ def testencoder(encoder_module, full):
def testencoding(encoder_module, encoding, full):
#test a bit bigger so we exercise more code:
W = 64
H = 32
H = 64
do_testencoding(encoder_module, encoding, W, H, full)

def get_encoder_max_sizes(encoder_module):
Expand Down
5 changes: 3 additions & 2 deletions xpra/codecs/codec_constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"mpeg4+mp4", "h264+mp4", "vp8+webm", "vp9+webm",
"png", "png/P", "png/L", "webp", "avif",
"rgb", "rgb24", "rgb32", "jpeg", "jpega",
"h265", "mpeg1", "mpeg2",
"h265", "av1",
"mpeg1", "mpeg2",
"scroll",
"grayscale",
)
Expand All @@ -28,7 +29,7 @@
HELP_ORDER = (
"auto",
"grayscale",
"h264", "h265", "vp8", "vp9", "mpeg4",
"h264", "h265", "av1", "vp8", "vp9", "mpeg4",
"png", "png/P", "png/L", "webp", "avif",
"rgb", "jpeg", "jpega",
"scroll",
Expand Down
2 changes: 1 addition & 1 deletion xpra/codecs/gstreamer/codec_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def process_buffer(self, buf):
log.error("Error: unable to push image buffer")
return None
try:
r = self.frame_queue.get(timeout=2 if self.frames==0 else 0.5)
r = self.frame_queue.get(timeout=2 if self.frames==0 else 1)
self.frames += 1
return r
except Empty:
Expand Down
9 changes: 6 additions & 3 deletions xpra/codecs/gstreamer/decoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
assert get_version and get_type and init_module and cleanup_module


CODECS = ("vp8", "h264")
CODECS = ("vp8", "h264", "av1")
def get_encodings():
return CODECS

Expand Down Expand Up @@ -51,9 +51,12 @@ def create_pipeline(self, options):
decode = ["vp8dec"]
elif self.encoding=="h264":
stream_caps = f"video/x-h264,profile=main,stream-format=avc,alignment=au,width={self.width},height={self.height}"
decode = ["vaapih264dec"]
decode = ["openh264dec"]
#decode = ["vaapih264dec"]
#decode = ["openh264dec"]
decode = ["avdec_h264"]
elif self.encoding=="av1":
stream_caps = f"video/x-av1,width={self.width},height={self.height}"
decode = ["av1dec"]
else:
raise RuntimeError(f"invalid encoding {self.encoding}")
elements = [
Expand Down
50 changes: 42 additions & 8 deletions xpra/codecs/gstreamer/encoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
# later version. See the file COPYING for details.

import os
from queue import Empty

from xpra.os_util import hexstr
from xpra.util import parse_simple_dict
from xpra.codecs.codec_constants import video_spec
from xpra.gst_common import (
Expand All @@ -26,9 +26,10 @@
log = Logger("encoder", "gstreamer")

assert get_version and init_module and cleanup_module
#ENCODER_PLUGIN = "vaapih264enc"
#ENCODER_PLUGIN = "x264enc"
ENCODER_PLUGIN = os.environ.get("XPRA_GSTREAMER_ENCODER_PLUGIN", "vaapih264enc")
ENCODERS = {
"h264" : ("vaapih264enc", "x264enc"),
"av1" : ("av1enc", ),
}
DEFAULT_ENCODER_OPTIONS = {
"vaapih264enc" : {
"max-bframes" : 0,
Expand All @@ -48,12 +49,35 @@
"threads" : 1,
"key-int-max" : 15,
"intra-refresh" : True,
},
#"svtav1enc" : {
# "speed" : 12,
# "gop-size" : 251,
# "intra-refresh" : 1, #open gop
# "lookahead" : 0,
# "rc" : 1, #vbr
# },
#"svthevcenc" : {
# "b-pyramid" : 0,
# "baselayer-mode" : 1,
# "enable-open-gop" : True,
# "key-int-max" : 255,
# "lookahead" : 0,
# "pred-struct" : 0,
# "rc" : 1, #vbr
# "speed" : 9,
# "tune" : 0,
# }
"av1enc" : {
"cpu-used" : 5,
"end-usage" : 2, #cq
}
}


CODECS = ("h264", "av1")
def get_encodings():
return ("h264", )
return CODECS

def get_input_colorspaces(encoding):
assert encoding in get_encodings()
Expand All @@ -77,7 +101,9 @@ def get_spec(encoding, colorspace):
codec_class=Encoder, codec_type=get_type(),
quality=40, speed=40,
setup_cost=100, cpu_cost=cpu_cost, gpu_cost=gpu_cost,
width_mask=0xFFFE, height_mask=0xFFFE, max_w=4096, max_h=4096)
width_mask=0xFFFE, height_mask=0xFFFE,
min_w=64, min_h=64,
max_w=4096, max_h=4096)


class Encoder(VideoPipeline):
Expand Down Expand Up @@ -112,7 +138,14 @@ def create_pipeline(self, options):
}[self.colorspace]
CAPS = f"video/x-raw,width={self.width},height={self.height},format=(string){gst_rgb_format},framerate=60/1,interlace=progressive"
#parse encoder plugin string:
parts = ENCODER_PLUGIN.split(" ", 1)
encoder_options = ENCODERS.get(self.encoding, ())
encoder_str = os.environ.get("XPRA_GSTREAMER_ENCODER_PLUGIN")
if not encoder_str:
if not encoder_options:
raise ValueError(f"{self.encoding} is not supported here")
#choose the first option (ie: "vaapih264enc"):
encoder_str = encoder_options[0]
parts = encoder_str.split(" ", 1)
encoder = parts[0]
encoder_options = DEFAULT_ENCODER_OPTIONS.get(encoder, {})
if len(parts)==2:
Expand Down Expand Up @@ -149,6 +182,7 @@ def on_new_sample(self, _bus):
log("on_new_sample size=%s", size)
if size:
data = buf.extract_dup(0, size)
#log(" output=%s", hexstr(data))
client_info = {}
pts = normv(buf.pts)
if pts>=0:
Expand Down Expand Up @@ -196,4 +230,4 @@ def selftest(full=False):
log("gstreamer encoder selftest: %s", get_info())
from xpra.codecs.codec_checks import testencoder
from xpra.codecs.gstreamer import encoder
assert testencoder(encoder, full)
encoder.CODECS = testencoder(encoder, full)
4 changes: 3 additions & 1 deletion xpra/codecs/loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ def get_encoding_name(encoding):
"jpeg" : "JPEG",
"jpega" : "JPEG with alpha",
"avif" : "AVIF",
"av1" : "AV1",
"rgb" : " + ".join(get_rgb_compression_options()) + " (24/32bpp)",
}
return ENCODINGS_TO_NAME.get(encoding, encoding)
Expand All @@ -342,7 +343,8 @@ def get_encoding_help(encoding):
"png/L" : "Portable Network Graphics (lossy, 8bpp grayscale)",
"webp" : "WebP compression (supports lossless and lossy modes)",
"jpeg" : "JPEG lossy compression",
"avif" : "AVIF",
"avif" : "AVIF: AV1 Image File Format",
"av1" : "AV1: AOMedia Video 1",
"rgb" : "Raw RGB pixels, lossless,"
+f" compressed using {compressors_str} (24bpp or 32bpp for transparency)",
"scroll" : "motion vectors, supplemented with picture codecs",
Expand Down
4 changes: 2 additions & 2 deletions xpra/codecs/video_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ def try_import_modules(*codec_names):

#all the codecs we know about:
#try to import the module that contains them (cheap check):
ALL_VIDEO_ENCODER_OPTIONS = try_import_modules("x264", "vpx", "x265", "nvenc", "ffmpeg", "nvjpeg", "jpeg", "webp", "enc_gstreamer", "dec_gstreamer")
ALL_VIDEO_ENCODER_OPTIONS = try_import_modules("x264", "vpx", "x265", "nvenc", "ffmpeg", "nvjpeg", "jpeg", "webp", "enc_gstreamer")
HARDWARE_ENCODER_OPTIONS = try_import_modules("nvenc", "nvjpeg")
ALL_CSC_MODULE_OPTIONS = try_import_modules("swscale", "cython", "libyuv")
NO_GFX_CSC_OPTIONS = []
ALL_VIDEO_DECODER_OPTIONS = try_import_modules("avcodec2", "vpx")
ALL_VIDEO_DECODER_OPTIONS = try_import_modules("avcodec2", "vpx", "dec_gstreamer")

PREFERRED_ENCODER_ORDER = ("nvenc", "nvjpeg", "x264", "vpx", "jpeg", "webp", "x265", "enc_gstreamer")
PREFERRED_DECODER_ORDER = ("avcodec2", "vpx", "dec_gstreamer")
Expand Down

0 comments on commit c952c11

Please sign in to comment.