diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1077f456..11bfea703 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,7 +38,7 @@ checks:tests: tags: - vm-kvm include: -- file: /r4.2/gitlab-base.yml +- file: /r4.3/gitlab-base.yml project: QubesOS/qubes-continuous-integration -- file: /r4.2/gitlab-host.yml +- file: /r4.3/gitlab-host.yml project: QubesOS/qubes-continuous-integration diff --git a/qubes/api/__init__.py b/qubes/api/__init__.py index 28b0d99d4..7bce2a369 100644 --- a/qubes/api/__init__.py +++ b/qubes/api/__init__.py @@ -217,9 +217,6 @@ def validate_size(self, untrusted_size: bytes) -> int: class QubesDaemonProtocol(asyncio.Protocol): buffer_size = 65536 header = struct.Struct('Bx') - # keep track of connections, to gracefully close them at server exit - # (including cleanup of integration test) - connections = set() def __init__(self, handler, *args, app, debug=False, **kwargs): super().__init__(*args, **kwargs) @@ -234,7 +231,6 @@ def __init__(self, handler, *args, app, debug=False, **kwargs): def connection_made(self, transport): self.transport = transport - self.connections.add(self) def connection_lost(self, exc): self.untrusted_buffer.close() @@ -242,7 +238,6 @@ def connection_lost(self, exc): if self.mgmt is not None: self.mgmt.cancel() self.transport = None - self.connections.remove(self) # pylint: disable=arguments-differ,arguments-renamed def data_received(self, untrusted_data): diff --git a/qubes/tests/__init__.py b/qubes/tests/__init__.py index 61d571762..f0db98fbb 100644 --- a/qubes/tests/__init__.py +++ b/qubes/tests/__init__.py @@ -517,7 +517,9 @@ def cleanup_loop(self): # lifecycle, and it is unwatched only at loop.close(); so we cannot just # check selector for non-emptiness assert len(self.loop._selector.get_map()) \ - == int(self.loop._ssock is not None) + == int(self.loop._ssock is not None), \ + f"unexpected selector map entries: " \ + f"{dict(self.loop._selector.get_map())!r}" del self.loop @@ -783,17 +785,12 @@ def cleanup_app(self): for sock in server.sockets: os.unlink(sock.getsockname()) server.close() + # close all existing connections, especially this will interrupt + # running admin.Events calls, which do keep reference to Qubes() and + # libvirt connection + server.close_clients() del server - # close all existing connections, especially this will interrupt - # running admin.Events calls, which do keep reference to Qubes() and - # libvirt connection - conn = None - for conn in qubes.api.QubesDaemonProtocol.connections: - if conn.transport: - conn.transport.abort() - del conn - self.loop.run_until_complete(asyncio.wait([ self.loop.create_task(server.wait_closed()) for server in self.qubesd])) del self.qubesd diff --git a/qubes/tests/integ/devices_block.py b/qubes/tests/integ/devices_block.py index 74e60b4ae..e3d17b73b 100644 --- a/qubes/tests/integ/devices_block.py +++ b/qubes/tests/integ/devices_block.py @@ -132,7 +132,7 @@ def test_010_list_dm(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.serial, self.img_path, + self.assertNotEqual(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) @@ -163,12 +163,12 @@ def test_011_list_dm_mounted(self): dev_list = list(self.vm.devices['block']) for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.serial, self.img_path, + self.assertNotEqual(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) else: - self.assertNotEquals(dev.serial, 'test-dm', + self.assertNotEqual(dev.serial, 'test-dm', "Device {} ({}) should not be listed as it is " "mounted".format(dev, 'test-dm') ) @@ -188,7 +188,7 @@ def test_012_list_dm_delayed(self): found = False for dev in dev_list: if dev.ident.startswith('loop'): - self.assertNotEquals(dev.serial, self.img_path, + self.assertNotEqual(dev.serial, self.img_path, "Device {} ({}) should not be listed as it is used in " "device-mapper".format(dev, self.img_path) ) diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py index 196f04cc2..cde89049c 100644 --- a/qubes/tests/integ/dispvm.py +++ b/qubes/tests/integ/dispvm.py @@ -108,7 +108,7 @@ def test_003_cleanup_destroyed(self): lines = lines_task.result().splitlines() self.assertTrue(lines, 'No output received from DispVM') dispvm_name = lines[0] - self.assertNotEquals(dispvm_name, b"ERROR") + self.assertNotEqual(dispvm_name, b"ERROR") self.assertNotIn(dispvm_name, self.app.domains) @@ -496,6 +496,11 @@ def test_100_open_in_dispvm(self): test_txt_content = f.read() self.assertEqual(test_txt_content.strip(), b"test1") + # this doesn't really close the application, only the qrexec-client + # process that started it; but clean it up anyway to not leak processes + app.terminate() + self.loop.run_until_complete(app.wait()) + def create_testcases_for_templates(): return qubes.tests.create_testcases_for_templates('TC_20_DispVM', diff --git a/qubes/tests/integ/vm_qrexec_gui.py b/qubes/tests/integ/vm_qrexec_gui.py index 1b0bda315..c90ce0b6f 100644 --- a/qubes/tests/integ/vm_qrexec_gui.py +++ b/qubes/tests/integ/vm_qrexec_gui.py @@ -155,7 +155,7 @@ def test_012_qubes_desktop_run(self): else: xterm_desktop_path = xterm_desktop_path_debian self.loop.run_until_complete(self.wait_for_session(self.testvm1)) - self.loop.run_until_complete( + p = self.loop.run_until_complete( self.testvm1.run('qubes-desktop-run {}'.format(xterm_desktop_path))) title = 'user@{}'.format(self.testvm1.name) if self.template.count("whonix"): @@ -168,6 +168,11 @@ def test_012_qubes_desktop_run(self): 'windowactivate', '--sync', 'type', 'exit\n']) self.wait_for_window(title, show=False) + try: + self.loop.run_until_complete(asyncio.wait_for(p.wait(), 5)) + except asyncio.TimeoutError: + p.terminate() + self.loop.run_until_complete(p.wait()) def test_100_qrexec_filecopy(self): self.loop.run_until_complete(asyncio.gather( @@ -473,10 +478,10 @@ def test_210_time_sync(self): # get current time current_time, _ = self.loop.run_until_complete( self.testvm1.run_for_stdio('date -u +%s')) - self.assertAlmostEquals(int(vm_time), int(current_time), delta=30) + self.assertAlmostEqual(int(vm_time), int(current_time), delta=30) dom0_time = subprocess.check_output(['date', '-u', '+%s']) - self.assertAlmostEquals(int(dom0_time), int(current_time), delta=30) + self.assertAlmostEqual(int(dom0_time), int(current_time), delta=30) except: # reset time to some approximation of the real time diff --git a/qubes/tests/integ/vm_update.py b/qubes/tests/integ/vm_update.py index e9732d9e2..0d32e7565 100644 --- a/qubes/tests/integ/vm_update.py +++ b/qubes/tests/integ/vm_update.py @@ -20,6 +20,7 @@ # License along with this library; if not, see . # import asyncio +import contextlib import os import shutil import subprocess @@ -226,6 +227,13 @@ def setUp(self): name=self.make_vm_name('vm1'), label='red') self.loop.run_until_complete(self.testvm1.create_on_disk()) + self.repo_proc = None + + def tearDown(self): + if self.repo_proc: + self.repo_proc.terminate() + self.loop.run_until_complete(self.repo_proc.wait()) + super().tearDown() def test_000_simple_update(self): """ @@ -322,12 +330,12 @@ def create_repo_and_serve(self): """ if self.template.count("debian") or self.template.count("whonix"): self.create_repo_apt() - self.loop.run_until_complete(self.netvm_repo.run( + self.repo_proc = self.loop.run_until_complete(self.netvm_repo.run( 'cd /tmp/apt-repo && python3 -m http.server 8080', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) elif self.template.count("fedora"): self.create_repo_yum() - self.loop.run_until_complete(self.netvm_repo.run( + self.repo_proc = self.loop.run_until_complete(self.netvm_repo.run( 'cd /tmp/yum-repo && python3 -m http.server 8080', stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) else: @@ -482,9 +490,11 @@ def turn_off_repo(self): :type self: qubes.tests.SystemTestCase | VmUpdatesMixin """ - self.loop.run_until_complete(self.netvm_repo.run( - r"kill -9 `ps -ef | grep python | awk '{print $2}'`", + self.loop.run_until_complete(self.netvm_repo.run_for_stdio( + r"kill -9 `ps -ef | grep [h]ttp.server | awk '{print $2}'`", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)) + self.loop.run_until_complete(self.repo_proc.wait()) + self.repo_proc = None def update_via_proxy_qubes_vm_update_impl( self, diff --git a/qubes/tools/qubesd.py b/qubes/tools/qubesd.py index 7707ba31a..56bcd76f9 100644 --- a/qubes/tools/qubesd.py +++ b/qubes/tools/qubesd.py @@ -24,6 +24,7 @@ def sighandler(loop, signame, servers): print('caught {}, exiting'.format(signame)) for server in servers: server.close() + server.close_clients() loop.stop() parser = qubes.tools.QubesArgumentParser(description='Qubes OS daemon') @@ -71,13 +72,6 @@ def main(args=None): loop.run_forever() loop.run_until_complete(asyncio.wait([ loop.create_task(server.wait_closed()) for server in servers])) - for sockname in socknames: - try: - os.unlink(sockname) - except FileNotFoundError: - args.app.log.warning( - 'socket {} got unlinked sometime before shutdown'.format( - sockname)) finally: loop.close() diff --git a/tests-data/dispvm-open-thunderbird-attachment b/tests-data/dispvm-open-thunderbird-attachment index 3b7234ac8..8ab88edcb 100644 --- a/tests-data/dispvm-open-thunderbird-attachment +++ b/tests-data/dispvm-open-thunderbird-attachment @@ -82,6 +82,9 @@ def skip_autoconf2(tb): def open_attachment(tb, name): # message subject msg = tb.child(name='Test Message - .*', roleName='frame') + # resize the window so that button is on screen even on 1024x768 + subprocess.call(["xdotool", "search", "Test Message", "windowsize", "900", "700"]) + time.sleep(1) msg.button(name).click() confirm = tb.child(name='Opening ' + name, roleName='frame') time.sleep(3)