tests/debug: plug some resource leaks

copyparty/u2idx.py

@@ -70,6 +70,9 @@ def log(self, msg: str, c: Union[int, str] = 0) -> None:
         self.log_func("u2idx", msg, c)
 
     def shutdown(self) -> None:
+        if not HAVE_SQLITE3:
+            return
+
         for cur in self.cur.values():
             db = cur.connection
             try:
@@ -80,6 +83,12 @@ def shutdown(self) -> None:
             cur.close()
             db.close()
 
+        for cur in (self.mem_cur, self.sh_cur):
+            if cur:
+                db = cur.connection
+                cur.close()
+                db.close()
+
     def fsearch(
         self, uname: str, vols: list[VFS], body: dict[str, Any]
     ) -> list[dict[str, Any]]:

copyparty/up2k.py

@@ -5297,6 +5297,11 @@ def shutdown(self) -> None:
             cur.close()
             db.close()
 
+        if self.mem_cur:
+            db = self.mem_cur.connection
+            self.mem_cur.close()
+            db.close()
+
         self.registry = {}
 
 

tests/test_cp.py

@@ -19,6 +19,8 @@ def setUp(self):
         self.td = tu.get_ramdisk()
 
     def tearDown(self):
+        if self.conn:
+            self.conn.shutdown()
         os.chdir(tempfile.gettempdir())
         shutil.rmtree(self.td)
 
@@ -62,7 +64,6 @@ def test(self):
 
         self.conn = None
         self.fstab = None
-        self.ctr = 0  # 2304
         for dedup, act_exp in product(tc_dedup, tcs):
             action, expect = act_exp.split(" ", 1)
             t = "dedup:%s  action:%s" % (dedup, action)

tests/test_dedup.py

@@ -34,6 +34,8 @@ def setUp(self):
         ]
 
     def tearDown(self):
+        if self.conn:
+            self.conn.shutdown()
         os.chdir(tempfile.gettempdir())
         shutil.rmtree(self.td)
 

tests/test_dots.py

@@ -17,6 +17,11 @@
 from tests import util as tu
 from tests.util import Cfg
 
+try:
+    from typing import Optional
+except:
+    pass
+
 
 def hdr(query, uname):
     h = "GET /%s HTTP/1.1\r\nPW: %s\r\nConnection: close\r\n\r\n"
@@ -29,12 +34,21 @@ def __init__(self, *a, **ka):
         self.is_dut = True
 
     def setUp(self):
+        self.conn: Optional[tu.VHttpConn] = None
         self.td = tu.get_ramdisk()
 
     def tearDown(self):
+        if self.conn:
+            self.conn.shutdown()
         os.chdir(tempfile.gettempdir())
         shutil.rmtree(self.td)
 
+    def cinit(self):
+        if self.conn:
+            self.conn.shutdown()
+            self.conn = None
+        self.conn = tu.VHttpConn(self.args, self.asrv, self.log, b"")
+
     def test_dots(self):
         td = os.path.join(self.td, "vfs")
         os.mkdir(td)
@@ -57,6 +71,7 @@ def test_dots(self):
         vcfg = [".::r,u1:r.,u2", "a:a:r,u1:r,u2", ".b:.b:r.,u1:r,u2"]
         self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], e2dsa=True)
         self.asrv = AuthSrv(self.args, self.log)
+        self.cinit()
 
         self.assertEqual(self.tardir("", "u1"), "f0 t/f1 a/f3 a/da/f4")
         self.assertEqual(self.tardir(".t", "u1"), "f2")
@@ -88,6 +103,7 @@ def test_dots(self):
         self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"], dotsrch=False, e2d=True)
         self.asrv = AuthSrv(self.args, self.log)
         u2idx = U2idx(self)
+        self.cinit()
 
         x = u2idx.search("u1", self.asrv.vfs.all_vols.values(), "", 999)
         x = " ".join(sorted([x["rp"] for x in x[0]]))
@@ -113,6 +129,8 @@ def test_dots(self):
         ]
         self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"])
         self.asrv = AuthSrv(self.args, self.log)
+        self.cinit()
+
         zj = json.loads(self.curl("?ls", "u1")[1])
         url = "?k=" + zj["dk"]
         # should descend into folders, but not other volumes:
@@ -148,6 +166,7 @@ def test_dk_fk(self):
 
         self.args = Cfg(v=vcfg, a=["u1:u1", "u2:u2"])
         self.asrv = AuthSrv(self.args, self.log)
+        self.cinit()
 
         dk = {}
         for d in "dk dks dk,fk dks,fk".split():
@@ -353,7 +372,7 @@ def tarsel(self, url, uname, sel):
 
     def curl(self, url, uname, binary=False, req=b""):
         req = req or hdr(url, uname)
-        conn = tu.VHttpConn(self.args, self.asrv, self.log, req)
+        conn = self.conn.setbuf(req)
         HttpCli(conn).run()
         if binary:
             h, b = conn.s._reply.split(b"\r\n\r\n", 1)

tests/test_hooks.py

@@ -12,6 +12,11 @@
 from tests import util as tu
 from tests.util import Cfg
 
+try:
+    from typing import Optional
+except:
+    pass
+
 
 def hdr(query):
     h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n"
@@ -20,6 +25,7 @@ def hdr(query):
 
 class TestHooks(unittest.TestCase):
     def setUp(self):
+        self.conn: Optional[tu.VHttpConn] = None
         self.td = tu.get_ramdisk()
 
     def tearDown(self):
@@ -34,6 +40,12 @@ def reset(self):
         os.chdir(td)
         return td
 
+    def cinit(self):
+        if self.conn:
+            self.conn.shutdown()
+            self.conn = None
+        self.conn = tu.VHttpConn(self.args, self.asrv, self.log, b"")
+
     def test(self):
         vcfg = ["a/b/c/d:c/d:A", "a:a:r"]
 
@@ -59,6 +71,7 @@ def test(self):
                     ka = {hooktype: ["j,c1,h.py"]}
                     self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka)
                     self.asrv = AuthSrv(self.args, self.log)
+                    self.cinit()
 
                     h, b = upfun(url_up)
                     self.assertIn("201 Created", h)
@@ -73,7 +86,7 @@ def put(self, url):
         buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n"
         buf = buf.format(url, len(url) + 4).encode("utf-8")
         print("PUT -->", buf)
-        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        conn = self.conn.setbuf(buf)
         HttpCli(conn).run()
         ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
         print("PUT <--", ret)
@@ -92,14 +105,14 @@ def bup(self, url):
         buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8")
         buf = (hdr % (url, len(buf))).encode("utf-8") + buf
         print("PoST -->", buf)
-        conn = tu.VHttpConn(self.args, self.asrv, self.log, buf)
+        conn = self.conn.setbuf(buf)
         HttpCli(conn).run()
         ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1)
         print("POST <--", ret)
         return ret
 
     def curl(self, url, binary=False):
-        conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url))
+        conn = self.conn.setbuf(hdr(url))
         HttpCli(conn).run()
         if binary:
             h, b = conn.s._reply.split(b"\r\n\r\n", 1)

tests/test_httpcli.py

@@ -178,6 +178,8 @@ def test(self):
                     ap = os.path.join(vn.realpath, rem)
                     os.unlink(ap)
 
+            self.conn.shutdown()
+
     def can_rw(self, fp):
         # lowest non-neutral folder declares permissions
         expect = fp.split("/")[:-1]

tests/test_metrics.py

@@ -27,6 +27,8 @@ def setUp(self):
         os.chdir(self.td)
 
     def tearDown(self):
+        if self.conn:
+            self.conn.shutdown()
         os.chdir(tempfile.gettempdir())
         shutil.rmtree(self.td)
 

tests/test_mv.py

@@ -25,6 +25,8 @@ def setUp(self):
         self.td = tu.get_ramdisk()
 
     def tearDown(self):
+        if not PY2 and self.conn:
+            self.conn.shutdown()
         os.chdir(tempfile.gettempdir())
         shutil.rmtree(self.td)
 

tests/util.py

@@ -122,7 +122,7 @@ class Cfg(Namespace):
     def __init__(self, a=None, v=None, c=None, **ka0):
         ka = {}
 
-        ex = "chpw daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic hardlink_only nid nih no_acode no_athumb no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title ohead q rand re_dirsz rss smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol zs"
+        ex = "chpw daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic hardlink_only nid nih no_acode no_athumb no_clone no_cp no_dav no_db_ip no_del no_dirsz no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nsort nw og og_no_head og_s_title ohead q rand re_dirsz rss smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol zs"
         ka.update(**{k: False for k in ex.split()})
 
         ex = "dedup dotpart dotsrch hook_v no_dhash no_fastboot no_fpool no_htp no_rescan no_sendfile no_ses no_snap no_up_list no_voldump re_dhash plain_ip"
@@ -278,13 +278,22 @@ def get_u2idx(self):
         self.u2idx = self.u2idx or U2idx(self)
         return self.u2idx
 
+    def shutdown(self):
+        if self.u2idx:
+            self.u2idx.shutdown()
+
 
 class VHttpSrvUp2k(VHttpSrv):
     def __init__(self, args, asrv, log):
         super(VHttpSrvUp2k, self).__init__(args, asrv, log)
         self.hub = VHub(args, asrv, log)
         self.broker = VBrokerThr(self.hub)
 
+    def shutdown(self):
+        self.hub.up2k.shutdown()
+        if self.u2idx:
+            self.u2idx.shutdown()
+
 
 class VHttpConn(object):
     def __init__(self, args, asrv, log, buf, use_up2k=False):
@@ -322,6 +331,9 @@ def setbuf(self, buf):
         self.sr = Unrecv(self.s, None)  # type: ignore
         return self
 
+    def shutdown(self):
+        self.hsrv.shutdown()
+
 
 if WINDOWS:
     os.system("rem")