From 6c94a63f1c0fa4aa6d67fd2141c012ae41e76b76 Mon Sep 17 00:00:00 2001 From: ed Date: Sun, 11 Aug 2024 14:52:32 +0000 Subject: [PATCH] add hook side-effects; closes #86 hooks can now interrupt or redirect actions, and initiate related actions, by printing json on stdout with commands mainly to mitigate limitations such as sharex/sharex#3992 xbr/xau can redirect uploads to other destinations with `reloc` and most hooks can initiate indexing or deletion of additional files by giving a list of vpaths in json-keys `idx` or `del` there are limitations; * xbu/xau effects don't apply to ftp, tftp, smb * xau will intentionally fail if a reloc destination exists * xau effects do not apply to up2k also provides more details for hooks: * xbu/xau: basic-uploader vpath with filename * xbr/xar: add client ip --- README.md | 2 + bin/hooks/into-the-cache-it-goes.py | 4 +- bin/hooks/reloc-by-ext.py | 94 ++++++++++ copyparty/__main__.py | 6 + copyparty/authsrv.py | 6 +- copyparty/ftpd.py | 7 +- copyparty/httpcli.py | 248 ++++++++++++++++++------- copyparty/smbd.py | 22 ++- copyparty/tftpd.py | 20 ++- copyparty/up2k.py | 268 +++++++++++++++++++--------- copyparty/util.py | 131 ++++++++++++-- docs/devnotes.md | 28 +++ docs/versus.md | 4 + tests/test_hooks.py | 111 ++++++++++++ tests/util.py | 17 +- 15 files changed, 793 insertions(+), 175 deletions(-) create mode 100644 bin/hooks/reloc-by-ext.py create mode 100644 tests/test_hooks.py diff --git a/README.md b/README.md index c73b8658..1a00bc35 100644 --- a/README.md +++ b/README.md @@ -1313,6 +1313,8 @@ you can set hooks before and/or after an event happens, and currently you can ho there's a bunch of flags and stuff, see `--help-hooks` +if you want to write your own hooks, see [devnotes](./docs/devnotes.md#event-hooks) + ### upload events diff --git a/bin/hooks/into-the-cache-it-goes.py b/bin/hooks/into-the-cache-it-goes.py index 99c5b669..b07e87e3 100644 --- a/bin/hooks/into-the-cache-it-goes.py +++ b/bin/hooks/into-the-cache-it-goes.py @@ -41,8 +41,8 @@ t10 = abort download and continue if it takes longer than 10sec example usage as a volflag (per-volume config): - -v srv/inc:inc:r:rw,ed:xau=j,t10,bin/hooks/into-the-cache-it-goes.py - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + -v srv/inc:inc:r:rw,ed:c,xau=j,t10,bin/hooks/into-the-cache-it-goes.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (share filesystem-path srv/inc as volume /inc, readable by everyone, read-write for user 'ed', diff --git a/bin/hooks/reloc-by-ext.py b/bin/hooks/reloc-by-ext.py new file mode 100644 index 00000000..89033016 --- /dev/null +++ b/bin/hooks/reloc-by-ext.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import json +import os +import sys + + +_ = r""" +relocate/redirect incoming uploads according to file extension + +example usage as global config: + --xbu j,c1,bin/hooks/reloc-by-ext.py + +parameters explained, + xbu = execute before upload + j = this hook needs upload information as json (not just the filename) + c1 = this hook returns json on stdout, so tell copyparty to read that + +example usage as a volflag (per-volume config): + -v srv/inc:inc:r:rw,ed:c,xbu=j,c1,bin/hooks/reloc-by-ext.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + (share filesystem-path srv/inc as volume /inc, + readable by everyone, read-write for user 'ed', + running this plugin on all uploads with the params explained above) + +example usage as a volflag in a copyparty config file: + [/inc] + srv/inc + accs: + r: * + rw: ed + flags: + xbu: j,c1,bin/hooks/reloc-by-ext.py + +note: this only works with the basic uploader (sharex and such), + does not work with up2k / dragdrop into browser + +note: this could also work as an xau hook (after-upload), but + because it doesn't need to read the file contents its better + as xbu (before-upload) since that's safer / less buggy +""" + + +PICS = "avif bmp gif heic heif jpeg jpg jxl png psd qoi tga tif tiff webp" +VIDS = "3gp asf avi flv mkv mov mp4 mpeg mpeg2 mpegts mpg mpg2 nut ogm ogv rm ts vob webm wmv" +MUSIC = "aac aif aiff alac amr ape dfpwm flac m4a mp3 ogg opus ra tak tta wav wma wv" + + +def main(): + inf = json.loads(sys.argv[1]) + vdir, fn = os.path.split(inf["vp"]) + + try: + fn, ext = fn.rsplit(".", 1) + except: + # no file extension; abort + return + + ext = ext.lower() + + ## + ## some example actions to take; pick one by + ## selecting it inside the print at the end: + ## + + # create a subfolder named after the filetype and move it into there + into_subfolder = {"vp": ext} + + # move it into a toplevel folder named after the filetype + into_toplevel = {"vp": "/" + ext} + + # move it into a filetype-named folder next to the target folder + into_sibling = {"vp": "../" + ext} + + # move images into "/just/pics", vids into "/just/vids", + # music into "/just/tunes", and anything else as-is + if ext in PICS.split(): + by_category = {"vp": "/just/pics"} + elif ext in VIDS.split(): + by_category = {"vp": "/just/vids"} + elif ext in MUSIC.split(): + by_category = {"vp": "/just/tunes"} + else: + by_category = {} + + # now choose the effect to apply; can be any of these: + # into_subfolder into_toplevel into_sibling by_category + effect = into_subfolder + print(json.dumps({"reloc": effect})) + + +if __name__ == "__main__": + main() diff --git a/copyparty/__main__.py b/copyparty/__main__.py index bdd41baa..917fab50 100644 --- a/copyparty/__main__.py +++ b/copyparty/__main__.py @@ -704,6 +704,11 @@ def get_sects(): \033[36mxban\033[0m can be used to overrule / cancel a user ban event; if the program returns 0 (true/OK) then the ban will NOT happen + effects can be used to redirect uploads into other + locations, and to delete or index other files based + on new uploads, but with certain limitations. See + bin/hooks/reloc* and docs/devnotes.md#hook-effects + except for \033[36mxm\033[0m, only one hook / one action can run at a time, so it's recommended to use the \033[36mf\033[0m flag unless you really need to wait for the hook to finish before continuing (without \033[36mf\033[0m @@ -1132,6 +1137,7 @@ def add_hooks(ap): ap2.add_argument("--xad", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m after a file delete") ap2.add_argument("--xm", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m on message") ap2.add_argument("--xban", metavar="CMD", type=u, action="append", help="execute \033[33mCMD\033[0m if someone gets banned (pw/404/403/url)") + ap2.add_argument("--hook-v", action="store_true", help="verbose hooks") def add_stats(ap): diff --git a/copyparty/authsrv.py b/copyparty/authsrv.py index 4932404b..d73f8a5c 100644 --- a/copyparty/authsrv.py +++ b/copyparty/authsrv.py @@ -521,8 +521,8 @@ def get( t = "{} has no {} in [{}] => [{}] => [{}]" self.log("vfs", t.format(uname, msg, vpath, cvpath, ap), 6) - t = 'you don\'t have %s-access in "/%s"' - raise Pebkac(err, t % (msg, cvpath)) + t = 'you don\'t have %s-access in "/%s" or below "/%s"' + raise Pebkac(err, t % (msg, cvpath, vn.vpath)) return vn, rem @@ -1898,7 +1898,7 @@ def _reload(self) -> None: self.log(t.format(vol.vpath), 1) del vol.flags["lifetime"] - needs_e2d = [x for x in hooks if x != "xm"] + needs_e2d = [x for x in hooks if x in ("xau", "xiu")] drop = [x for x in needs_e2d if vol.flags.get(x)] if drop: t = 'removing [{}] from volume "/{}" because e2d is disabled' diff --git a/copyparty/ftpd.py b/copyparty/ftpd.py index 7137e0ad..3d54a8bd 100644 --- a/copyparty/ftpd.py +++ b/copyparty/ftpd.py @@ -353,7 +353,7 @@ def rename(self, src: str, dst: str) -> None: svp = join(self.cwd, src).lstrip("/") dvp = join(self.cwd, dst).lstrip("/") try: - self.hub.up2k.handle_mv(self.uname, svp, dvp) + self.hub.up2k.handle_mv(self.uname, self.h.cli_ip, svp, dvp) except Exception as ex: raise FSE(str(ex)) @@ -471,6 +471,9 @@ def ftp_STOR(self, file: str, mode: str = "w") -> Any: xbu = vfs.flags.get("xbu") if xbu and not runhook( None, + None, + self.hub.up2k, + "xbu.ftpd", xbu, ap, vp, @@ -480,7 +483,7 @@ def ftp_STOR(self, file: str, mode: str = "w") -> Any: 0, 0, self.cli_ip, - 0, + time.time(), "", ): raise FSE("Upload blocked by xbu server config") diff --git a/copyparty/httpcli.py b/copyparty/httpcli.py index ffd5184e..3232000d 100644 --- a/copyparty/httpcli.py +++ b/copyparty/httpcli.py @@ -73,7 +73,9 @@ humansize, ipnorm, loadpy, + log_reloc, min_ex, + pathmod, quotep, rand_name, read_header, @@ -695,6 +697,9 @@ def cbonk(self, g: Garda, v: str, reason: str, descr: str) -> bool: xban = self.vn.flags.get("xban") if not xban or not runhook( self.log, + self.conn.hsrv.broker, + None, + "xban", xban, self.vn.canonical(self.rem), self.vpath, @@ -1172,7 +1177,8 @@ def handle_propfind(self) -> bool: if self.args.no_dav: raise Pebkac(405, "WebDAV is disabled in server config") - vn, rem = self.asrv.vfs.get(self.vpath, self.uname, False, False, err=401) + vn = self.vn + rem = self.rem tap = vn.canonical(rem) if "davauth" in vn.flags and self.uname == "*": @@ -1556,8 +1562,8 @@ def handle_put(self) -> bool: self.log("PUT %s @%s" % (self.req, self.uname)) if not self.can_write: - t = "user {} does not have write-access here" - raise Pebkac(403, t.format(self.uname)) + t = "user %s does not have write-access under /%s" + raise Pebkac(403, t % (self.uname, self.vn.vpath)) if not self.args.no_dav and self._applesan(): return self.headers.get("content-length") == "0" @@ -1632,6 +1638,9 @@ def handle_post(self) -> bool: if xm: runhook( self.log, + self.conn.hsrv.broker, + None, + "xm", xm, self.vn.canonical(self.rem), self.vpath, @@ -1780,11 +1789,15 @@ def dump_to_file(self, is_put: bool) -> tuple[int, str, str, int, str, str]: if xbu: at = time.time() - lifetime - if not runhook( + vp = vjoin(self.vpath, fn) if nameless else self.vpath + hr = runhook( self.log, + self.conn.hsrv.broker, + None, + "xbu.http.dump", xbu, path, - self.vpath, + vp, self.host, self.uname, self.asrv.vfs.get_perms(self.vpath, self.uname), @@ -1793,10 +1806,25 @@ def dump_to_file(self, is_put: bool) -> tuple[int, str, str, int, str, str]: self.ip, at, "", - ): + ) + if not hr: t = "upload blocked by xbu server config" self.log(t, 1) raise Pebkac(403, t) + if hr.get("reloc"): + x = pathmod(self.asrv.vfs, path, vp, hr["reloc"]) + if x: + if self.args.hook_v: + log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem) + fdir, self.vpath, fn, (vfs, rem) = x + if self.args.nw: + fn = os.devnull + else: + bos.makedirs(fdir) + path = os.path.join(fdir, fn) + if not nameless: + self.vpath = vjoin(self.vpath, fn) + params["fdir"] = fdir if is_put and not (self.args.no_dav or self.args.nw) and bos.path.exists(path): # allow overwrite if... @@ -1871,24 +1899,45 @@ def dump_to_file(self, is_put: bool) -> tuple[int, str, str, int, str, str]: fn = fn2 path = path2 - if xau and not runhook( - self.log, - xau, - path, - self.vpath, - self.host, - self.uname, - self.asrv.vfs.get_perms(self.vpath, self.uname), - mt, - post_sz, - self.ip, - at, - "", - ): - t = "upload blocked by xau server config" - self.log(t, 1) - wunlink(self.log, path, vfs.flags) - raise Pebkac(403, t) + if xau: + vp = vjoin(self.vpath, fn) if nameless else self.vpath + hr = runhook( + self.log, + self.conn.hsrv.broker, + None, + "xau.http.dump", + xau, + path, + vp, + self.host, + self.uname, + self.asrv.vfs.get_perms(self.vpath, self.uname), + mt, + post_sz, + self.ip, + at, + "", + ) + if not hr: + t = "upload blocked by xau server config" + self.log(t, 1) + wunlink(self.log, path, vfs.flags) + raise Pebkac(403, t) + if hr.get("reloc"): + x = pathmod(self.asrv.vfs, path, vp, hr["reloc"]) + if x: + if self.args.hook_v: + log_reloc(self.log, hr["reloc"], x, path, vp, fn, vfs, rem) + fdir, self.vpath, fn, (vfs, rem) = x + bos.makedirs(fdir) + path2 = os.path.join(fdir, fn) + atomic_move(self.log, path, path2, vfs.flags) + path = path2 + if not nameless: + self.vpath = vjoin(self.vpath, fn) + sz = bos.path.getsize(path) + else: + sz = post_sz vfs, rem = vfs.get_dbv(rem) self.conn.hsrv.broker.say( @@ -1911,7 +1960,7 @@ def dump_to_file(self, is_put: bool) -> tuple[int, str, str, int, str, str]: alg, self.args.fk_salt, path, - post_sz, + sz, 0 if ANYWIN else bos.stat(path).st_ino, )[: vfs.flags["fk"]] @@ -2536,18 +2585,15 @@ def handle_plain_upload( fname = sanitize_fn( p_file or "", "", [".prologue.html", ".epilogue.html"] ) + abspath = os.path.join(fdir, fname) + suffix = "-%.6f-%s" % (time.time(), dip) if p_file and not nullwrite: if rnd: fname = rand_name(fdir, fname, rnd) - if not bos.path.isdir(fdir): - raise Pebkac(404, "that folder does not exist") - - suffix = "-{:.6f}-{}".format(time.time(), dip) open_args = {"fdir": fdir, "suffix": suffix} if "replace" in self.uparam: - abspath = os.path.join(fdir, fname) if not self.can_delete: self.log("user not allowed to overwrite with ?replace") elif bos.path.exists(abspath): @@ -2557,16 +2603,6 @@ def handle_plain_upload( except: t = "toctou while deleting for ?replace: %s" self.log(t % (abspath,)) - - # reserve destination filename - with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw: - fname = zfw["orz"][1] - - tnam = fname + ".PARTIAL" - if self.args.dotpart: - tnam = "." + tnam - - abspath = os.path.join(fdir, fname) else: open_args = {} tnam = fname = os.devnull @@ -2574,23 +2610,65 @@ def handle_plain_upload( if xbu: at = time.time() - lifetime - if not runhook( + hr = runhook( self.log, + self.conn.hsrv.broker, + None, + "xbu.http.bup", xbu, abspath, - self.vpath, + vjoin(upload_vpath, fname), self.host, self.uname, - self.asrv.vfs.get_perms(self.vpath, self.uname), + self.asrv.vfs.get_perms(upload_vpath, self.uname), at, 0, self.ip, at, "", - ): + ) + if not hr: t = "upload blocked by xbu server config" self.log(t, 1) raise Pebkac(403, t) + if hr.get("reloc"): + zs = vjoin(upload_vpath, fname) + x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"]) + if x: + if self.args.hook_v: + log_reloc( + self.log, + hr["reloc"], + x, + abspath, + zs, + fname, + vfs, + rem, + ) + fdir, upload_vpath, fname, (vfs, rem) = x + abspath = os.path.join(fdir, fname) + if nullwrite: + fdir = abspath = "" + else: + open_args["fdir"] = fdir + + if p_file and not nullwrite: + bos.makedirs(fdir) + + # reserve destination filename + with ren_open(fname, "wb", fdir=fdir, suffix=suffix) as zfw: + fname = zfw["orz"][1] + + tnam = fname + ".PARTIAL" + if self.args.dotpart: + tnam = "." + tnam + + abspath = os.path.join(fdir, fname) + else: + open_args = {} + tnam = fname = os.devnull + fdir = abspath = "" if lim: lim.chk_bup(self.ip) @@ -2634,29 +2712,58 @@ def handle_plain_upload( tabspath = "" + at = time.time() - lifetime + if xau: + hr = runhook( + self.log, + self.conn.hsrv.broker, + None, + "xau.http.bup", + xau, + abspath, + vjoin(upload_vpath, fname), + self.host, + self.uname, + self.asrv.vfs.get_perms(upload_vpath, self.uname), + at, + sz, + self.ip, + at, + "", + ) + if not hr: + t = "upload blocked by xau server config" + self.log(t, 1) + wunlink(self.log, abspath, vfs.flags) + raise Pebkac(403, t) + if hr.get("reloc"): + zs = vjoin(upload_vpath, fname) + x = pathmod(self.asrv.vfs, abspath, zs, hr["reloc"]) + if x: + if self.args.hook_v: + log_reloc( + self.log, + hr["reloc"], + x, + abspath, + zs, + fname, + vfs, + rem, + ) + fdir, upload_vpath, fname, (vfs, rem) = x + ap2 = os.path.join(fdir, fname) + if nullwrite: + fdir = ap2 = "" + else: + bos.makedirs(fdir) + atomic_move(self.log, abspath, ap2, vfs.flags) + abspath = ap2 + sz = bos.path.getsize(abspath) + files.append( (sz, sha_hex, sha_b64, p_file or "(discarded)", fname, abspath) ) - at = time.time() - lifetime - if xau and not runhook( - self.log, - xau, - abspath, - self.vpath, - self.host, - self.uname, - self.asrv.vfs.get_perms(self.vpath, self.uname), - at, - sz, - self.ip, - at, - "", - ): - t = "upload blocked by xau server config" - self.log(t, 1) - wunlink(self.log, abspath, vfs.flags) - raise Pebkac(403, t) - dbv, vrem = vfs.get_dbv(rem) self.conn.hsrv.broker.say( "up2k.hash_file", @@ -2712,13 +2819,14 @@ def handle_plain_upload( for sz, sha_hex, sha_b64, ofn, lfn, ap in files: vsuf = "" if (self.can_read or self.can_upget) and "fk" in vfs.flags: + st = bos.stat(ap) alg = 2 if "fka" in vfs.flags else 1 vsuf = "?k=" + self.gen_fk( alg, self.args.fk_salt, ap, - sz, - 0 if ANYWIN or not ap else bos.stat(ap).st_ino, + st.st_size, + 0 if ANYWIN or not ap else st.st_ino, )[: vfs.flags["fk"]] if "media" in self.uparam or "medialinks" in vfs.flags: @@ -2885,6 +2993,9 @@ def handle_text_upload(self) -> bool: if xbu: if not runhook( self.log, + self.conn.hsrv.broker, + None, + "xbu.http.txt", xbu, fp, self.vpath, @@ -2924,6 +3035,9 @@ def handle_text_upload(self) -> bool: xau = vfs.flags.get("xau") if xau and not runhook( self.log, + self.conn.hsrv.broker, + None, + "xau.http.txt", xau, fp, self.vpath, @@ -4156,7 +4270,7 @@ def _mv(self, vsrc: str, vdst: str) -> bool: if self.args.no_mv: raise Pebkac(403, "the rename/move feature is disabled in server config") - x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, vsrc, vdst) + x = self.conn.hsrv.broker.ask("up2k.handle_mv", self.uname, self.ip, vsrc, vdst) self.loud_reply(x.get(), status=201) return True diff --git a/copyparty/smbd.py b/copyparty/smbd.py index a5f6681b..25873a7e 100644 --- a/copyparty/smbd.py +++ b/copyparty/smbd.py @@ -187,6 +187,8 @@ def _v2a( debug('%s("%s", %s) %s @%s\033[K\033[0m', caller, vpath, str(a), perms, uname) vfs, rem = self.asrv.vfs.get(vpath, uname, *perms) + if not vfs.realpath: + raise Exception("unmapped vfs") return vfs, vfs.canonical(rem) def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]: @@ -195,6 +197,8 @@ def _listdir(self, vpath: str, *a: Any, **ka: Any) -> list[str]: uname = self._uname() # debug('listdir("%s", %s) @%s\033[K\033[0m', vpath, str(a), uname) vfs, rem = self.asrv.vfs.get(vpath, uname, False, False) + if not vfs.realpath: + raise Exception("unmapped vfs") _, vfs_ls, vfs_virt = vfs.ls( rem, uname, not self.args.no_scandir, [[False, False]] ) @@ -240,7 +244,21 @@ def _open( xbu = vfs.flags.get("xbu") if xbu and not runhook( - self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "1.7.6.2", 0, "" + self.nlog, + None, + self.hub.up2k, + "xbu.smb", + xbu, + ap, + vpath, + "", + "", + "", + 0, + 0, + "1.7.6.2", + time.time(), + "", ): yeet("blocked by xbu server config: " + vpath) @@ -297,7 +315,7 @@ def _rename(self, vp1: str, vp2: str) -> None: t = "blocked rename (no-move-acc %s): /%s @%s" yeet(t % (vfs1.axs.umove, vp1, uname)) - self.hub.up2k.handle_mv(uname, vp1, vp2) + self.hub.up2k.handle_mv(uname, "1.7.6.2", vp1, vp2) try: bos.makedirs(ap2) except: diff --git a/copyparty/tftpd.py b/copyparty/tftpd.py index 1da648e5..91ebb3ae 100644 --- a/copyparty/tftpd.py +++ b/copyparty/tftpd.py @@ -244,6 +244,8 @@ def _v2a(self, caller: str, vpath: str, perms: list, *a: Any) -> tuple[VFS, str] debug('%s("%s", %s) %s\033[K\033[0m', caller, vpath, str(a), perms) vfs, rem = self.asrv.vfs.get(vpath, "*", *perms) + if not vfs.realpath: + raise Exception("unmapped vfs") return vfs, vfs.canonical(rem) def _ls(self, vpath: str, raddress: str, rport: int, force=False) -> Any: @@ -331,7 +333,21 @@ def _open(self, vpath: str, mode: str, *a: Any, **ka: Any) -> Any: xbu = vfs.flags.get("xbu") if xbu and not runhook( - self.nlog, xbu, ap, vpath, "", "", "", 0, 0, "8.3.8.7", 0, "" + self.nlog, + None, + self.hub.up2k, + "xbu.tftpd", + xbu, + ap, + vpath, + "", + "", + "", + 0, + 0, + "8.3.8.7", + time.time(), + "", ): yeet("blocked by xbu server config: " + vpath) @@ -339,7 +355,7 @@ def _open(self, vpath: str, mode: str, *a: Any, **ka: Any) -> Any: return self._ls(vpath, "", 0, True) if not a: - a = [self.args.iobuf] + a = (self.args.iobuf,) return open(ap, mode, *a, **ka) diff --git a/copyparty/up2k.py b/copyparty/up2k.py index 3aad2f00..359c8509 100644 --- a/copyparty/up2k.py +++ b/copyparty/up2k.py @@ -46,6 +46,7 @@ hidedir, humansize, min_ex, + pathmod, quotep, rand_name, ren_open, @@ -165,6 +166,7 @@ def __init__(self, hub: "SvcHub") -> None: self.xiu_ptn = re.compile(r"(?:^|,)i([0-9]+)") self.xiu_busy = False # currently running hook self.xiu_asleep = True # needs rescan_cond poke to schedule self + self.fx_backlog: list[tuple[str, dict[str, str], str]] = [] self.cur: dict[str, "sqlite3.Cursor"] = {} self.mem_cur = None @@ -2544,7 +2546,7 @@ def handle_json( if self.mutex.acquire(timeout=10): got_lock = True with self.reg_mutex: - return self._handle_json(cj) + ret = self._handle_json(cj) else: t = "cannot receive uploads right now;\nserver busy with {}.\nPlease wait; the client will retry..." raise Pebkac(503, t.format(self.blocked or "[unknown]")) @@ -2552,11 +2554,16 @@ def handle_json( if not PY2: raise with self.mutex, self.reg_mutex: - return self._handle_json(cj) + ret = self._handle_json(cj) finally: if got_lock: self.mutex.release() + if self.fx_backlog: + self.do_fx_backlog() + + return ret + def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]: ptop = cj["ptop"] if not self.register_vpath(ptop, cj["vcfg"]): @@ -2758,28 +2765,43 @@ def _handle_json(self, cj: dict[str, Any]) -> dict[str, Any]: job["name"] = rand_name( pdir, cj["name"], vfs.flags["nrand"] ) - else: - job["name"] = self._untaken(pdir, cj, now) dst = djoin(job["ptop"], job["prel"], job["name"]) xbu = vfs.flags.get("xbu") - if xbu and not runhook( - self.log, - xbu, # type: ignore - dst, - job["vtop"], - job["host"], - job["user"], - self.asrv.vfs.get_perms(job["vtop"], job["user"]), - job["lmod"], - job["size"], - job["addr"], - job["at"], - "", - ): - t = "upload blocked by xbu server config: {}".format(dst) - self.log(t, 1) - raise Pebkac(403, t) + if xbu: + vp = djoin(job["vtop"], job["prel"], job["name"]) + hr = runhook( + self.log, + None, + self, + "xbu.up2k.dupe", + xbu, # type: ignore + dst, + vp, + job["host"], + job["user"], + self.asrv.vfs.get_perms(job["vtop"], job["user"]), + job["lmod"], + job["size"], + job["addr"], + job["at"], + "", + ) + if not hr: + t = "upload blocked by xbu server config: %s" % (dst,) + self.log(t, 1) + raise Pebkac(403, t) + if hr.get("reloc"): + x = pathmod(self.asrv.vfs, dst, vp, hr["reloc"]) + if x: + pdir, _, job["name"], (vfs, rem) = x + dst = os.path.join(pdir, job["name"]) + job["ptop"] = vfs.realpath + job["vtop"] = vfs.vpath + job["prel"] = rem + bos.makedirs(pdir) + + job["name"] = self._untaken(pdir, cj, now) if not self.args.nw: dvf: dict[str, Any] = vfs.flags @@ -3142,6 +3164,9 @@ def finish_upload(self, ptop: str, wark: str, busy_aps: dict[str, int]) -> None: with self.mutex, self.reg_mutex: self._finish_upload(ptop, wark) + if self.fx_backlog: + self.do_fx_backlog() + def _finish_upload(self, ptop: str, wark: str) -> None: """mutex(main,reg) me""" try: @@ -3335,25 +3360,30 @@ def db_add( xau = False if skip_xau else vflags.get("xau") dst = djoin(ptop, rd, fn) - if xau and not runhook( - self.log, - xau, - dst, - djoin(vtop, rd, fn), - host, - usr, - self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr), - int(ts), - sz, - ip, - at or time.time(), - "", - ): - t = "upload blocked by xau server config" - self.log(t, 1) - wunlink(self.log, dst, vflags) - self.registry[ptop].pop(wark, None) - raise Pebkac(403, t) + if xau: + hr = runhook( + self.log, + None, + self, + "xau.up2k", + xau, + dst, + djoin(vtop, rd, fn), + host, + usr, + self.asrv.vfs.get_perms(djoin(vtop, rd, fn), usr), + ts, + sz, + ip, + at or time.time(), + "", + ) + if not hr: + t = "upload blocked by xau server config" + self.log(t, 1) + wunlink(self.log, dst, vflags) + self.registry[ptop].pop(wark, None) + raise Pebkac(403, t) xiu = vflags.get("xiu") if xiu: @@ -3537,6 +3567,9 @@ def _handle_rm( if xbd: if not runhook( self.log, + None, + self, + "xbd", xbd, abspath, vpath, @@ -3546,7 +3579,7 @@ def _handle_rm( stl.st_mtime, st.st_size, ip, - 0, + time.time(), "", ): t = "delete blocked by xbd server config: {}" @@ -3571,6 +3604,9 @@ def _handle_rm( if xad: runhook( self.log, + None, + self, + "xad", xad, abspath, vpath, @@ -3580,7 +3616,7 @@ def _handle_rm( stl.st_mtime, st.st_size, ip, - 0, + time.time(), "", ) @@ -3596,7 +3632,7 @@ def _handle_rm( return n_files, ok + ok2, ng + ng2 - def handle_mv(self, uname: str, svp: str, dvp: str) -> str: + def handle_mv(self, uname: str, ip: str, svp: str, dvp: str) -> str: if svp == dvp or dvp.startswith(svp + "/"): raise Pebkac(400, "mv: cannot move parent into subfolder") @@ -3613,7 +3649,7 @@ def handle_mv(self, uname: str, svp: str, dvp: str) -> str: if stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): with self.mutex: try: - ret = self._mv_file(uname, svp, dvp, curs) + ret = self._mv_file(uname, ip, svp, dvp, curs) finally: for v in curs: v.connection.commit() @@ -3646,7 +3682,7 @@ def handle_mv(self, uname: str, svp: str, dvp: str) -> str: raise Pebkac(500, "mv: bug at {}, top {}".format(svpf, svp)) dvpf = dvp + svpf[len(svp) :] - self._mv_file(uname, svpf, dvpf, curs) + self._mv_file(uname, ip, svpf, dvpf, curs) finally: for v in curs: v.connection.commit() @@ -3671,7 +3707,7 @@ def handle_mv(self, uname: str, svp: str, dvp: str) -> str: return "k" def _mv_file( - self, uname: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"] + self, uname: str, ip: str, svp: str, dvp: str, curs: set["sqlite3.Cursor"] ) -> str: """mutex(main) me; will mutex(reg)""" svn, srem = self.asrv.vfs.get(svp, uname, True, False, True) @@ -3705,21 +3741,27 @@ def _mv_file( except: pass # broken symlink; keep as-is + ftime = stl.st_mtime + fsize = st.st_size + xbr = svn.flags.get("xbr") xar = dvn.flags.get("xar") if xbr: if not runhook( self.log, + None, + self, + "xbr", xbr, sabs, svp, "", uname, self.asrv.vfs.get_perms(svp, uname), - stl.st_mtime, - st.st_size, - "", - 0, + ftime, + fsize, + ip, + time.time(), "", ): t = "move blocked by xbr server config: {}".format(svp) @@ -3747,16 +3789,19 @@ def _mv_file( if xar: runhook( self.log, + None, + self, + "xar.ln", xar, dabs, dvp, "", uname, self.asrv.vfs.get_perms(dvp, uname), - 0, - 0, - "", - 0, + ftime, + fsize, + ip, + time.time(), "", ) @@ -3765,13 +3810,6 @@ def _mv_file( c1, w, ftime_, fsize_, ip, at = self._find_from_vpath(svn.realpath, srem) c2 = self.cur.get(dvn.realpath) - if ftime_ is None: - ftime = stl.st_mtime - fsize = st.st_size - else: - ftime = ftime_ - fsize = fsize_ or 0 - has_dupes = False if w: assert c1 @@ -3779,7 +3817,9 @@ def _mv_file( self._copy_tags(c1, c2, w) with self.reg_mutex: - has_dupes = self._forget_file(svn.realpath, srem, c1, w, is_xvol, fsize) + has_dupes = self._forget_file( + svn.realpath, srem, c1, w, is_xvol, fsize_ or fsize + ) if not is_xvol: has_dupes = self._relink(w, svn.realpath, srem, dabs) @@ -3849,7 +3889,7 @@ def _mv_file( if is_link: try: - times = (int(time.time()), int(stl.st_mtime)) + times = (int(time.time()), int(ftime)) bos.utime(dabs, times, False) except: pass @@ -3859,16 +3899,19 @@ def _mv_file( if xar: runhook( self.log, + None, + self, + "xar.mv", xar, dabs, dvp, "", uname, self.asrv.vfs.get_perms(dvp, uname), - 0, - 0, - "", - 0, + ftime, + fsize, + ip, + time.time(), "", ) @@ -4152,23 +4195,35 @@ def _new_upload(self, job: dict[str, Any]) -> None: xbu = self.flags[job["ptop"]].get("xbu") ap_chk = djoin(pdir, job["name"]) vp_chk = djoin(job["vtop"], job["prel"], job["name"]) - if xbu and not runhook( - self.log, - xbu, - ap_chk, - vp_chk, - job["host"], - job["user"], - self.asrv.vfs.get_perms(vp_chk, job["user"]), - int(job["lmod"]), - job["size"], - job["addr"], - int(job["t0"]), - "", - ): - t = "upload blocked by xbu server config: {}".format(vp_chk) - self.log(t, 1) - raise Pebkac(403, t) + if xbu: + hr = runhook( + self.log, + None, + self, + "xbu.up2k", + xbu, + ap_chk, + vp_chk, + job["host"], + job["user"], + self.asrv.vfs.get_perms(vp_chk, job["user"]), + job["lmod"], + job["size"], + job["addr"], + job["t0"], + "", + ) + if not hr: + t = "upload blocked by xbu server config: {}".format(vp_chk) + self.log(t, 1) + raise Pebkac(403, t) + if hr.get("reloc"): + x = pathmod(self.asrv.vfs, ap_chk, vp_chk, hr["reloc"]) + if x: + pdir, _, job["name"], (vfs, rem) = x + job["ptop"] = vfs.realpath + job["vtop"] = vfs.vpath + job["prel"] = rem tnam = job["name"] + ".PARTIAL" if self.args.dotpart: @@ -4442,6 +4497,9 @@ def _hash_t( with self.rescan_cond: self.rescan_cond.notify_all() + if self.fx_backlog: + self.do_fx_backlog() + return True def hash_file( @@ -4473,6 +4531,48 @@ def hash_file( self.hashq.put(zt) self.n_hashq += 1 + def do_fx_backlog(self): + with self.mutex, self.reg_mutex: + todo = self.fx_backlog + self.fx_backlog = [] + for act, hr, req_vp in todo: + self.hook_fx(act, hr, req_vp) + + def hook_fx(self, act: str, hr: dict[str, str], req_vp: str) -> None: + bad = [k for k in hr if k != "vp"] + if bad: + t = "got unsupported key in %s from hook: %s" + raise Exception(t % (act, bad)) + + for fvp in hr.get("vp") or []: + # expect vpath including filename; either absolute + # or relative to the client's vpath (request url) + if fvp.startswith("/"): + fvp, fn = vsplit(fvp[1:]) + fvp = "/" + fvp + else: + fvp, fn = vsplit(fvp) + + x = pathmod(self.asrv.vfs, "", req_vp, {"vp": fvp, "fn": fn}) + if not x: + t = "hook_fx(%s): failed to resolve %s based on %s" + self.log(t % (act, fvp, req_vp)) + continue + + ap, rd, fn, (vn, rem) = x + vp = vjoin(rd, fn) + if not vp: + raise Exception("hook_fx: blank vp from pathmod") + + if act == "idx": + rd = rd[len(vn.vpath) :].strip("/") + self.hash_file( + vn.realpath, vn.vpath, vn.flags, rd, fn, "", time.time(), "", True + ) + + if act == "del": + self._handle_rm(LEELOO_DALLAS, "", vp, [], False, False) + def shutdown(self) -> None: self.stop = True diff --git a/copyparty/util.py b/copyparty/util.py index 1ea3cd03..0f879367 100644 --- a/copyparty/util.py +++ b/copyparty/util.py @@ -146,6 +146,8 @@ def __call__(self, msg: str, c: Union[int, str] = 0) -> None: import magic from .authsrv import VFS + from .broker_util import BrokerCli + from .up2k import Up2k FAKE_MP = False @@ -2117,6 +2119,72 @@ def ujoin(rd: str, fn: str) -> str: return rd or fn +def log_reloc( + log: "NamedLogger", + re: dict[str, str], + pm: tuple[str, str, str, tuple["VFS", str]], + ap: str, + vp: str, + fn: str, + vn: "VFS", + rem: str, +) -> None: + nap, nvp, nfn, (nvn, nrem) = pm + t = "reloc %s:\nold ap [%s]\nnew ap [%s\033[36m/%s\033[0m]\nold vp [%s]\nnew vp [%s\033[36m/%s\033[0m]\nold fn [%s]\nnew fn [%s]\nold vfs [%s]\nnew vfs [%s]\nold rem [%s]\nnew rem [%s]" + log(t % (re, ap, nap, nfn, vp, nvp, nfn, fn, nfn, vn.vpath, nvn.vpath, rem, nrem)) + + +def pathmod( + vfs: "VFS", ap: str, vp: str, mod: dict[str, str] +) -> Optional[tuple[str, str, str, tuple["VFS", str]]]: + # vfs: authsrv.vfs + # ap: original abspath to a file + # vp: original urlpath to a file + # mod: modification (ap/vp/fn) + + nvp = "\n" # new vpath + ap = os.path.dirname(ap) + vp, fn = vsplit(vp) + if mod.get("fn"): + fn = mod["fn"] + nvp = vp + + for ref, k in ((ap, "ap"), (vp, "vp")): + if k not in mod: + continue + + ms = mod[k].replace(os.sep, "/") + if ms.startswith("/"): + np = ms + elif k == "vp": + np = undot(vjoin(ref, ms)) + else: + np = os.path.abspath(os.path.join(ref, ms)) + + if k == "vp": + nvp = np.lstrip("/") + continue + + # try to map abspath to vpath + np = np.replace("/", os.sep) + for vn_ap, vn in vfs.all_aps: + if not np.startswith(vn_ap): + continue + zs = np[len(vn_ap) :].replace(os.sep, "/") + nvp = vjoin(vn.vpath, zs) + break + + if nvp == "\n": + return None + + vn, rem = vfs.get(nvp, "*", False, False) + if not vn.realpath: + raise Exception("unmapped vfs") + + ap = vn.canonical(rem) + return ap, nvp, fn, (vn, rem) + + def _w8dec2(txt: bytes) -> str: """decodes filesystem-bytes to wtf8""" return surrogateescape.decodefilename(txt) @@ -3130,6 +3198,7 @@ def runihook( def _runhook( log: Optional["NamedLogger"], + src: str, cmd: str, ap: str, vp: str, @@ -3141,14 +3210,16 @@ def _runhook( ip: str, at: float, txt: str, -) -> bool: +) -> dict[str, Any]: + ret = {"rc": 0} areq, chk, fork, jtxt, wait, sp_ka, acmd = _parsehook(log, cmd) if areq: for ch in areq: if ch not in perms: t = "user %s not allowed to run hook %s; need perms %s, have %s" - log(t % (uname, cmd, areq, perms)) - return True # fallthrough to next hook + if log: + log(t % (uname, cmd, areq, perms)) + return ret # fallthrough to next hook if jtxt: ja = { "ap": ap, @@ -3160,6 +3231,7 @@ def _runhook( "host": host, "user": uname, "perms": perms, + "src": src, "txt": txt, } arg = json.dumps(ja) @@ -3178,18 +3250,34 @@ def _runhook( else: rc, v, err = runcmd(bcmd, **sp_ka) # type: ignore if chk and rc: + ret["rc"] = rc retchk(rc, bcmd, err, log, 5) - return False + else: + try: + ret = json.loads(v) + except: + ret = {} + + try: + if "stdout" not in ret: + ret["stdout"] = v + if "rc" not in ret: + ret["rc"] = rc + except: + ret = {"rc": rc, "stdout": v} wait -= time.time() - t0 if wait > 0: time.sleep(wait) - return True + return ret def runhook( log: Optional["NamedLogger"], + broker: Optional["BrokerCli"], + up2k: Optional["Up2k"], + src: str, cmds: list[str], ap: str, vp: str, @@ -3201,19 +3289,42 @@ def runhook( ip: str, at: float, txt: str, -) -> bool: +) -> dict[str, Any]: + assert broker or up2k + asrv = (broker or up2k).asrv + args = (broker or up2k).args vp = vp.replace("\\", "/") + ret = {"rc": 0} for cmd in cmds: try: - if not _runhook(log, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt): - return False + hr = _runhook( + log, src, cmd, ap, vp, host, uname, perms, mt, sz, ip, at, txt + ) + if log and args.hook_v: + log("hook(%s) [%s] => \033[32m%s" % (src, cmd, hr), 6) + if not hr: + return {} + for k, v in hr.items(): + if k in ("idx", "del") and v: + if broker: + broker.say("up2k.hook_fx", k, v, vp) + else: + up2k.fx_backlog.append((k, v, vp)) + elif k == "reloc" and v: + # idk, just take the last one ig + ret["reloc"] = v + elif k in ret: + if k == "rc" and v: + ret[k] = v + else: + ret[k] = v except Exception as ex: (log or print)("hook: {}".format(ex)) if ",c," in "," + cmd: - return False + return {} break - return True + return ret def loadpy(ap: str, hot: bool) -> Any: diff --git a/docs/devnotes.md b/docs/devnotes.md index 1bb99bed..5130f187 100644 --- a/docs/devnotes.md +++ b/docs/devnotes.md @@ -12,6 +12,8 @@ * [write](#write) * [admin](#admin) * [general](#general) +* [event hooks](#event-hooks) - on writing your own [hooks](../README.md#event-hooks) + * [hook effects](#hook-effects) - hooks can cause intentional side-effects * [assumptions](#assumptions) * [mdns](#mdns) * [sfx repack](#sfx-repack) - reduce the size of an sfx by removing features @@ -204,6 +206,32 @@ upload modifiers: | GET | `?pw=x` | logout | +# event hooks + +on writing your own [hooks](../README.md#event-hooks) + +## hook effects + +hooks can cause intentional side-effects, such as redirecting an upload into another location, or creating+indexing additional files, or deleting existing files, by returning json on stdout + +* `reloc` can redirect uploads before/after uploading has finished, based on filename, extension, file contents, uploader ip/name etc. +* `idx` informs copyparty about a new file to index as a consequence of this upload +* `del` tells copyparty to delete an unrelated file by vpath + +for these to take effect, the hook must be defined with the `c1` flag; see example [reloc-by-ext](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reloc-by-ext.py) + +a subset of effect types are available for a subset of hook types, + +* most hook types (xbu/xau/xbr/xar/xbd/xad/xm) support `idx` and `del` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb +* most hook types will abort/reject the action if the hook returns nonzero, assuming flag `c` is given, see examples [reject-extension](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-extension.py) and [reject-mimetype](https://github.com/9001/copyparty/blob/hovudstraum/bin/hooks/reject-mimetype.py) +* `xbu` supports `reloc` for all http protocols (up2k / basic-uploader / webdav), but not ftp/tftp/smb +* `xau` supports `reloc` for basic-uploader / webdav only, not up2k or ftp/tftp/smb + * so clients like sharex are supported, but not dragdrop into browser + +to trigger indexing of files `/foo/1.txt` and `/foo/bar/2.txt`, a hook can `print(json.dumps({"idx":{"vp":["/foo/1.txt","/foo/bar/2.txt"]}}))` (and replace "idx" with "del" to delete instead) +* note: paths starting with `/` are absolute URLs, but you can also do `../3.txt` relative to the destination folder of each uploaded file + + # assumptions ## mdns diff --git a/docs/versus.md b/docs/versus.md index 3d7e561a..6a71cf46 100644 --- a/docs/versus.md +++ b/docs/versus.md @@ -175,6 +175,7 @@ symbol legend, | ┗ randomize filename | █ | | | | | | | █ | █ | | | | | | ┗ mimetype reject-list | ╱ | | | | | | | | • | ╱ | | ╱ | • | | ┗ extension reject-list | ╱ | | | | | | | █ | • | ╱ | | ╱ | • | +| ┗ upload routing | █ | | | | | | | | | | | | | | checksums provided | | | | █ | █ | | | | █ | ╱ | | | | | cloud storage backend | ╱ | ╱ | ╱ | █ | █ | █ | ╱ | | | ╱ | █ | █ | ╱ | @@ -188,6 +189,9 @@ symbol legend, * `race the beam` = files can be downloaded while they're still uploading; downloaders are slowed down such that the uploader is always ahead +* `upload routing` = depending on filetype / contents / uploader etc., the file can be redirected to another location or otherwise transformed; mitigates limitations such as [sharex#3992](https://github.com/ShareX/ShareX/issues/3992) + * copyparty example: [reloc-by-ext](https://github.com/9001/copyparty/tree/hovudstraum/bin/hooks#before-upload) + * `checksums provided` = when downloading a file from the server, the file's checksum is provided for verification client-side * `cloud storage backend` = able to serve files from (and write to) s3 or similar cloud services; `╱` means the software can do this with some help from `rclone mount` as a bridge diff --git a/tests/test_hooks.py b/tests/test_hooks.py new file mode 100644 index 00000000..93de60c2 --- /dev/null +++ b/tests/test_hooks.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# coding: utf-8 +from __future__ import print_function, unicode_literals + +import os +import shutil +import tempfile +import unittest + +from copyparty.authsrv import AuthSrv +from copyparty.httpcli import HttpCli +from tests import util as tu +from tests.util import Cfg + + +def hdr(query): + h = "GET /{} HTTP/1.1\r\nPW: o\r\nConnection: close\r\n\r\n" + return h.format(query).encode("utf-8") + + +class TestHooks(unittest.TestCase): + def setUp(self): + self.td = tu.get_ramdisk() + + def tearDown(self): + os.chdir(tempfile.gettempdir()) + shutil.rmtree(self.td) + + def reset(self): + td = os.path.join(self.td, "vfs") + if os.path.exists(td): + shutil.rmtree(td) + os.mkdir(td) + os.chdir(td) + return td + + def test(self): + vcfg = ["a/b/c/d:c/d:A", "a:a:r"] + + scenarios = ( + ('{"vp":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"), + ('{"vp":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"), + ('{"vp":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"), + ('{"ap":"x/y"}', "c/d/a.png", "c/d/x/y/a.png"), + ('{"ap":"x/y"}', "c/d/e/a.png", "c/d/e/x/y/a.png"), + ('{"ap":"../x/y"}', "c/d/e/a.png", "c/d/x/y/a.png"), + ('{"ap":"../x/y"}', "c/d/a.png", "a/b/c/x/y/a.png"), + ('{"fn":"b.png"}', "c/d/a.png", "c/d/b.png"), + ('{"vp":"x","fn":"b.png"}', "c/d/a.png", "c/d/x/b.png"), + ) + + for x in scenarios: + print("\n\n\n", x) + hooktxt, url_up, url_dl = x + for hooktype in ("xbu", "xau"): + for upfun in (self.put, self.bup): + self.reset() + self.makehook("""print('{"reloc":%s}')""" % (hooktxt,)) + ka = {hooktype: ["j,c1,h.py"]} + self.args = Cfg(v=vcfg, a=["o:o"], e2d=True, **ka) + self.asrv = AuthSrv(self.args, self.log) + + h, b = upfun(url_up) + self.assertIn("201 Created", h) + h, b = self.curl(url_dl) + self.assertEqual(b, "ok %s\n" % (url_up)) + + def makehook(self, hs): + with open("h.py", "wb") as f: + f.write(hs.encode("utf-8")) + + def put(self, url): + buf = "PUT /{0} HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Length: {1}\r\n\r\nok {0}\n" + buf = buf.format(url, len(url) + 4).encode("utf-8") + print("PUT -->", buf) + conn = tu.VHttpConn(self.args, self.asrv, self.log, buf) + HttpCli(conn).run() + ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) + print("PUT <--", ret) + return ret + + def bup(self, url): + hdr = "POST /%s HTTP/1.1\r\nPW: o\r\nConnection: close\r\nContent-Type: multipart/form-data; boundary=XD\r\nContent-Length: %d\r\n\r\n" + bdy = '--XD\r\nContent-Disposition: form-data; name="act"\r\n\r\nbput\r\n--XD\r\nContent-Disposition: form-data; name="f"; filename="%s"\r\n\r\n' + ftr = "\r\n--XD--\r\n" + try: + url, fn = url.rsplit("/", 1) + except: + fn = url + url = "" + + buf = (bdy % (fn,) + "ok %s/%s\n" % (url, fn) + ftr).encode("utf-8") + buf = (hdr % (url, len(buf))).encode("utf-8") + buf + print("PoST -->", buf) + conn = tu.VHttpConn(self.args, self.asrv, self.log, buf) + HttpCli(conn).run() + ret = conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) + print("POST <--", ret) + return ret + + def curl(self, url, binary=False): + conn = tu.VHttpConn(self.args, self.asrv, self.log, hdr(url)) + HttpCli(conn).run() + if binary: + h, b = conn.s._reply.split(b"\r\n\r\n", 1) + return [h.decode("utf-8"), b] + + return conn.s._reply.decode("utf-8").split("\r\n\r\n", 1) + + def log(self, src, msg, c=0): + print(msg) diff --git a/tests/util.py b/tests/util.py index 7b87a930..e7768ad5 100644 --- a/tests/util.py +++ b/tests/util.py @@ -68,6 +68,13 @@ def chkcmd(argv): def get_ramdisk(): def subdir(top): + for d in os.listdir(top): + if not d.startswith("cptd-"): + continue + p = os.path.join(top, d) + st = os.stat(p) + if time.time() - st.st_mtime > 300: + shutil.rmtree(p) ret = os.path.join(top, "cptd-{}".format(os.getpid())) shutil.rmtree(ret, True) os.mkdir(ret) @@ -111,10 +118,10 @@ class Cfg(Namespace): def __init__(self, a=None, v=None, c=None, **ka0): ka = {} - ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver xdev xlink xvol" + ex = "daw dav_auth dav_inf dav_mac dav_rt e2d e2ds e2dsa e2t e2ts e2tsr e2v e2vu e2vp early_ban ed emp exp force_js getmod grid gsel hardlink ih ihead magic never_symlink nid nih no_acode no_athumb no_dav no_dedup no_del no_dupe no_lifetime no_logues no_mv no_pipe no_poll no_readme no_robots no_sb_md no_sb_lg no_scandir no_tarcmp no_thumb no_vthumb no_zip nrand nw og og_no_head og_s_title q rand smb srch_dbg stats uqe vague_403 vc ver write_uplog xdev xlink xvol" ka.update(**{k: False for k in ex.split()}) - ex = "dotpart dotsrch no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip" + ex = "dotpart dotsrch hook_v no_dhash no_fastboot no_rescan no_sendfile no_snap no_voldump re_dhash plain_ip" ka.update(**{k: True for k in ex.split()}) ex = "ah_cli ah_gen css_browser hist js_browser mime mimes no_forget no_hash no_idx nonsus_urls og_tpl og_ua" @@ -178,6 +185,10 @@ def __init__(self, a=None, v=None, c=None, **ka0): class NullBroker(object): + def __init__(self, args, asrv): + self.args = args + self.asrv = asrv + def say(self, *args): pass @@ -213,7 +224,7 @@ def __init__(self, args, asrv, log): self.asrv = asrv self.log = log - self.broker = NullBroker() + self.broker = NullBroker(args, asrv) self.prism = None self.bans = {} self.nreq = 0