Skip to content

Commit

Permalink
url-option for upload checksum type
Browse files Browse the repository at this point in the history
url-param / header `ck` specifies hashing algo;
md5 sha1 sha256 sha512 b2 blake2 b2s blake2s

value 'no' or blank disables checksumming,
for when copyparty is running on ancient gear
and you don't really care about file integrity
  • Loading branch information
9001 committed Dec 2, 2024
1 parent 94d1924 commit c5a000d
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 11 deletions.
84 changes: 74 additions & 10 deletions copyparty/httpcli.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import socket
import stat
import string
import sys
import threading # typechk
import time
import uuid
Expand Down Expand Up @@ -76,6 +77,7 @@
html_escape,
humansize,
ipnorm,
justcopy,
load_resource,
loadpy,
log_reloc,
Expand Down Expand Up @@ -124,6 +126,8 @@

_ = (argparse, threading)

USED4SEC = {"usedforsecurity": False} if sys.version_info > (3, 9) else {}

NO_CACHE = {"Cache-Control": "no-cache"}

ALL_COOKIES = "k304 no304 js idxh dots cppwd cppws".split()
Expand All @@ -137,6 +141,10 @@

RSS_SORT = {"m": "mt", "u": "at", "n": "fn", "s": "sz"}

A_FILE = os.stat_result(
(0o644, -1, -1, 1, 1000, 1000, 8, 0x39230101, 0x39230101, 0x39230101)
)


class HttpCli(object):
"""
Expand Down Expand Up @@ -2060,10 +2068,31 @@ def dump_to_file(self, is_put: bool) -> tuple[int, str, str, int, str, str]:
# small toctou, but better than clobbering a hardlink
wunlink(self.log, path, vfs.flags)

hasher = None
copier = hashcopy
if "ck" in self.ouparam or "ck" in self.headers:
zs = self.ouparam.get("ck") or self.headers.get("ck") or ""
if not zs or zs == "no":
copier = justcopy
elif zs == "md5":
hasher = hashlib.md5(**USED4SEC)
elif zs == "sha1":
hasher = hashlib.sha1(**USED4SEC)
elif zs == "sha256":
hasher = hashlib.sha256(**USED4SEC)
elif zs in ("blake2", "b2"):
hasher = hashlib.blake2b(**USED4SEC)
elif zs in ("blake2s", "b2s"):
hasher = hashlib.blake2s(**USED4SEC)
elif zs == "sha512":
pass
else:
raise Pebkac(500, "unknown hash alg")

f, fn = ren_open(fn, *open_a, **params)
try:
path = os.path.join(fdir, fn)
post_sz, sha_hex, sha_b64 = hashcopy(reader, f, None, 0, self.args.s_wr_slp)
post_sz, sha_hex, sha_b64 = copier(reader, f, hasher, 0, self.args.s_wr_slp)
finally:
f.close()

Expand Down Expand Up @@ -2300,8 +2329,8 @@ def handle_post_multipart(self) -> bool:
# kinda silly but has the least side effects
return self.handle_new_md()

if act == "bput":
return self.handle_plain_upload(file0)
if act in ("bput", "uput"):
return self.handle_plain_upload(file0, act == "uput")

if act == "tput":
return self.handle_text_upload()
Expand Down Expand Up @@ -2918,13 +2947,41 @@ def upload_flags(self, vfs: VFS) -> tuple[int, bool, int, list[str], list[str]]:
)

def handle_plain_upload(
self, file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]]
self,
file0: list[tuple[str, Optional[str], Generator[bytes, None, None]]],
nohash: bool,
) -> bool:
assert self.parser
nullwrite = self.args.nw
vfs, rem = self.asrv.vfs.get(self.vpath, self.uname, False, True)
self._assert_safe_rem(rem)

halg = "sha512"
hasher = None
copier = hashcopy
if nohash:
halg = ""
copier = justcopy
elif "ck" in self.ouparam or "ck" in self.headers:
halg = self.ouparam.get("ck") or self.headers.get("ck") or ""
if not halg or halg == "no":
copier = justcopy
halg = ""
elif halg == "md5":
hasher = hashlib.md5(**USED4SEC)
elif halg == "sha1":
hasher = hashlib.sha1(**USED4SEC)
elif halg == "sha256":
hasher = hashlib.sha256(**USED4SEC)
elif halg in ("blake2", "b2"):
hasher = hashlib.blake2b(**USED4SEC)
elif halg in ("blake2s", "b2s"):
hasher = hashlib.blake2s(**USED4SEC)
elif halg == "sha512":
pass
else:
raise Pebkac(500, "unknown hash alg")

upload_vpath = self.vpath
lim = vfs.get_dbv(rem)[0].lim
fdir_base = vfs.canonical(rem)
Expand Down Expand Up @@ -3054,8 +3111,8 @@ def handle_plain_upload(
try:
tabspath = os.path.join(fdir, tnam)
self.log("writing to {}".format(tabspath))
sz, sha_hex, sha_b64 = hashcopy(
p_data, f, None, max_sz, self.args.s_wr_slp
sz, sha_hex, sha_b64 = copier(
p_data, f, hasher, max_sz, self.args.s_wr_slp
)
if sz == 0:
raise Pebkac(400, "empty files in post")
Expand Down Expand Up @@ -3187,10 +3244,15 @@ def handle_plain_upload(
jmsg["error"] = errmsg
errmsg = "ERROR: " + errmsg

if halg:
file_fmt = '{0}: {1} // {2} // {3} bytes // <a href="/{4}">{5}</a> {6}\n'
else:
file_fmt = '{3} bytes // <a href="/{4}">{5}</a> {6}\n'

for sz, sha_hex, sha_b64, ofn, lfn, ap in files:
vsuf = ""
if (self.can_read or self.can_upget) and "fk" in vfs.flags:
st = bos.stat(ap)
st = A_FILE if nullwrite else bos.stat(ap)
alg = 2 if "fka" in vfs.flags else 1
vsuf = "?k=" + self.gen_fk(
alg,
Expand All @@ -3205,7 +3267,8 @@ def handle_plain_upload(

vpath = "{}/{}".format(upload_vpath, lfn).strip("/")
rel_url = quotep(self.args.RS + vpath) + vsuf
msg += 'sha512: {} // {} // {} bytes // <a href="/{}">{}</a> {}\n'.format(
msg += file_fmt.format(
halg,
sha_hex[:56],
sha_b64,
sz,
Expand All @@ -3221,13 +3284,14 @@ def handle_plain_upload(
self.host,
rel_url,
),
"sha512": sha_hex[:56],
"sha_b64": sha_b64,
"sz": sz,
"fn": lfn,
"fn_orig": ofn,
"path": rel_url,
}
if halg:
jpart[halg] = sha_hex[:56]
jpart["sha_b64"] = sha_b64
jmsg["files"].append(jpart)

vspd = self._spd(sz_total, False)
Expand Down
21 changes: 20 additions & 1 deletion copyparty/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -2796,6 +2796,26 @@ def yieldfile(fn: str, bufsz: int) -> Generator[bytes, None, None]:
yield buf


def justcopy(
fin: Generator[bytes, None, None],
fout: Union[typing.BinaryIO, typing.IO[Any]],
hashobj: Optional["hashlib._Hash"],
max_sz: int,
slp: float,
) -> tuple[int, str, str]:
tlen = 0
for buf in fin:
tlen += len(buf)
if max_sz and tlen > max_sz:
continue

fout.write(buf)
if slp:
time.sleep(slp)

return tlen, "checksum-disabled", "checksum-disabled"


def hashcopy(
fin: Generator[bytes, None, None],
fout: Union[typing.BinaryIO, typing.IO[Any]],
Expand Down Expand Up @@ -3506,7 +3526,6 @@ def runhook(
txt: str,
) -> dict[str, Any]:
assert broker or up2k # !rm
asrv = (broker or up2k).asrv
args = (broker or up2k).args
vp = vp.replace("\\", "/")
ret = {"rc": 0}
Expand Down
12 changes: 12 additions & 0 deletions copyparty/web/browser.js
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ var Ls = {
"u_pott": "<p>files: &nbsp; <b>{0}</b> finished, &nbsp; <b>{1}</b> failed, &nbsp; <b>{2}</b> busy, &nbsp; <b>{3}</b> queued</p>",
"u_ever": "this is the basic uploader; up2k needs at least<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
"u_su2k": 'this is the basic uploader; <a href="#" id="u2yea">up2k</a> is better',
"u_uput": 'optimize for speed (skip checksum)',
"u_ewrite": 'you do not have write-access to this folder',
"u_eread": 'you do not have read-access to this folder',
"u_enoi": 'file-search is not enabled in server config',
Expand Down Expand Up @@ -1105,6 +1106,7 @@ var Ls = {
"u_pott": "<p>filer: &nbsp; <b>{0}</b> ferdig, &nbsp; <b>{1}</b> feilet, &nbsp; <b>{2}</b> behandles, &nbsp; <b>{3}</b> i kø</p>",
"u_ever": "dette er den primitive opplasteren; up2k krever minst:<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
"u_su2k": 'dette er den primitive opplasteren; <a href="#" id="u2yea">up2k</a> er bedre',
"u_uput": 'litt raskere (uten sha512)',
"u_ewrite": 'du har ikke skrivetilgang i denne mappen',
"u_eread": 'du har ikke lesetilgang i denne mappen',
"u_enoi": 'filsøk er deaktivert i serverkonfigurasjonen',
Expand Down Expand Up @@ -1689,6 +1691,7 @@ var Ls = {
"u_pott": "<p>个文件: &nbsp; <b>{0}</b> 已完成, &nbsp; <b>{1}</b> 失败, &nbsp; <b>{2}</b> 正在处理, &nbsp; <b>{3}</b> 排队中</p>",
"u_ever": "这是基本的上传工具; up2k 需要至少<br>chrome 21 // firefox 13 // edge 12 // opera 12 // safari 5.1",
"u_su2k": '这是基本的上传工具;<a href="#" id="u2yea">up2k</a> 更好',
"u_uput": '提高速度(跳过校验和)',
"u_ewrite": '你对这个文件夹没有写入权限',
"u_eread": '你对这个文件夹没有读取权限',
"u_enoi": '文件搜索在服务器配置中未启用',
Expand Down Expand Up @@ -1935,6 +1938,10 @@ ebi('op_up2k').innerHTML = (

ebi('wrap').insertBefore(mknod('div', 'lazy'), ebi('epi'));

var x = ebi('bbsw');
x.parentNode.insertBefore(mknod('div', null,
'<input type="checkbox" id="uput" name="uput"><label for="uput">' + L.u_uput + '</label>'), x);


(function () {
var o = mknod('div');
Expand Down Expand Up @@ -4210,6 +4217,11 @@ function eval_hash() {
}
bcfg_bind(props, 'mcmp', 'au_compact', false, setacmp);
setacmp();

// toggle bup checksums
ebi('uput').onchange = function() {
QS('#op_bup input[name="act"]').value = this.checked ? 'uput' : 'bput';
};
})();


Expand Down
10 changes: 10 additions & 0 deletions docs/devnotes.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,14 @@ authenticate using header `Cookie: cppwd=foo` or url param `&pw=foo`
| method | params | body | result |
|--|--|--|--|
| PUT | | (binary data) | upload into file at URL |
| PUT | `?ck` | (binary data) | upload without checksum gen (faster) |
| PUT | `?ck=md5` | (binary data) | return md5 instead of sha512 |
| PUT | `?gz` | (binary data) | compress with gzip and write into file at URL |
| PUT | `?xz` | (binary data) | compress with xz and write into file at URL |
| mPOST | | `f=FILE` | upload `FILE` into the folder at URL |
| mPOST | `?j` | `f=FILE` | ...and reply with json |
| mPOST | `?ck` | `f=FILE` | ...and disable checksum gen (faster) |
| mPOST | `?ck=md5` | `f=FILE` | ...and return md5 instead of sha512 |
| mPOST | `?replace` | `f=FILE` | ...and overwrite existing files |
| mPOST | `?media` | `f=FILE` | ...and return medialink (not hotlink) |
| mPOST | | `act=mkdir`, `name=foo` | create directory `foo` at URL |
Expand All @@ -192,6 +196,12 @@ upload modifiers:
| `Accept: url` | `want=url` | return just the file URL |
| `Rand: 4` | `rand=4` | generate random filename with 4 characters |
| `Life: 30` | `life=30` | delete file after 30 seconds |
| `CK: no` | `ck` | disable serverside checksum (maybe faster) |
| `CK: md5` | `ck=md5` | return md5 checksum instead of sha512 |
| `CK: sha1` | `ck=sha1` | return sha1 checksum |
| `CK: sha256` | `ck=sha256` | return sha256 checksum |
| `CK: b2` | `ck=b2` | return blake2b checksum |
| `CK: b2s` | `ck=b2s` | return blake2s checksum |

* `life` only has an effect if the volume has a lifetime, and the volume lifetime must be greater than the file's

Expand Down

0 comments on commit c5a000d

Please sign in to comment.