-
-
Notifications
You must be signed in to change notification settings - Fork 10
/
scatter.py
297 lines (244 loc) · 16.3 KB
/
scatter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
import argparse, random, shutil, tempfile
from pathlib import Path
from statistics import median
from typing import List
from tabulate import tabulate
from xklb import consts, db, utils
from xklb.utils import log
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
usage="""library scatter [--limit LIMIT] [--policy POLICY] [--sort SORT] [--srcmounts SRCMOUNTS] [database] [relative_paths ...]
Balance your disks
$ library scatter -m /mnt/d1:/mnt/d2:/mnt/d3:/mnt/d4/:/mnt/d5:/mnt/d6:/mnt/d7 ~/lb/fs/scatter.db --sort size subfolder/of/mergerfs/mnt
Current path distribution:
╒═════════╤══════════════╤══════════════╤═══════════════╤════════════════╤═════════════════╤════════════════╕
│ mount │ file_count │ total_size │ median_size │ time_created │ time_modified │ time_scanned │
╞═════════╪══════════════╪══════════════╪═══════════════╪════════════════╪═════════════════╪════════════════╡
│ /mnt/d1 │ 12793 │ 169.5 GB │ 4.5 MB │ Jan 27 │ Jul 19 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d2 │ 13226 │ 177.9 GB │ 4.7 MB │ Jan 27 │ Jul 19 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d3 │ 1 │ 717.6 kB │ 717.6 kB │ Jan 31 │ Jul 18 2022 │ yesterday │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d4 │ 82 │ 1.5 GB │ 12.5 MB │ Jan 31 │ Apr 22 2022 │ yesterday │
╘═════════╧══════════════╧══════════════╧═══════════════╧════════════════╧═════════════════╧════════════════╛
Simulated path distribution:
5845 files should be moved
20257 files should not be moved
╒═════════╤══════════════╤══════════════╤═══════════════╤════════════════╤═════════════════╤════════════════╕
│ mount │ file_count │ total_size │ median_size │ time_created │ time_modified │ time_scanned │
╞═════════╪══════════════╪══════════════╪═══════════════╪════════════════╪═════════════════╪════════════════╡
│ /mnt/d1 │ 9989 │ 46.0 GB │ 2.4 MB │ Jan 27 │ Jul 19 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d2 │ 10185 │ 46.0 GB │ 2.4 MB │ Jan 27 │ Jul 19 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d3 │ 1186 │ 53.6 GB │ 30.8 MB │ Jan 27 │ Apr 07 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d4 │ 1216 │ 49.5 GB │ 29.5 MB │ Jan 27 │ Apr 07 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d5 │ 1146 │ 53.0 GB │ 30.9 MB │ Jan 27 │ Apr 07 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d6 │ 1198 │ 48.8 GB │ 30.6 MB │ Jan 27 │ Apr 07 2022 │ Jan 31 │
├─────────┼──────────────┼──────────────┼───────────────┼────────────────┼─────────────────┼────────────────┤
│ /mnt/d7 │ 1182 │ 52.0 GB │ 30.9 MB │ Jan 27 │ Apr 07 2022 │ Jan 31 │
╘═════════╧══════════════╧══════════════╧═══════════════╧════════════════╧═════════════════╧════════════════╛
### Move 1182 files to /mnt/d7 with this command: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from=/tmp/tmpmr1628ij / /mnt/d7
### Move 1198 files to /mnt/d6 with this command: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from=/tmp/tmp9yd75f6j / /mnt/d6
### Move 1146 files to /mnt/d5 with this command: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from=/tmp/tmpfrj141jj / /mnt/d5
### Move 1185 files to /mnt/d3 with this command: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from=/tmp/tmpqh2euc8n / /mnt/d3
### Move 1134 files to /mnt/d4 with this command: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from=/tmp/tmphzb0gj92 / /mnt/d4
Scatter the most recent 100 files
$ library scatter -m /mnt/d1:/mnt/d2:/mnt/d3:/mnt/d4/:/mnt/d5:/mnt/d6:/mnt/d7 -l 100 -s 'time_modified desc' ~/lb/fs/scatter.db /
Show disk usage (why not?)
$ library scatter -m /mnt/d1:/mnt/d2:/mnt/d3:/mnt/d4/:/mnt/d5:/mnt/d6:/mnt/d7 ~/lb/fs/scatter.db blah/ --usage
Relative disk utilization:
/mnt/d1: ################# 22.2%
/mnt/d2: ################# 22.2%
/mnt/d3: #### 5.5%
/mnt/d4: ########################## 33.4%
/mnt/d5: ############# 16.6%
/mnt/d6: 0.0%
/mnt/d7: 0.0%
Relative free space:
/mnt/d1: 0.1%
/mnt/d2: 0.1%
/mnt/d3: 0.1%
/mnt/d4: 0.1%
/mnt/d5: ########## 13.6%
/mnt/d6: ############################## 37.6%
/mnt/d7: ###################################### 48.4%
"""
)
parser.add_argument("--limit", "-L", "-l", "-queue", "--queue")
parser.add_argument("--policy", "-p", default="rand")
parser.add_argument("--group", "-g", default="size")
parser.add_argument("--sort", "-s", default="random()", help="Sort files before moving")
parser.add_argument("--usage", "-u", action="store_true", help="Show disk usage")
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--srcmounts", "-m", required=True, help="/mnt/d1:/mnt/d2")
parser.add_argument("database")
parser.add_argument(
"relative_paths",
nargs="+",
help="Paths to scatter, relative to the root of your mergerfs mount; any path substring is valid",
)
args = parser.parse_args()
args.db = db.connect(args)
args.srcmounts = [m.rstrip("\\/") for m in args.srcmounts.split(":")]
args.relative_paths = [p.lstrip(".") for p in args.relative_paths]
log.info(utils.dict_filter_bool(args.__dict__))
return args
def get_table(args) -> List[dict]:
and_paths = ""
for i, _path in enumerate(args.relative_paths):
and_paths += f" and path like :path_{i}"
media = list(
args.db.query(
f"""
select
path
, size
, time_created
, time_modified
, time_downloaded
from media
where 1=1
and time_deleted = 0
and is_dir is NULL
{and_paths}
order by {args.sort}
{'limit :limit' if args.limit else ''}
""",
{
"limit": args.limit,
**{f"path_{i}": f"%{path}%" for i, path in enumerate(args.relative_paths) if args.relative_paths},
},
)
)
return media
def get_disk_stats(src_mounts):
mount_space = []
total_used = 0
total_free = 0
grand_total = 0
for src_mount in src_mounts:
total, used, free = shutil.disk_usage(src_mount)
total_used += used
total_free += free
grand_total += total
mount_space.append((src_mount, used, free, total))
return [
{"mount": mount, "used": used / total_used, "free": free / total_free, "total": total / grand_total}
for mount, used, free, total in mount_space
]
def get_path_stats(args, data):
result = []
for srcmount in args.srcmounts:
disk_files = [d for d in data if d["path"].startswith(srcmount)]
if disk_files:
result.append(
{
"mount": srcmount,
"file_count": len(disk_files),
"total_size": sum(d["size"] for d in disk_files),
"median_size": median(d["size"] for d in disk_files),
"time_created": median(d["time_created"] for d in disk_files),
"time_modified": median(d["time_modified"] for d in disk_files),
"time_scanned": median(d["time_downloaded"] for d in disk_files),
},
)
return result
def print_disk_stats(space):
print("Relative disk utilization:")
for d in space:
print(f"{d['mount']}: {'#' * int(d['used'] * 80)} {d['used']:.1%}")
print("\nRelative free space:")
for d in space:
print(f"{d['mount']}: {'#' * int(d['free'] * 80)} {d['free']:.1%}")
def print_path_stats(tbl):
tbl = utils.list_dict_filter_bool(tbl, keep_0=False)
tbl = utils.col_naturalsize(tbl, "total_size")
tbl = utils.col_naturalsize(tbl, "median_size")
for t in consts.TIME_COLUMNS:
utils.col_naturaldate(tbl, t)
print(tabulate(tbl, tablefmt="fancy_grid", headers="keys", showindex=False))
def rebin_files(args, disk_stats, all_files):
total_size = sum(d["size"] for d in all_files)
untouched = []
to_rebin = []
full_disks = []
for disk_stat in disk_stats:
disk_files = [d for d in all_files if d["path"].startswith(disk_stat["mount"])]
disk_rebin = []
if disk_files:
if args.group == "size":
ideal_allocation_size = total_size * disk_stat["total"]
size = 0
for file in disk_files:
size += file["size"]
if size < ideal_allocation_size:
untouched.append(file)
else:
disk_rebin.append({"mount": disk_stat["mount"], **file})
else:
ideal_allocation_count = len(disk_files) // len(disk_stats)
untouched.extend(disk_files[:ideal_allocation_count])
disk_rebin.extend({"mount": disk_stat["mount"], **file} for file in disk_files[ideal_allocation_count:])
if len(disk_rebin) > 0:
full_disks.append(disk_stat["mount"])
to_rebin.extend(disk_rebin)
rebinned = []
for file in to_rebin:
valid_targets = [d for d in disk_stats if d["mount"] not in [*full_disks, file["mount"]]]
mount_list = [d["mount"] for d in valid_targets]
if args.policy in ["free", "pfrd"]:
new_mount = random.choices(mount_list, weights=[stat["free"] for stat in valid_targets], k=1)[0]
elif args.policy in ["used", "purd"]:
new_mount = random.choices(mount_list, weights=[stat["used"] for stat in valid_targets], k=1)[0]
elif args.policy in ["total", "ptrd"]:
new_mount = random.choices(mount_list, weights=[stat["total"] for stat in valid_targets], k=1)[0]
else:
new_mount = random.choices(mount_list, k=1)[0]
file["from_path"] = file["path"]
file["path"] = file["path"].replace(file["mount"], new_mount)
rebinned.append(file)
return untouched, rebinned
def scatter() -> None:
args = parse_args()
disk_stats = get_disk_stats(args.srcmounts)
if args.usage:
print_disk_stats(disk_stats)
raise SystemExit(0)
files = get_table(args)
path_stats = get_path_stats(args, files)
print("\nCurrent path distribution:")
print_path_stats(path_stats)
untouched, rebinned = rebin_files(args, disk_stats, files)
print("\nSimulated path distribution:")
print(len(rebinned), "files should be moved")
print(len(untouched), "files should not be moved")
path_stats = get_path_stats(args, rebinned + untouched)
print_path_stats(path_stats)
print("\n######### Commands to run #########")
for disk_stat in sorted(disk_stats, key=lambda d: d["free"], reverse=True):
dest_disk_files = [
d["from_path"].replace(d["mount"], d["mount"] + "/.")
for d in rebinned
if d["path"].startswith(disk_stat["mount"])
]
if len(dest_disk_files) == 0:
continue
temp_file = Path(tempfile.mktemp())
with temp_file.open("w") as f:
f.writelines("\n".join(dest_disk_files))
print(
f"""### Move {len(dest_disk_files)} files to {disk_stat['mount']}: ###
rsync -aE --xattrs --info=progress2 --no-inc-recursive --remove-source-files --files-from={temp_file} / {disk_stat['mount']}"""
)
if __name__ == "__main__":
scatter()