forked from emacs-w3m/emacs-w3m
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathw3m-download.el
2534 lines (2296 loc) · 93.4 KB
/
w3m-download.el
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
;;; w3m-download.el --- download using emacs-w3m -*- coding: utf-8 ; lexical-binding: t -*-
;; Copyright © 2019-2021 Boruch Baum <[email protected]>
;; Authors: Boruch Baum <[email protected]>
;; Keywords: w3m, WWW, hypermedia
;; Homepage: http://emacs-w3m.namazu.org/
;; Repository: https://github.com/emacs-w3m/emacs-w3m
;; This file is part of `emacs-w3m'.
;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 2, or (at your option)
;; any later version.
;; This program is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;; You should have received a copy of the GNU General Public License
;; along with this program; see the file COPYING. If not, write to
;; the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
;; Boston, MA 02110-1301, USA.
;;; Commentary:
;; This file provides download features for the `emacs-w3m' project.
;; Although the project is meant to be a front-end to the `w3m'
;; browser, this code uses elisp and, when available, external
;; back-end programs (currently just `wget') in order to offer
;; additional download features not native to `w3m':
;;
;; + Partial downloads are stored on disk, not in memory (yes...).
;;
;; + Resumption of partial downloads that were interrupted due to
;; aborts or other failures.
;;
;; + Bulk downloading of a buffer's links or region's links,
;; selectable by regex and a presentation buffer. Throughout the
;; code, this feature is referred to as `download-select'.
;;
;; + The number of simultaneous downloads may be controlled, and can
;; be dynamically changed.
;;
;; + Downloads are queued, and the queue can be modified in
;; real-time.
;;
;; + Video downloads do not need actual download links, but can be
;; scraped from web pages. This extension requires the external
;; program `youtube-dl' (as is not limited to youtube).
;;
;; + Downloads may be 'throttled' to limit bandwidth per download.
;;
;; + Optional appending of meta-data to a download.
;;
;; + Defcustom `w3m-download-save-metadata' controls whether
;; downloaded `png' and `jpeg' files should have their link's
;; "alt" caption is stored as meta-data element
;; "Exif.Image.ImageDescription". Note that enabling this option
;; will modify the file's checksum. This option currently uses
;; `exif' and `exiv2' as external back-end programs.
;;
;; Some ways to view from the command-line an embedded caption
;; are:
;;
;; exiv2 -g "Exif.Image.ImageDescription" foo.png
;; exif --ifd=0 -t0x010e foo.jpg |grep value
;;
;; + Defcustom `w3m-download-enable-xattr' controls whether to
;; save a file's original URL and Referer HTTP header value.
;; This feature uses the `wget' `--xattr' argument and thus
;; requires that the save-path be on a file-system that supports
;; extended attributes.
;;
;; Be aware that the URL might contain private information like
;; access tokens or credentials.
;;
;; + Individual detailed download progress logs in dedicated
;; buffers, automatically deleted upon successful completion.
;;
;; + A single central buffer to view all downloads in all states
;; (queued, running, paused, failed, and completed), act on them,
;; and see their details. The buffer includes a summary statistics
;; line.
;;
;; + Downloads can be paused, resumed, and removed.
;;
;; + The category order of the display can be easily customized.
;;
;; + Dedicated faces for each download category.
;;
;; + Download links that are 'hidden' within HTML source code inside
;; <script> elements or other places can optionally be used. See
;; variable `w3m-download-select-deep-search'.
;;
;; + Downloads with commonly duplicated names can be uniquified to
;; user-meaningful file-names. See variable
;; `w3m-download-ambiguous-basename-alist'.
;; This file also absorbed most of the legacy download functions,
;; those which were basically wrappers for `w3m-download', to keep all
;; (most) functionality in one place. The functions and defcustom not
;; moved here are:
;;
;; --FILE--
;; mew-w3m.el mew-w3m-ext-url-fetch (dummy url)
;; w3m.el w3m-external-view (url &optional no-cache handler)
;; w3m-lnum.el w3m-lnum-actions-image-alist
;; w3m-lnum.el w3m-lnum-save-image ()
;;; Dependencies:
;; w3m-util ; for ??
;; key-assist ; for key-assist
;;; Usage:
;; From anywhere within `emacs-w3m' you can get a combination context
;; help and command launcher:
;;
;; `w3m-download-help'
;;
;; For downloading single items at point, there exist multiple
;; functions that seem redundant:
;;
;; `w3m-download-this-url' (legacy)
;; `w3m-download-this-image' (legacy)
;; `w3m-save-image' (legacy)
;; `w3m-download-video' (uses `youtube-dl')
;; `w3m-download-video-at-point' (uses `youtube-dl')
;; `w3m-download-using-wget'
;; `w3m-download-using-w3m' (not recommended)
;; `w3m-download'
;;
;; For downloading more than one link in a buffer or region:
;;
;; `w3m-download-select'
;;
;; At any time, you can change the number of allowable
;; simultaneous downloads:
;;
;; `w3m-download-increase-simultaneous'
;; `w3m-download-decrease-simultaneous'
;;
;; This is easiest done from a w3m-download-select buffer,
;; where the functions are bound to the +/- keys.
;;
;; The current recommended way to kill a download in progress is
;; to kill its progress buffer. To kill all current downloads:
;;
;; `w3m-download-delete-all-download-buffers'
;; `w3m-download-kill-all-wget-processes' (just an alias)
;;
;; There also exists for now a slightly different function
;; that was originally intended as a possible hook function:
;;
;; `w3m-download-kill-all-asociated-processes'
;;
;; The download queue can be examined in a special buffer that allows
;; one to delete or change the order of items:
;;
;; `w3m-download-view-queue'
;;
;; This function is intended to be bound to keybinding `S-C-y' in
;; a nod to what seems standard in many modern browsers. You can
;; manually bind the function by evaluating:
;;
;; (define-key w3m-mode-map [(shift ctrl y)] 'w3m-download-view-queue)
;; (define-key w3m-mode-map "Y" 'w3m-download-view-queue)
;;
;;
;; The buffer auto-refreshes every `w3m-download-refresh-interval'
;; seconds (default 16), but you can always force a refresh using
;; `w3m-download-refresh-buffer' (`C-g'), and you can change the
;; value using `w3m-download-set-refresh-interval' (`C-u C-g').
;;
;; The download resumption behavior is controlled by variable
;; `w3m-download-resume' to either auto-restart them, pause them, or
;; start with a clean empty slate.
;;; TODO:
;; + branch w3m-history-scrub needs now to shred
;; `w3m-download-save-file'.
;;
;; + Easier way to find a particular download to kill
;;
;; + function: w3m-download-select-exec
;;
;; + pre-screen the list of downloads for possible name conflicts
;; with existing files, and duplicates, and prompt the user
;; accordingly.
;;
;; + maybe do this instead when creating the select buffer, ie.
;; gray out entries that already exist on the queue
;;
;; + this should be done for downloads outside of select-exec also
;;
;;
;; + `w3m--download-check-and-use-cache'
;;
;; + does not emulate `w3m-download-enable-xattr'
;;
;; + add hooks for quitting emacs-w3m
;;
;; + this might not be necessary. the current method using the save
;; and load list may be sufficient.
;;
;; + feature creep: None of these-items are necessary. At some point
;; the project could consider turning to an external program which
;; specializes in downloading, but anyway ...
;;
;; + add an annotation field in lists, to be a visible comment that
;; a user can make on a download record.
;;
;; + apply certain command upon a region, ie. more than one download
;; at a time. This has posed problematic for me because although
;; `transient-mark-mode' is non-nil, `region-active-p' always
;; returns nil, so I haven't been able to get region commands to
;; work (see programming note in-line below). Anyway, here are
;; candidates for commands on which to apply regions:
;;
;; + move position in queue
;;
;; + toggle details
;;
;; + toggle pause
;;
;; + delete lines / entries
;;
;; + an alist of extensions and metadata commands
;;
;; + an alist of download commands (eg. aria2, curl, axel)
;;
;; + offloading to a dedicated downloader (eg. uget)
;;
;; + persistence across crashes (ie. maintain a disk file)
;;
;; + renice download processes
;;
;; + delete a partially downloaded file when manually aborted
;; (eg. when killing the progress buffer)
;;
;; + use the process-fiter function to add detail to each download.
;;
;; + Think about the need / desirability of hook functions to
;; optionally kill running downloads, for `kill-emacs-hook'
;; (without user interaction), `kill-emacs-query-functions'
;; (with user interaction), and the mode exit hook for
;; `emacs-w3m' (look it up, what is it?). Maybe there should
;; also be download recovery/resume/re-attach hooks for either
;; `after-init-hook', `emacs-startup-hook' or the startup hook
;; for `emacs-w3m' (look it up, what is it?)
;;; Code:
;;; Dependencies:
(require 'w3m-util) ; for ??
(require 'key-assist nil t) ; for key-assist
;;; Temporary compatability operation(s):
;; My development git fork has a different messaging standard, based
;; upon a pending pull request from my branch `bb_messaging', so
;; support this git branch being merged into a branch lacking that
;; other merge.
(eval-when-compile
(when (not (fboundp 'w3m--message))
(defun w3m--message (timeout face &rest args)
(apply 'w3m-message args))))
;;; Global constants:
(defconst w3m--download-queue-header-prefix
(concat (propertize "w3m download queue:" 'face 'underline) " ")
"Beginning of download queue buffer's variable `header-line-format'.")
(defconst w3m--download-select-header-prefix
(concat (propertize "w3m download select:" 'face 'underline) " ")
"Beginning of download select buffer's variable `header-line-format'.")
(defconst w3m--download-mutex (make-mutex "w3m-download")
"Control manipulation of download state lists.
These are:`w3m--download-queued', `w3m--download-running',
`w3m--download-paused', `w3m--download-failed', and
`w3m--download-completed'.")
(defconst w3m--download-progress-regex-wget
"\\([0-9.]+%\\)?\\[[^]]+] +\\([^ ]+\\) +\\([^/]+/[^ ]+\\) +\\(.*\\)$"
;; %completed bytes done download speed eta
"Parses four values from wget progress messages.")
(defconst w3m--download-progress-regex-youtube-dl
"\\([0-9.]+%\\) of \\([^ ]+\\) at +\\([^ ]+\\) \\(.*\\)\n?$"
"Parses four values from youtube-dl progress messages.")
;;; Global variables:
(defvar w3m-download-select-mode-map nil
"Major mode for bulk download of links in a buffer or region.")
(unless w3m-download-select-mode-map
(let ((map (make-keymap)))
(suppress-keymap map)
(define-key map [? ] 'w3m-download-select-toggle-line); [? ] = <SPC>
(define-key map "\C-k" 'w3m-download-delete-line)
(define-key map "q" 'w3m-download-buffer-quit)
(define-key map "Q" 'w3m-download-buffer-quit)
(define-key map "\C-c\C-k" 'w3m-download-buffer-quit)
(define-key map "\C-c\C-c" 'w3m-download-select-exec)
(define-key map "+" 'w3m-download-increase-simultaneous)
(define-key map "-" 'w3m-download-decrease-simultaneous)
(define-key map "?" 'w3m-download-help)
(setq w3m-download-select-mode-map map)))
(defvar w3m-download-queue-mode-map nil
"Major mode for viewing and editing the `w3m-download' queue.")
(unless w3m-download-queue-mode-map
(let ((map (make-sparse-keymap)))
(suppress-keymap map)
(define-key map "\C-o" 'w3m-download-toggle-details)
(define-key map "\C-c\C-o" 'w3m-download-toggle-all-details)
(define-key map [?\t] 'w3m-download-queue-next-entry)
(define-key map [backtab] 'w3m-download-queue-previous-entry)
(define-key map "p" 'w3m-download-toggle-pause)
(define-key map "r" 'w3m-download-retry)
(define-key map "\C-k" 'w3m-download-delete-line)
(define-key map "q" 'w3m-download-buffer-quit)
(define-key map "Q" 'w3m-download-buffer-quit)
(define-key map "\C-c\C-k" 'w3m-download-buffer-quit)
(define-key map "\C-c\C-n" 'w3m-download-queue-drop)
(define-key map "\C-c\C-p" 'w3m-download-queue-raise)
(define-key map "\C-c\C-a" 'w3m-download-queue-top)
(define-key map "\C-c\C-e" 'w3m-download-queue-bottom)
(define-key map "\C-g" 'w3m-download-refresh-buffer)
(define-key map "+" 'w3m-download-increase-simultaneous)
(define-key map "-" 'w3m-download-decrease-simultaneous)
(define-key map "w" 'w3m-download-toggle-line-wrap)
(define-key map "?" 'w3m-download-help)
(setq w3m-download-queue-mode-map map)))
(defvar w3m--download-queued nil
"List of pending downloads.
Manipulation of this variable should only be done when holding
`w3m--download-mutex'. Each element is a list of five fields:
1. URL - the link to download, as a string.
2. WGET-CMD-LIST - the complete wget command line, as a list of
strings suitable for function `start-process'.
3. SAVE-PATH - the fully qualified path to place the download.
4. METADATA - the complete command line for tagging the download
with metadata, as a string sutiable for function
`shell-command'.
5. SHOW-STATE - the boolean whether its details are being shown
or hidden.
Additionally, if the element had once been a running download,
ie. it had been paused and then re-queued:
6. START_TIMESTAMP - a string.
7. WGET_PROGRESS - a string, expected to include % completed and
bytes downloaded")
(defvar w3m--download-running nil
"List of running downloads.
This is a reference for use by the display buffer and the resume
operation. It is different from `w3m--download-processes-list'.
Manipulation of this variable should only be done when holding
`w3m--download-mutex'. Each element here is inherited from
`w3m--download-queued', and adds the following fields:
6. START_TIMESTAMP - a string.
7. WGET_PROGRESS - a string, expected to include % completed,
bytes downloaded, rate, and eta.
8. PROC-ID - the Emacs process id for the wget operation.
9. BUFFER - the process buffer, where STDOUT/STDERR is sent.")
(defvar w3m--download-paused nil
"List of paused downloads.
Manipulation of this variable should only be done when holding
`w3m--download-mutex'. Each entry has the seven elements of
`w3m--download-queued', as inherited from either
`w3m--download-queued' or `w3m--download-running', with the
following modifications:
8. ERROR_CODE - the 'event' string from the Emacs process sentinel.
9. Removed.")
(defvar w3m--download-failed nil
"List of failed downloads.
Manipulation of this variable should only be done when holding
`w3m--download-mutex'. Each element here is inherited from
`w3m--download-running', with the following modifications:
6. STOP_TIMESTAMP - a string.
7. WGET_PROGRESS - a string, expected to include % completed,
bytes downloaded, rate, and eta.
8. ERROR_CODE - the 'event' string from the Emacs process sentinel.
9. Removed.")
(defvar w3m--download-completed nil
"List of completed downloads.
Manipulation of this variable should only be done when holding
`w3m--download-mutex'. Each element here is inherited from
`w3m--download-running', with the following modifications:
8. Removed.
9. Removed.")
(defvar w3m--download-processes-list nil
"Global list of all running `w3m-download' processes.")
(defvar w3m--download-select-filter-history nil
"Record of past download-select filter use.
Needed for function `completing-read' to be able to display the
most recently used.")
(defvar w3m--download-refresh-buffer-timer nil
"Timer object for function `w3m--download-refresh-buffer'.")
(defvar w3m--download-buffer-sequence '(
(w3m--download-running "R " w3m-download-running)
(w3m--download-queued "Q " w3m-download-queued)
(w3m--download-paused "P " w3m-download-paused)
(w3m--download-failed "F " w3m-download-failed)
(w3m--download-completed "C " w3m-download-completed))
"Sort order details for the `w3m-download-queue' display buffer.
This is just a bit too tricky and error-prone to become a
defcustom. There must be a list element for each category of
items to be displayed. Each element is itself a list of three
elements:
1. The queue list symbol name.
2. A two character string to identify members of the list. For
the first character, you should be able to go to town with the
unicode emoji of your choice, but your choice needs to also be
put into `w3m--download-category-regex', and the second character
must be a space.'
3. The face symbol name.")
(defvar w3m--download-category-regex "^[RQPFC]"
"A regex collection download category identifiers.
These are five single characters that appear at the beginning of
lines denoting 'running', 'queued', 'paused', 'failed', and
'completed' downloads. These need to match elements of
`w3m--download-buffer-sequence'.")
;;; User-options:
(defcustom w3m-download-max-simultaneous 4
"Maximum number of simultaneous downloads.
This value can be temporarily changed at any time from the
download queue buffer using
`w3m-download-decrease-simultaneous' (bound to
`\\<w3m-download-queue-mode-map>\\[w3m-download-decrease-simultaneous]')
or `w3m-download-increase-simultaneous' (bound to
`\\[w3m-download-increase-simultaneous]')."
:group 'w3m
:type 'integer)
(defcustom w3m-download-resume 'pause
"What to do with unfinished downloads when beginning a new `emacs-w3m' session.
NOTHING means ignore any past downloads. The record of past
downloads will be deleted.
PAUSE means to reload the record of past downloads, but
re-categorize all running or queued ones as paused.
AUTO-RESTART means to reload the record of past downloads and
immediately restart them."
:group 'w3m
:type '(radio (const :tag "Pause" pause)
(const :tag "Nothing" nil)
(const :tag "Auto-restart" auto-restart)))
(defcustom w3m-download-refresh-interval 1
"How often (in seconds) to refresh the download display buffer.
This is also the interval for saving the download lists to
`w3m-download-save-file'.
This value can be temporarily changed at any time from the download queue
buffer using `w3m-download-refresh-buffer' (bound to `C-u \\<w3m-download-queue-mode-map>\\[w3m-download-refresh-buffer]')."
:group 'w3m
:type 'integer)
(defcustom w3m-download-save-file
(concat (file-name-as-directory w3m-profile-directory)
".emacs-w3m-downloads")
"Where to save download information.
This will include information about running, queued, paused,
failed, and completed downloads, so that upon resuming emacs-w3m
you can continue where you left off and remember what you
recently did."
:group 'w3m
:type 'file)
(defcustom w3m-download-save-metadata t
"Whether image downloads should save their descriptions.
This sets a feature for function `w3m-download' that operates on
jpeg and png images, and saves their link 'alt' caption strings
as the file's \"Exif.Image.ImageDescription\" metadata element.
From within emas-w3m you can see what the value would be by
moving point to the link (the string will appear in the
mini-buffer).
This operation is performed by calls to external programs; if
those programs are not executable on your system, a message will
be logged. Currently, the selected programs are hard-coded,
`exif' for jpeg files, and `exiv2' for png files.
Note that enabling this option will modify the file's checksum."
:group 'w3m
:type 'boolean)
(defcustom w3m-download-enable-xattr nil
"Save extended attributes when using wget for downloading.
This uses the `wget' `--xattr' argument, to enable use of file a
system's extended attributes to save the original URL and the
Referer HTTP header value if used. Be aware that the URL might
contain private information like access tokens or credentials."
:group 'w3m
:type 'boolean)
(defcustom w3m-download-throttle nil
"Download rate limit (bytes/second). NIL for no throttling.
This (currently) only takes effect for future queued items, ie.
not items already on the queue."
:group 'w3m
:type 'integer)
(defcustom w3m-download-wget-options "--server-response --progress=bar:noscroll"
"Catch-all for your preferred `wget' options.
This (currently) only takes effect for future queued items, ie.
not items already on the queue.
Use this option at your own risk! Refer to `man(1) wget', and
consider how anything you put here will affect the interaction
between `wget' and this Emacs package!
Note that the `wget' option `--limit-rate' is already handled by
`w3m-download-throttle', `--enable-xattr' is already handled by
`w3m-download-enable-xattr', `--continue' is hard-coded into this
package, and `--output-document' is used by this package to label
partial downloads."
:group 'w3m
:type 'string)
(defcustom w3m-download-select-filter-list
'("all_links: .*$"
"e-books: \\.\\(\\(pdf\\)\\|\\(epub\\)\\|\\(mobi\\)\\)$"
"archives: \\.\\(\\(t?gz\\)\\|\\(zip\\)\\|\\(xz\\)\\|\\(lzma\\)\\)$"
"text: \\.txt$"
"images: \\.\\(\\(png\\)\\|\\(jpe?g\\)\\|\\(gif\\)\\)$"
"audio: \\.\\(\\(og[ag]\\)\\|\\(mp3\\)\\)$"
"video: \\.\\(\\(ogv\\)\\|\\(mp4\\)\\)$"
"iso/img: \\.\\(\\(iso\\)\\|\\(img\\)\\)$")
"Regex patterns for filtering downloads.
Each regex string may begin with an optional single descriptive word
ending with a colon `:', followed by the regex for the filter."
:group 'w3m
:type '(repeat (string :format "%v\n")))
(defcustom w3m-download-video-alist
'(("\\.youtube\\." . "-f 18 --newline")
("\\.youtu\\.be\\." . "-f 18 --newline"))
"*Specific arguments to use for downloading from specific urls.
This data is used for function `w3m-download-video-at-point' and
`w3m-download-video-at-point', which use the external program
`youtube-dl'.
Each element should be of the form (URL . ARGS),
for example (\"\\.youtube\\.\" . \"-f 18\").
Refer to \"man(1) `youtube-dl'\" for information on available
arguments."
:group 'w3m
:type '(repeat
(cons :format "%v" :indent 12
(string :format "url base: %v")
(string :format "args for youtube-dl: %v"))))
(defcustom w3m-download-ambiguous-basename-alist
'(("^downloadhandler.ashx$"
("hebrewbooks.org" nil w3m-download--make-basename-from-path+query (".pdf"))
("^.*\\?[^#]+$" nil w3m-download--make-basename-from-path+query (".mp3"))))
"How to handle ambiguous or commonly duplicated save-file names.
The car of each element of this list is a regex describing a
troublesome download file-name string.
The cdr is a list. Its first element is a url regex string
WHITE-LIST rule describing where FILE-NAME is troublesome and
undesirable. Its second element is either nil or a list of url
regex strings BLACK-LIST for specific over-rides of the
WHITE-LIST rule. The third element is a function to be used to
generate the desired replacement FILE-NAME; it must accept two
arguments, the download url string, and an ARG-LIST. The
final element is said ARG-LIST."
:group 'w3m
:type '(repeat
(cons :indent 2
(regexp :format "Regexp for troublesome filename: %v" :length 20)
(repeat (list :indent 4
(regexp :format "Regexp matching url to apply rule: %v")
(choice :format "Exceptions: %[Value Menu%] %v"
(const :tag "none" nil)
(list :tag "list of url regexps not to be filtered"
(repeat (regexp :format "Regexp matching url not to be filtered: %v"))))
(function :format "Function to create good filename: %v")
(repeat :indent 6
(string :format "Arg for the function: %v")))))))
(defcustom w3m-download-select-deep-search nil
"Whether to thoroughly search the HTML source for links.
The default for this option is NIL, because it searches even
inside <script> elements, and even inside HTML comments, and is
not able to respect a user's request to limit the search to a
just a buffer region; the HTML source for the entire buffer will be searched."
:group 'w3m
:type 'boolean)
;;; Faces:
(defface w3m-download-selected
'((t :underline nil :inherit 'w3m-session-select))
"Face of URL selected for download in w3m-download-select
buffer."
:group 'w3m)
(defface w3m-download-progress
'((t :background "blue")) ;; "brightblack"))
"Face of running downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
(defface w3m-download-queued
'((t :weight normal :inherit 'default))
"Face of running downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
(defface w3m-download-running
'((t :weight normal :foreground "green"))
"Face of running downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
(defface w3m-download-paused
'((t :weight normal :underline nil :foreground "yellow"))
"Face of failed downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
(defface w3m-download-failed
'((t :weight normal :inherit 'w3m-error))
"Face of failed downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
(defface w3m-download-completed
'((t :weight normal :underline nil :foreground "blue"))
"Face of failed downloads in w3m-download buffer."
; TODO: Make this more sophisticated to be consistent with other
; project faces, to account for light/dark themes and window-display-p
:group 'w3m)
;;; Buffer-local variables:
(defvar-local w3m--download-url nil
"The URL to be downloaded.
This variable is set buffer-local in the download's progress
buffer.")
(defvar-local w3m--download-cmd nil
"The command used to perform the download.
This variable is set buffer-local in the download's progress
buffer.")
(defvar-local w3m--download-save-path nil
"The full path-name of the downloaded file.
This variable is set buffer-local in the download's progress
buffer.")
(defvar-local w3m--download-local-proc nil
"The download's process id.
This variable is set buffer-local in the download's progress
buffer.")
(defvar-local w3m--download-metadata-operation nil
"The complete shell command to be used to tag a download.
A text string. This variable is set buffer-local in the
download's progress buffer.")
;;; Hook functions:
(defun w3m--download-kill-associated-process ()
"Hook function for `kill-buffer-hook' for `w3m-download' buffers.
`w3m--download-local-proc' should have been set as a local
variable at buffer creation."
(if (not (processp w3m--download-local-proc))
(w3m--message t 'w3m-warning
"Warning: no process found to kill (w3m-download).")
(delete-process w3m--download-local-proc)
(setq w3m--download-processes-list
(assq-delete-all w3m--download-local-proc w3m--download-processes-list)))
(when (and w3m--download-queued
(>= w3m-download-max-simultaneous
(length w3m--download-processes-list)))
(w3m--download-from-queue)))
(defun w3m--download-update-statistics ()
"Hook function for `w3m-download-select' and `w3m-download-queue' buffers.
Meant for use with `post-command-hook'."
(let ((num-selected 0)
(inhibit-read-only t)
(pos (point)))
(when (eq major-mode 'w3m-download-select-mode)
(goto-char (point-min))
(while (re-search-forward "^\\[X" nil t)
(setq num-selected (1+ num-selected)))
(goto-char pos))
(setq header-line-format
(concat
(if (eq major-mode 'w3m-download-select-mode)
w3m--download-select-header-prefix
w3m--download-queue-header-prefix)
(if (eq major-mode 'w3m-download-select-mode)
(propertize (format "%d selected; " num-selected)
'face 'w3m-download-selected)
"")
(propertize (format "%d/%d running/max;"
(length w3m--download-running)
w3m-download-max-simultaneous)
'face 'w3m-download-running)
(propertize (format " %d queued;" (length w3m--download-queued))
'face 'w3m-download-queued)
(propertize (format " %d paused;" (length w3m--download-paused))
'face 'w3m-download-paused)
(propertize (format " %d failed;" (length w3m--download-failed))
'face 'w3m-download-failed)
(propertize (format " %d completed;" (length w3m--download-completed))
'face 'w3m-download-completed)))))
(defun w3m--download-update-faces-pre-command ()
"Hook function for `w3m-download-select' and `w3m-download-queue' buffers.
Meant for use with `pre-command-hook'."
(let ((pos (point))
(beg (if (eq major-mode 'w3m-download-select-mode)
(line-beginning-position)
(previous-single-property-change
(min (1+ (point)) (point-max))
'url nil (point-min))))
(end (if (eq major-mode 'w3m-download-select-mode)
(line-end-position)
(next-single-property-change (point) 'url nil (point-max))))
(inhibit-read-only t))
(unless (= (point) (point-max))
(w3m-remove-face-property beg end '(:weight bold)))
(goto-char pos)))
(defun w3m--download-update-faces-post-command ()
"Hook function for `w3m-download-select' and `w3m-download-queue' buffers.
Meant for use with `post-command-hook'."
(let ((pos (point))
(beg (if (eq major-mode 'w3m-download-select-mode)
(line-beginning-position)
(previous-single-property-change
(min (1+ (point)) (point-max))
'url nil (point-min))))
(end (if (eq major-mode 'w3m-download-select-mode)
(line-end-position)
(next-single-property-change (point) 'url nil (point-max))))
(inhibit-read-only t))
(unless (or (= (point) (point-max))
(>= beg end))
(add-face-text-property beg end '(:weight bold)))
(goto-char pos)))
(defun w3m--download-queue-buffer-kill ()
"Hook function to run when killing a w3m-download-queue buffer."
; intentionally redundant and unnecessary code, for safety
(when (timerp w3m--download-refresh-buffer-timer)
(cancel-timer w3m--download-refresh-buffer-timer))
(cancel-function-timers 'w3m--download-refresh-buffer)
(setq w3m--download-refresh-buffer-timer nil))
;;; Internal functions:
(defun w3m-download-select-mode ()
"Major mode for bulk download of links in a buffer or region.
\\[w3m-download-select-toggle-line]\tToggle current line's status.
\\[w3m-download-delete-line]\tDelete current line.
\\[w3m-download-increase-simultaneous]\tIncrease numner of parallel downloads.
\\[w3m-download-decrease-simultaneous]\tDecrease number of parralel downloads.
\\[w3m-download-buffer-quit]\tQuit.
\\[w3m-download-select-exec]\tPerform the downloading.\n\n
\\{w3m-download-select-mode-map}\n\n"
(setq
mode-name "w3m download select"
header-line-format "w3m download select -- (selected, running, queued, paused, failed, and completed)"
truncate-lines t
major-mode 'w3m-download-select-mode
buffer-read-only t)
(cursor-intangible-mode)
(use-local-map w3m-download-select-mode-map)
(add-hook 'pre-command-hook 'w3m--download-update-faces-pre-command t t)
(add-hook 'post-command-hook 'w3m--download-update-faces-post-command t t)
(add-hook 'post-command-hook 'w3m--download-update-statistics t t))
(defun w3m--download-queue-buffer-help-string ()
"Return a list of current keybindings for `w3m-download-queue-mode-map'.
This is displayed at the top of the buffer as cheat sheet.
IMPORTANT: Keep this function's value synchronized with the
docstring of `w3m-download-queue-mode'."
(let ((zz (lambda (x)
(key-description
(where-is-internal x w3m-download-queue-mode-map t)))))
(concat " "
(funcall zz 'w3m-download-increase-simultaneous)"/" (funcall zz 'w3m-download-decrease-simultaneous)" Adjust maximum number of simultaneous downloads.
" (funcall zz 'w3m-download-toggle-details) " Toggle a line's details (C-c C-o for all lines)
" (funcall zz 'w3m-download-toggle-pause) " Pause or re-queue an entry
" (funcall zz 'w3m-download-queue-drop) " Move an entry line down the queue (later download)
" (funcall zz 'w3m-download-queue-raise) " Move an entry line up the queue (sooner download)
" (funcall zz 'w3m-download-queue-raise) " Move an entry line to top of queue (first to download)
" (funcall zz 'w3m-download-queue-bottom) " Move an entry line to bottom of queue (last to download)
" (funcall zz 'w3m-download-delete-line) " Delete an entry line (file or fragment will remain on disk)
" (funcall zz 'w3m-download-refresh-buffer) " Refresh this buffer now (C-u "
(funcall zz 'w3m-download-refresh-buffer)" to set auto-refresh rate)
" (funcall zz 'w3m-download-buffer-quit) " Close this buffer (or just kill the buffer)")))
(defun w3m-download-queue-mode ()
"Major mode for viewing and editing the `w3m-download' queue.
\\[w3m-download-toggle-pause] Toggle pause state of the download.
\\[w3m-download-queue-drop] Move current line down the queue.
\\[w3m-download-queue-raise] Move current line up the queue.
\\[w3m-download-queue-top] Move current line to the top of the queue.
\\[w3m-download-queue-bottom] Move current to the bottom of the queue.
\\[w3m-download-delete-line] Delete current line.
\\[w3m-download-increase-simultaneous] / \\[w3m-download-decrease-simultaneous] \
Increase /Decrease numner of parallel downloads.
\\[w3m-download-toggle-details] Toggle a line's details (\\[w3m-download-toggle-all-details] for all lines)
\\[w3m-download-toggle-line-wrap] Toggle `visual-line-mode' for the buffer.
\\[w3m-download-refresh-buffer] Refresh (update) the buffer contents.
\\[universal-argument] \\[w3m-download-refresh-buffer]\
Change the buffer auto-refresh interval.
\\[w3m-download-buffer-quit] Quit.\n\n
\\{w3m-download-queue-mode-map}\n\n"
; Note that this mode shares functions with `w3m-download-select-mode'.
(setq
mode-name "w3m download queue"
header-line-format "w3m download queue -- (running, queued, paused, failed, and completed)"
truncate-lines t
major-mode 'w3m-download-queue-mode
buffer-read-only t
buffer-invisibility-spec (list 'yes))
(face-remap-add-relative 'header-line '((default :inherit default)))
(cursor-intangible-mode)
(buffer-disable-undo)
(use-local-map w3m-download-queue-mode-map)
(setq w3m--download-refresh-buffer-timer
(run-with-timer
w3m-download-refresh-interval
w3m-download-refresh-interval
'w3m--download-refresh-buffer))
(add-hook 'pre-command-hook 'w3m--download-update-faces-pre-command t t)
(add-hook 'post-command-hook 'w3m--download-update-faces-post-command t t)
(add-hook 'post-command-hook 'w3m--download-update-statistics t t)
(add-hook 'kill-buffer-hook 'w3m--download-queue-buffer-kill t t))
(defun w3m--download-load-lists (&optional file)
;; FIXME !!
"Retrieve the lists of downloads from `w3m-download-save-file' or FILE.
The saved lists are `w3m--download-queued',
`w3m--download-running', `w3m--download-paused',
`w3m--download-failed', and `w3m--download-completed'."
(let ((all-lists (w3m-load-list (or file w3m-download-save-file)))
one-list)
(when all-lists
(dolist (this-list '(w3m--download-queued
; w3m--download-running (merged with queued)
w3m--download-paused
w3m--download-failed
w3m--download-completed))
(setq one-list (pop all-lists))
(when (and (not (eval this-list)) one-list)
(eval `(setq ,this-list (quote ,one-list))))))))
(defun w3m--download-save-lists (&optional file)
"Save the lists of downloads to `w3m-download-save-file' or FILE.
The saved lists are `w3m--download-queued',
`w3m--download-running', `w3m--download-paused',
`w3m--download-failed', and `w3m--download-completed'."
(with-mutex w3m--download-mutex
(w3m-save-list
(or file w3m-download-save-file)
(list
; merge 'running' into 'queued'
`( ,@(mapcar
(lambda (x) `( ,@(butlast x 2)))
w3m--download-running)
,@w3m--download-queued)
w3m--download-paused
w3m--download-failed
w3m--download-completed))))
(defun w3m--download-update-progress ()
"Collect latest progress information from all running downloads."
(with-mutex w3m--download-mutex
(dolist (entry w3m--download-running)
(when (and (< 8 (length entry))
(buffer-live-p (nth 8 entry)))
(with-current-buffer (nth 8 entry)
(let ((url (nth 0 entry))
regex txt)
(if (string= (car w3m--download-cmd) "wget")
(setq regex w3m--download-progress-regex-wget)
(goto-char (point-max))
(backward-char 2)
(setq regex w3m--download-progress-regex-youtube-dl))
(setq txt (buffer-substring-no-properties
(line-beginning-position) (point-max)))
(when (string-match regex txt)
(setq txt
(format "%s, %s, %s, %s"
(or (match-string 1 txt) "??.?%")
(match-string 2 txt)
(match-string 3 txt)
(match-string 4 txt)))
(setf (nth 6 entry) txt))))))))
(defun w3m--download-progress-percent-display ()
"Colorize running queue entries based upon their % completion.
This is a primitve. It expects the current buffer and point to be
prepared! This function is meant to be called by
`w3m--download-display-queue-list'."
(let ((win-width (window-width))
(pos-1 (point-min)) ;;(next-single-property-change (point-min) 'url nil (point-max)))
(pos-2 0)
percent cols)
(while (and (< pos-1 (point-max))
(setq pos-2
(next-single-property-change pos-1 'url nil (point-max)))
(memq (get-text-property pos-1 'state)
'(w3m--download-running w3m--download-paused)))
(goto-char pos-1)
(if (not (and (re-search-forward "^ +\\([0-9.]+\\)" pos-2 t)
(setq percent
(string-to-number
(replace-regexp-in-string