-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathll_rw_blk.c
1618 lines (1431 loc) · 43.5 KB
/
ll_rw_blk.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*
* linux/drivers/block/ll_rw_blk.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
* Copyright (C) 1994, Karl Keyte: Added support for disk statistics
* Elevator latency, (C) 2000 Andrea Arcangeli <[email protected]> SuSE
* Queue request tables / lock, selectable elevator, Jens Axboe <[email protected]>
* kernel-doc documentation started by NeilBrown <[email protected]> - July2000
*/
/*
* This handles all read/write requests to block devices
*/
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/kernel_stat.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/config.h>
#include <linux/locks.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/init.h>
#include <linux/smp_lock.h>
#include <linux/completion.h>
#include <linux/bootmem.h>
#include <asm/system.h>
#include <asm/io.h>
#include <linux/blk.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/module.h>
/*
* MAC Floppy IWM hooks
*/
#ifdef CONFIG_MAC_FLOPPY_IWM
extern int mac_floppy_init(void);
#endif
/*
* For the allocated request tables
*/
static kmem_cache_t *request_cachep;
/*
* The "disk" task queue is used to start the actual requests
* after a plug
*/
DECLARE_TASK_QUEUE(tq_disk);
/*
* Protect the request list against multiple users..
*
* With this spinlock the Linux block IO subsystem is 100% SMP threaded
* from the IRQ event side, and almost 100% SMP threaded from the syscall
* side (we still have protect against block device array operations, and
* the do_request() side is casually still unsafe. The kernel lock protects
* this part currently.).
*
* there is a fair chance that things will work just OK if these functions
* are called with no global kernel lock held ...
*/
spinlock_t io_request_lock = SPIN_LOCK_UNLOCKED;
/* This specifies how many sectors to read ahead on the disk. */
int read_ahead[MAX_BLKDEV];
/* blk_dev_struct is:
* *request_fn
* *current_request
*/
struct blk_dev_struct blk_dev[MAX_BLKDEV]; /* initialized by blk_dev_init() */
/*
* blk_size contains the size of all block-devices in units of 1024 byte
* sectors:
*
* blk_size[MAJOR][MINOR]
*
* if (!blk_size[MAJOR]) then no minor size checking is done.
*/
int * blk_size[MAX_BLKDEV];
/*
* blksize_size contains the size of all block-devices:
*
* blksize_size[MAJOR][MINOR]
*
* if (!blksize_size[MAJOR]) then 1024 bytes is assumed.
*/
int * blksize_size[MAX_BLKDEV];
/*
* hardsect_size contains the size of the hardware sector of a device.
*
* hardsect_size[MAJOR][MINOR]
*
* if (!hardsect_size[MAJOR])
* then 512 bytes is assumed.
* else
* sector_size is hardsect_size[MAJOR][MINOR]
* This is currently set by some scsi devices and read by the msdos fs driver.
* Other uses may appear later.
*/
int * hardsect_size[MAX_BLKDEV];
/*
* The following tunes the read-ahead algorithm in mm/filemap.c
*/
int * max_readahead[MAX_BLKDEV];
/*
* Max number of sectors per request
*/
int * max_sectors[MAX_BLKDEV];
unsigned long blk_max_low_pfn, blk_max_pfn;
int blk_nohighio = 0;
static inline int get_max_sectors(kdev_t dev)
{
if (!max_sectors[MAJOR(dev)])
return MAX_SECTORS;
return max_sectors[MAJOR(dev)][MINOR(dev)];
}
inline request_queue_t *blk_get_queue(kdev_t dev)
{
struct blk_dev_struct *bdev = blk_dev + MAJOR(dev);
if (bdev->queue)
return bdev->queue(dev);
else
return &blk_dev[MAJOR(dev)].request_queue;
}
static int __blk_cleanup_queue(struct request_list *list)
{
struct list_head *head = &list->free;
struct request *rq;
int i = 0;
while (!list_empty(head)) {
rq = list_entry(head->next, struct request, queue);
list_del(&rq->queue);
kmem_cache_free(request_cachep, rq);
i++;
};
if (i != list->count)
printk("request list leak!\n");
list->count = 0;
return i;
}
/**
* blk_cleanup_queue: - release a &request_queue_t when it is no longer needed
* @q: the request queue to be released
*
* Description:
* blk_cleanup_queue is the pair to blk_init_queue(). It should
* be called when a request queue is being released; typically
* when a block device is being de-registered. Currently, its
* primary task it to free all the &struct request structures that
* were allocated to the queue.
* Caveat:
* Hopefully the low level driver will have finished any
* outstanding requests first...
**/
void blk_cleanup_queue(request_queue_t * q)
{
int count = q->nr_requests;
count -= __blk_cleanup_queue(&q->rq);
if (count)
printk("blk_cleanup_queue: leaked requests (%d)\n", count);
if (atomic_read(&q->nr_sectors))
printk("blk_cleanup_queue: leaked sectors (%d)\n", atomic_read(&q->nr_sectors));
memset(q, 0, sizeof(*q));
}
/**
* blk_queue_headactive - indicate whether head of request queue may be active
* @q: The queue which this applies to.
* @active: A flag indication where the head of the queue is active.
*
* Description:
* The driver for a block device may choose to leave the currently active
* request on the request queue, removing it only when it has completed.
* The queue handling routines assume this by default for safety reasons
* and will not involve the head of the request queue in any merging or
* reordering of requests when the queue is unplugged (and thus may be
* working on this particular request).
*
* If a driver removes requests from the queue before processing them, then
* it may indicate that it does so, there by allowing the head of the queue
* to be involved in merging and reordering. This is done be calling
* blk_queue_headactive() with an @active flag of %0.
*
* If a driver processes several requests at once, it must remove them (or
* at least all but one of them) from the request queue.
*
* When a queue is plugged the head will be assumed to be inactive.
**/
void blk_queue_headactive(request_queue_t * q, int active)
{
q->head_active = active;
}
/**
* blk_queue_throttle_sectors - indicates you will call sector throttling funcs
* @q: The queue which this applies to.
* @active: A flag indication if you want sector throttling on
*
* Description:
* The sector throttling code allows us to put a limit on the number of
* sectors pending io to the disk at a given time, sending @active nonzero
* indicates you will call blk_started_sectors and blk_finished_sectors in
* addition to calling blk_started_io and blk_finished_io in order to
* keep track of the number of sectors in flight.
**/
void blk_queue_throttle_sectors(request_queue_t * q, int active)
{
q->can_throttle = active;
}
/**
* blk_queue_make_request - define an alternate make_request function for a device
* @q: the request queue for the device to be affected
* @mfn: the alternate make_request function
*
* Description:
* The normal way for &struct buffer_heads to be passed to a device
* driver is for them to be collected into requests on a request
* queue, and then to allow the device driver to select requests
* off that queue when it is ready. This works well for many block
* devices. However some block devices (typically virtual devices
* such as md or lvm) do not benefit from the processing on the
* request queue, and are served best by having the requests passed
* directly to them. This can be achieved by providing a function
* to blk_queue_make_request().
*
* Caveat:
* The driver that does this *must* be able to deal appropriately
* with buffers in "highmemory", either by calling bh_kmap() to get
* a kernel mapping, to by calling create_bounce() to create a
* buffer in normal memory.
**/
void blk_queue_make_request(request_queue_t * q, make_request_fn * mfn)
{
q->make_request_fn = mfn;
}
/**
* blk_queue_bounce_limit - set bounce buffer limit for queue
* @q: the request queue for the device
* @dma_addr: bus address limit
*
* Description:
* Different hardware can have different requirements as to what pages
* it can do I/O directly to. A low level driver can call
* blk_queue_bounce_limit to have lower memory pages allocated as bounce
* buffers for doing I/O to pages residing above @page. By default
* the block layer sets this to the highest numbered "low" memory page.
**/
void blk_queue_bounce_limit(request_queue_t *q, u64 dma_addr)
{
unsigned long bounce_pfn = dma_addr >> PAGE_SHIFT;
unsigned long mb = dma_addr >> 20;
static request_queue_t *old_q;
/*
* keep this for debugging for now...
*/
if (dma_addr != BLK_BOUNCE_HIGH && q != old_q) {
old_q = q;
printk("blk: queue %p, ", q);
if (dma_addr == BLK_BOUNCE_ANY)
printk("no I/O memory limit\n");
else
printk("I/O limit %luMb (mask 0x%Lx)\n", mb,
(long long) dma_addr);
}
q->bounce_pfn = bounce_pfn;
}
/*
* can we merge the two segments, or do we need to start a new one?
*/
inline int blk_seg_merge_ok(struct buffer_head *bh, struct buffer_head *nxt)
{
/*
* if bh and nxt are contigous and don't cross a 4g boundary, it's ok
*/
if (BH_CONTIG(bh, nxt) && BH_PHYS_4G(bh, nxt))
return 1;
return 0;
}
static inline int ll_new_segment(request_queue_t *q, struct request *req, int max_segments)
{
if (req->nr_segments < max_segments) {
req->nr_segments++;
return 1;
}
return 0;
}
static int ll_back_merge_fn(request_queue_t *q, struct request *req,
struct buffer_head *bh, int max_segments)
{
if (blk_seg_merge_ok(req->bhtail, bh))
return 1;
return ll_new_segment(q, req, max_segments);
}
static int ll_front_merge_fn(request_queue_t *q, struct request *req,
struct buffer_head *bh, int max_segments)
{
if (blk_seg_merge_ok(bh, req->bh))
return 1;
return ll_new_segment(q, req, max_segments);
}
static int ll_merge_requests_fn(request_queue_t *q, struct request *req,
struct request *next, int max_segments)
{
int total_segments = req->nr_segments + next->nr_segments;
if (blk_seg_merge_ok(req->bhtail, next->bh))
total_segments--;
if (total_segments > max_segments)
return 0;
req->nr_segments = total_segments;
return 1;
}
/*
* "plug" the device if there are no outstanding requests: this will
* force the transfer to start only after we have put all the requests
* on the list.
*
* This is called with interrupts off and no requests on the queue.
* (and with the request spinlock acquired)
*/
static void generic_plug_device(request_queue_t *q, kdev_t dev)
{
/*
* no need to replug device
*/
if (!list_empty(&q->queue_head) || q->plugged)
return;
q->plugged = 1;
queue_task(&q->plug_tq, &tq_disk);
}
/*
* remove the plug and let it rip..
*/
static inline void __generic_unplug_device(request_queue_t *q)
{
if (q->plugged) {
q->plugged = 0;
if (!list_empty(&q->queue_head))
q->request_fn(q);
}
}
void generic_unplug_device(void *data)
{
request_queue_t *q = (request_queue_t *) data;
unsigned long flags;
spin_lock_irqsave(&io_request_lock, flags);
__generic_unplug_device(q);
spin_unlock_irqrestore(&io_request_lock, flags);
}
/** blk_grow_request_list
* @q: The &request_queue_t
* @nr_requests: how many requests are desired
*
* More free requests are added to the queue's free lists, bringing
* the total number of requests to @nr_requests.
*
* The requests are added equally to the request queue's read
* and write freelists.
*
* This function can sleep.
*
* Returns the (new) number of requests which the queue has available.
*/
int blk_grow_request_list(request_queue_t *q, int nr_requests, int max_queue_sectors)
{
unsigned long flags;
/* Several broken drivers assume that this function doesn't sleep,
* this causes system hangs during boot.
* As a temporary fix, make the function non-blocking.
*/
spin_lock_irqsave(&io_request_lock, flags);
while (q->nr_requests < nr_requests) {
struct request *rq;
rq = kmem_cache_alloc(request_cachep, SLAB_ATOMIC);
if (rq == NULL)
break;
memset(rq, 0, sizeof(*rq));
rq->rq_status = RQ_INACTIVE;
list_add(&rq->queue, &q->rq.free);
q->rq.count++;
q->nr_requests++;
}
/*
* Wakeup waiters after both one quarter of the
* max-in-fligh queue and one quarter of the requests
* are available again.
*/
q->batch_requests = q->nr_requests / 4;
if (q->batch_requests > 32)
q->batch_requests = 32;
q->batch_sectors = max_queue_sectors / 4;
q->max_queue_sectors = max_queue_sectors;
BUG_ON(!q->batch_sectors);
atomic_set(&q->nr_sectors, 0);
spin_unlock_irqrestore(&io_request_lock, flags);
return q->nr_requests;
}
static void blk_init_free_list(request_queue_t *q)
{
struct sysinfo si;
int megs; /* Total memory, in megabytes */
int nr_requests, max_queue_sectors = MAX_QUEUE_SECTORS;
INIT_LIST_HEAD(&q->rq.free);
q->rq.count = 0;
q->rq.pending[READ] = q->rq.pending[WRITE] = 0;
q->nr_requests = 0;
si_meminfo(&si);
megs = si.totalram >> (20 - PAGE_SHIFT);
nr_requests = MAX_NR_REQUESTS;
if (megs < 30) {
nr_requests /= 2;
max_queue_sectors /= 2;
}
/* notice early if anybody screwed the defaults */
BUG_ON(!nr_requests);
BUG_ON(!max_queue_sectors);
blk_grow_request_list(q, nr_requests, max_queue_sectors);
init_waitqueue_head(&q->wait_for_requests);
spin_lock_init(&q->queue_lock);
}
static int __make_request(request_queue_t * q, int rw, struct buffer_head * bh);
/**
* blk_init_queue - prepare a request queue for use with a block device
* @q: The &request_queue_t to be initialised
* @rfn: The function to be called to process requests that have been
* placed on the queue.
*
* Description:
* If a block device wishes to use the standard request handling procedures,
* which sorts requests and coalesces adjacent requests, then it must
* call blk_init_queue(). The function @rfn will be called when there
* are requests on the queue that need to be processed. If the device
* supports plugging, then @rfn may not be called immediately when requests
* are available on the queue, but may be called at some time later instead.
* Plugged queues are generally unplugged when a buffer belonging to one
* of the requests on the queue is needed, or due to memory pressure.
*
* @rfn is not required, or even expected, to remove all requests off the
* queue, but only as many as it can handle at a time. If it does leave
* requests on the queue, it is responsible for arranging that the requests
* get dealt with eventually.
*
* A global spin lock $io_request_lock must be held while manipulating the
* requests on the request queue.
*
* The request on the head of the queue is by default assumed to be
* potentially active, and it is not considered for re-ordering or merging
* whenever the given queue is unplugged. This behaviour can be changed with
* blk_queue_headactive().
*
* Note:
* blk_init_queue() must be paired with a blk_cleanup_queue() call
* when the block device is deactivated (such as at module unload).
**/
void blk_init_queue(request_queue_t * q, request_fn_proc * rfn)
{
INIT_LIST_HEAD(&q->queue_head);
elevator_init(&q->elevator, ELEVATOR_LINUS);
blk_init_free_list(q);
q->request_fn = rfn;
q->back_merge_fn = ll_back_merge_fn;
q->front_merge_fn = ll_front_merge_fn;
q->merge_requests_fn = ll_merge_requests_fn;
q->make_request_fn = __make_request;
q->plug_tq.sync = 0;
q->plug_tq.routine = &generic_unplug_device;
q->plug_tq.data = q;
q->plugged = 0;
q->can_throttle = 0;
/*
* These booleans describe the queue properties. We set the
* default (and most common) values here. Other drivers can
* use the appropriate functions to alter the queue properties.
* as appropriate.
*/
q->plug_device_fn = generic_plug_device;
q->head_active = 1;
blk_queue_bounce_limit(q, BLK_BOUNCE_HIGH);
}
#define blkdev_free_rq(list) list_entry((list)->next, struct request, queue);
/*
* Get a free request. io_request_lock must be held and interrupts
* disabled on the way in. Returns NULL if there are no free requests.
*/
static struct request *get_request(request_queue_t *q, int rw)
{
struct request *rq = NULL;
struct request_list *rl = &q->rq;
if (blk_oversized_queue(q)) {
int rlim = q->nr_requests >> 5;
if (rlim < 4)
rlim = 4;
/*
* if its a write, or we have more than a handful of reads
* pending, bail out
*/
if ((rw == WRITE) || (rw == READ && rl->pending[READ] > rlim))
return NULL;
if (blk_oversized_queue_reads(q))
return NULL;
}
if (!list_empty(&rl->free)) {
rq = blkdev_free_rq(&rl->free);
list_del(&rq->queue);
rl->count--;
rl->pending[rw]++;
rq->rq_status = RQ_ACTIVE;
rq->cmd = rw;
rq->special = NULL;
rq->q = q;
}
return rq;
}
/*
* Here's the request allocation design, low latency version:
*
* 1: Blocking on request exhaustion is a key part of I/O throttling.
*
* 2: We want to be `fair' to all requesters. We must avoid starvation, and
* attempt to ensure that all requesters sleep for a similar duration. Hence
* no stealing requests when there are other processes waiting.
*
* There used to be more here, attempting to allow a process to send in a
* number of requests once it has woken up. But, there's no way to
* tell if a process has just been woken up, or if it is a new process
* coming in to steal requests from the waiters. So, we give up and force
* everyone to wait fairly.
*
* So here's what we do:
*
* a) A READA requester fails if free_requests < batch_requests
*
* We don't want READA requests to prevent sleepers from ever
* waking. Note that READA is used extremely rarely - a few
* filesystems use it for directory readahead.
*
* When a process wants a new request:
*
* b) If free_requests == 0, the requester sleeps in FIFO manner, and
* the queue full condition is set. The full condition is not
* cleared until there are no longer any waiters. Once the full
* condition is set, all new io must wait, hopefully for a very
* short period of time.
*
* When a request is released:
*
* c) If free_requests < batch_requests, do nothing.
*
* d) If free_requests >= batch_requests, wake up a single waiter.
*
* As each waiter gets a request, he wakes another waiter. We do this
* to prevent a race where an unplug might get run before a request makes
* it's way onto the queue. The result is a cascade of wakeups, so delaying
* the initial wakeup until we've got batch_requests available helps avoid
* wakeups where there aren't any requests available yet.
*/
static struct request *__get_request_wait(request_queue_t *q, int rw)
{
register struct request *rq;
DECLARE_WAITQUEUE(wait, current);
add_wait_queue_exclusive(&q->wait_for_requests, &wait);
do {
set_current_state(TASK_UNINTERRUPTIBLE);
spin_lock_irq(&io_request_lock);
if (blk_oversized_queue(q) || q->rq.count == 0) {
__generic_unplug_device(q);
spin_unlock_irq(&io_request_lock);
schedule();
spin_lock_irq(&io_request_lock);
}
rq = get_request(q, rw);
spin_unlock_irq(&io_request_lock);
} while (rq == NULL);
remove_wait_queue(&q->wait_for_requests, &wait);
current->state = TASK_RUNNING;
return rq;
}
static void get_request_wait_wakeup(request_queue_t *q, int rw)
{
/*
* avoid losing an unplug if a second __get_request_wait did the
* generic_unplug_device while our __get_request_wait was running
* w/o the queue_lock held and w/ our request out of the queue.
*/
if (waitqueue_active(&q->wait_for_requests))
wake_up(&q->wait_for_requests);
}
/* RO fail safe mechanism */
static long ro_bits[MAX_BLKDEV][8];
int is_read_only(kdev_t dev)
{
int minor,major;
major = MAJOR(dev);
minor = MINOR(dev);
if (major < 0 || major >= MAX_BLKDEV) return 0;
return ro_bits[major][minor >> 5] & (1 << (minor & 31));
}
void set_device_ro(kdev_t dev,int flag)
{
int minor,major;
major = MAJOR(dev);
minor = MINOR(dev);
if (major < 0 || major >= MAX_BLKDEV) return;
if (flag) ro_bits[major][minor >> 5] |= 1 << (minor & 31);
else ro_bits[major][minor >> 5] &= ~(1 << (minor & 31));
}
inline void drive_stat_acct (kdev_t dev, int rw,
unsigned long nr_sectors, int new_io)
{
unsigned int major = MAJOR(dev);
unsigned int index;
index = disk_index(dev);
if ((index >= DK_MAX_DISK) || (major >= DK_MAX_MAJOR))
return;
kstat.dk_drive[major][index] += new_io;
if (rw == READ) {
kstat.dk_drive_rio[major][index] += new_io;
kstat.dk_drive_rblk[major][index] += nr_sectors;
} else if (rw == WRITE) {
kstat.dk_drive_wio[major][index] += new_io;
kstat.dk_drive_wblk[major][index] += nr_sectors;
} else
printk(KERN_ERR "drive_stat_acct: cmd not R/W?\n");
}
#ifdef CONFIG_BLK_STATS
/*
* Return up to two hd_structs on which to do IO accounting for a given
* request.
*
* On a partitioned device, we want to account both against the partition
* and against the whole disk.
*/
static void locate_hd_struct(struct request *req,
struct hd_struct **hd1,
struct hd_struct **hd2)
{
struct gendisk *gd;
*hd1 = NULL;
*hd2 = NULL;
gd = get_gendisk(req->rq_dev);
if (gd && gd->part) {
/* Mask out the partition bits: account for the entire disk */
int devnr = MINOR(req->rq_dev) >> gd->minor_shift;
int whole_minor = devnr << gd->minor_shift;
*hd1 = &gd->part[whole_minor];
if (whole_minor != MINOR(req->rq_dev))
*hd2= &gd->part[MINOR(req->rq_dev)];
}
}
/*
* Round off the performance stats on an hd_struct.
*
* The average IO queue length and utilisation statistics are maintained
* by observing the current state of the queue length and the amount of
* time it has been in this state for.
* Normally, that accounting is done on IO completion, but that can result
* in more than a second's worth of IO being accounted for within any one
* second, leading to >100% utilisation. To deal with that, we do a
* round-off before returning the results when reading /proc/partitions,
* accounting immediately for all queue usage up to the current jiffies and
* restarting the counters again.
*/
void disk_round_stats(struct hd_struct *hd)
{
unsigned long now = jiffies;
hd->aveq += (hd->ios_in_flight * (jiffies - hd->last_queue_change));
hd->last_queue_change = now;
if (hd->ios_in_flight)
hd->io_ticks += (now - hd->last_idle_time);
hd->last_idle_time = now;
}
static inline void down_ios(struct hd_struct *hd)
{
disk_round_stats(hd);
--hd->ios_in_flight;
}
static inline void up_ios(struct hd_struct *hd)
{
disk_round_stats(hd);
++hd->ios_in_flight;
}
static void account_io_start(struct hd_struct *hd, struct request *req,
int merge, int sectors)
{
switch (req->cmd) {
case READ:
if (merge)
hd->rd_merges++;
hd->rd_sectors += sectors;
break;
case WRITE:
if (merge)
hd->wr_merges++;
hd->wr_sectors += sectors;
break;
}
if (!merge)
up_ios(hd);
}
static void account_io_end(struct hd_struct *hd, struct request *req)
{
unsigned long duration = jiffies - req->start_time;
switch (req->cmd) {
case READ:
hd->rd_ticks += duration;
hd->rd_ios++;
break;
case WRITE:
hd->wr_ticks += duration;
hd->wr_ios++;
break;
}
down_ios(hd);
}
void req_new_io(struct request *req, int merge, int sectors)
{
struct hd_struct *hd1, *hd2;
locate_hd_struct(req, &hd1, &hd2);
if (hd1)
account_io_start(hd1, req, merge, sectors);
if (hd2)
account_io_start(hd2, req, merge, sectors);
}
void req_merged_io(struct request *req)
{
struct hd_struct *hd1, *hd2;
locate_hd_struct(req, &hd1, &hd2);
if (hd1)
down_ios(hd1);
if (hd2)
down_ios(hd2);
}
void req_finished_io(struct request *req)
{
struct hd_struct *hd1, *hd2;
locate_hd_struct(req, &hd1, &hd2);
if (hd1)
account_io_end(hd1, req);
if (hd2)
account_io_end(hd2, req);
}
EXPORT_SYMBOL(req_finished_io);
#endif /* CONFIG_BLK_STATS */
/*
* add-request adds a request to the linked list.
* io_request_lock is held and interrupts disabled, as we muck with the
* request queue list.
*
* By this point, req->cmd is always either READ/WRITE, never READA,
* which is important for drive_stat_acct() above.
*/
static inline void add_request(request_queue_t * q, struct request * req,
struct list_head *insert_here)
{
drive_stat_acct(req->rq_dev, req->cmd, req->nr_sectors, 1);
if (!q->plugged && q->head_active && insert_here == &q->queue_head) {
spin_unlock_irq(&io_request_lock);
BUG();
}
/*
* elevator indicated where it wants this request to be
* inserted at elevator_merge time
*/
list_add(&req->queue, insert_here);
}
/*
* Must be called with io_request_lock held and interrupts disabled
*/
void blkdev_release_request(struct request *req)
{
request_queue_t *q = req->q;
req->rq_status = RQ_INACTIVE;
req->q = NULL;
/*
* Request may not have originated from ll_rw_blk. if not,
* assume it has free buffers and check waiters
*/
if (q) {
struct request_list *rl = &q->rq;
int oversized_batch = 0;
if (q->can_throttle)
oversized_batch = blk_oversized_queue_batch(q);
rl->count++;
/*
* paranoia check
*/
if (req->cmd == READ || req->cmd == WRITE)
rl->pending[req->cmd]--;
if (rl->pending[READ] > q->nr_requests)
printk("blk: reads: %u\n", rl->pending[READ]);
if (rl->pending[WRITE] > q->nr_requests)
printk("blk: writes: %u\n", rl->pending[WRITE]);
if (rl->pending[READ] + rl->pending[WRITE] > q->nr_requests)
printk("blk: r/w: %u + %u > %u\n", rl->pending[READ], rl->pending[WRITE], q->nr_requests);
list_add(&req->queue, &rl->free);
if (rl->count >= q->batch_requests && !oversized_batch) {
smp_mb();
if (waitqueue_active(&q->wait_for_requests))
wake_up(&q->wait_for_requests);
}
}
}
/*
* Has to be called with the request spinlock acquired
*/
static void attempt_merge(request_queue_t * q,
struct request *req,
int max_sectors,
int max_segments)
{
struct request *next;
next = blkdev_next_request(req);
if (req->sector + req->nr_sectors != next->sector)
return;
if (req->cmd != next->cmd
|| req->rq_dev != next->rq_dev
|| req->nr_sectors + next->nr_sectors > max_sectors
|| next->waiting)
return;
/*
* If we are not allowed to merge these requests, then
* return. If we are allowed to merge, then the count
* will have been updated to the appropriate number,
* and we shouldn't do it here too.
*/
if (!q->merge_requests_fn(q, req, next, max_segments))
return;
q->elevator.elevator_merge_req_fn(req, next);
req->bhtail->b_reqnext = next->bh;
req->bhtail = next->bhtail;
req->nr_sectors = req->hard_nr_sectors += next->hard_nr_sectors;
list_del(&next->queue);
/* One last thing: we have removed a request, so we now have one
less expected IO to complete for accounting purposes. */
req_merged_io(req);
blkdev_release_request(next);
}
static inline void attempt_back_merge(request_queue_t * q,
struct request *req,
int max_sectors,
int max_segments)
{
if (&req->queue == q->queue_head.prev)
return;
attempt_merge(q, req, max_sectors, max_segments);
}
static inline void attempt_front_merge(request_queue_t * q,
struct list_head * head,
struct request *req,
int max_sectors,
int max_segments)
{
struct list_head * prev;
prev = req->queue.prev;
if (head == prev)
return;
attempt_merge(q, blkdev_entry_to_request(prev), max_sectors, max_segments);
}
static int __make_request(request_queue_t * q, int rw,
struct buffer_head * bh)
{
unsigned int sector, count, sync;
int max_segments = MAX_SEGMENTS;
struct request * req, *freereq = NULL;
int rw_ahead, max_sectors, el_ret;
struct list_head *head, *insert_here;
int latency;
elevator_t *elevator = &q->elevator;
int should_wake = 0;
count = bh->b_size >> 9;
sector = bh->b_rsector;
sync = test_and_clear_bit(BH_Sync, &bh->b_state);
rw_ahead = 0; /* normal case; gets changed below for READA */
switch (rw) {
case READA:
#if 0 /* bread() misinterprets failed READA attempts as IO errors on SMP */
rw_ahead = 1;
#endif
rw = READ; /* drop into READ */
case READ: