88 |
88 |
89 unsigned int bi_max_vecs; /* max bvl_vecs we can hold */ |
89 unsigned int bi_max_vecs; /* max bvl_vecs we can hold */ |
90 |
90 |
91 unsigned int bi_comp_cpu; /* completion CPU */ |
91 unsigned int bi_comp_cpu; /* completion CPU */ |
92 |
92 |
|
93 atomic_t bi_cnt; /* pin count */ |
|
94 |
93 struct bio_vec *bi_io_vec; /* the actual vec list */ |
95 struct bio_vec *bi_io_vec; /* the actual vec list */ |
94 |
96 |
95 bio_end_io_t *bi_end_io; |
97 bio_end_io_t *bi_end_io; |
96 atomic_t bi_cnt; /* pin count */ |
|
97 |
98 |
98 void *bi_private; |
99 void *bi_private; |
99 #if defined(CONFIG_BLK_DEV_INTEGRITY) |
100 #if defined(CONFIG_BLK_DEV_INTEGRITY) |
100 struct bio_integrity_payload *bi_integrity; /* data integrity */ |
101 struct bio_integrity_payload *bi_integrity; /* data integrity */ |
101 #endif |
102 #endif |
102 |
103 |
103 bio_destructor_t *bi_destructor; /* destructor */ |
104 bio_destructor_t *bi_destructor; /* destructor */ |
|
105 |
|
106 /* |
|
107 * We can inline a number of vecs at the end of the bio, to avoid |
|
108 * double allocations for a small number of bio_vecs. This member |
|
109 * MUST obviously be kept at the very end of the bio. |
|
110 */ |
|
111 struct bio_vec bi_inline_vecs[0]; |
104 }; |
112 }; |
105 |
113 |
106 /* |
114 /* |
107 * bio flags |
115 * bio flags |
108 */ |
116 */ |
115 #define BIO_USER_MAPPED 6 /* contains user pages */ |
123 #define BIO_USER_MAPPED 6 /* contains user pages */ |
116 #define BIO_EOPNOTSUPP 7 /* not supported */ |
124 #define BIO_EOPNOTSUPP 7 /* not supported */ |
117 #define BIO_CPU_AFFINE 8 /* complete bio on same CPU as submitted */ |
125 #define BIO_CPU_AFFINE 8 /* complete bio on same CPU as submitted */ |
118 #define BIO_NULL_MAPPED 9 /* contains invalid user pages */ |
126 #define BIO_NULL_MAPPED 9 /* contains invalid user pages */ |
119 #define BIO_FS_INTEGRITY 10 /* fs owns integrity data, not block layer */ |
127 #define BIO_FS_INTEGRITY 10 /* fs owns integrity data, not block layer */ |
|
128 #define BIO_QUIET 11 /* Make BIO Quiet */ |
120 #define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag))) |
129 #define bio_flagged(bio, flag) ((bio)->bi_flags & (1 << (flag))) |
121 |
130 |
122 /* |
131 /* |
123 * top 4 bits of bio flags indicate the pool this bio came from |
132 * top 4 bits of bio flags indicate the pool this bio came from |
124 */ |
133 */ |
133 * bit 0 -- data direction |
142 * bit 0 -- data direction |
134 * If not set, bio is a read from device. If set, it's a write to device. |
143 * If not set, bio is a read from device. If set, it's a write to device. |
135 * bit 1 -- rw-ahead when set |
144 * bit 1 -- rw-ahead when set |
136 * bit 2 -- barrier |
145 * bit 2 -- barrier |
137 * Insert a serialization point in the IO queue, forcing previously |
146 * Insert a serialization point in the IO queue, forcing previously |
138 * submitted IO to be completed before this oen is issued. |
147 * submitted IO to be completed before this one is issued. |
139 * bit 3 -- synchronous I/O hint: the block layer will unplug immediately |
148 * bit 3 -- synchronous I/O hint: the block layer will unplug immediately |
140 * Note that this does NOT indicate that the IO itself is sync, just |
149 * Note that this does NOT indicate that the IO itself is sync, just |
141 * that the block layer will not postpone issue of this IO by plugging. |
150 * that the block layer will not postpone issue of this IO by plugging. |
142 * bit 4 -- metadata request |
151 * bit 4 -- metadata request |
143 * Used for tracing to differentiate metadata and data IO. May also |
152 * Used for tracing to differentiate metadata and data IO. May also |
152 * Don't want driver retries for any fast fail whatever the reason. |
161 * Don't want driver retries for any fast fail whatever the reason. |
153 */ |
162 */ |
154 #define BIO_RW 0 /* Must match RW in req flags (blkdev.h) */ |
163 #define BIO_RW 0 /* Must match RW in req flags (blkdev.h) */ |
155 #define BIO_RW_AHEAD 1 /* Must match FAILFAST in req flags */ |
164 #define BIO_RW_AHEAD 1 /* Must match FAILFAST in req flags */ |
156 #define BIO_RW_BARRIER 2 |
165 #define BIO_RW_BARRIER 2 |
157 #define BIO_RW_SYNC 3 |
166 #define BIO_RW_SYNCIO 3 |
158 #define BIO_RW_META 4 |
167 #define BIO_RW_UNPLUG 4 |
159 #define BIO_RW_DISCARD 5 |
168 #define BIO_RW_META 5 |
160 #define BIO_RW_FAILFAST_DEV 6 |
169 #define BIO_RW_DISCARD 6 |
161 #define BIO_RW_FAILFAST_TRANSPORT 7 |
170 #define BIO_RW_FAILFAST_DEV 7 |
162 #define BIO_RW_FAILFAST_DRIVER 8 |
171 #define BIO_RW_FAILFAST_TRANSPORT 8 |
|
172 #define BIO_RW_FAILFAST_DRIVER 9 |
|
173 |
|
174 #define bio_rw_flagged(bio, flag) ((bio)->bi_rw & (1 << (flag))) |
|
175 |
|
176 /* |
|
177 * Old defines, these should eventually be replaced by direct usage of |
|
178 * bio_rw_flagged() |
|
179 */ |
|
180 #define bio_barrier(bio) bio_rw_flagged(bio, BIO_RW_BARRIER) |
|
181 #define bio_sync(bio) bio_rw_flagged(bio, BIO_RW_SYNCIO) |
|
182 #define bio_unplug(bio) bio_rw_flagged(bio, BIO_RW_UNPLUG) |
|
183 #define bio_failfast_dev(bio) bio_rw_flagged(bio, BIO_RW_FAILFAST_DEV) |
|
184 #define bio_failfast_transport(bio) \ |
|
185 bio_rw_flagged(bio, BIO_RW_FAILFAST_TRANSPORT) |
|
186 #define bio_failfast_driver(bio) \ |
|
187 bio_rw_flagged(bio, BIO_RW_FAILFAST_DRIVER) |
|
188 #define bio_rw_ahead(bio) bio_rw_flagged(bio, BIO_RW_AHEAD) |
|
189 #define bio_rw_meta(bio) bio_rw_flagged(bio, BIO_RW_META) |
|
190 #define bio_discard(bio) bio_rw_flagged(bio, BIO_RW_DISCARD) |
163 |
191 |
164 /* |
192 /* |
165 * upper 16 bits of bi_rw define the io priority of this bio |
193 * upper 16 bits of bi_rw define the io priority of this bio |
166 */ |
194 */ |
167 #define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS) |
195 #define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS) |
182 #define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx) |
210 #define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx) |
183 #define bio_page(bio) bio_iovec((bio))->bv_page |
211 #define bio_page(bio) bio_iovec((bio))->bv_page |
184 #define bio_offset(bio) bio_iovec((bio))->bv_offset |
212 #define bio_offset(bio) bio_iovec((bio))->bv_offset |
185 #define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx) |
213 #define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx) |
186 #define bio_sectors(bio) ((bio)->bi_size >> 9) |
214 #define bio_sectors(bio) ((bio)->bi_size >> 9) |
187 #define bio_barrier(bio) ((bio)->bi_rw & (1 << BIO_RW_BARRIER)) |
|
188 #define bio_sync(bio) ((bio)->bi_rw & (1 << BIO_RW_SYNC)) |
|
189 #define bio_failfast_dev(bio) ((bio)->bi_rw & (1 << BIO_RW_FAILFAST_DEV)) |
|
190 #define bio_failfast_transport(bio) \ |
|
191 ((bio)->bi_rw & (1 << BIO_RW_FAILFAST_TRANSPORT)) |
|
192 #define bio_failfast_driver(bio) ((bio)->bi_rw & (1 << BIO_RW_FAILFAST_DRIVER)) |
|
193 #define bio_rw_ahead(bio) ((bio)->bi_rw & (1 << BIO_RW_AHEAD)) |
|
194 #define bio_rw_meta(bio) ((bio)->bi_rw & (1 << BIO_RW_META)) |
|
195 #define bio_discard(bio) ((bio)->bi_rw & (1 << BIO_RW_DISCARD)) |
|
196 #define bio_empty_barrier(bio) (bio_barrier(bio) && !bio_has_data(bio) && !bio_discard(bio)) |
215 #define bio_empty_barrier(bio) (bio_barrier(bio) && !bio_has_data(bio) && !bio_discard(bio)) |
197 |
216 |
198 static inline unsigned int bio_cur_sectors(struct bio *bio) |
217 static inline unsigned int bio_cur_sectors(struct bio *bio) |
199 { |
218 { |
200 if (bio->bi_vcnt) |
219 if (bio->bi_vcnt) |
296 sector_t bip_sector; /* virtual start sector */ |
320 sector_t bip_sector; /* virtual start sector */ |
297 |
321 |
298 void *bip_buf; /* generated integrity data */ |
322 void *bip_buf; /* generated integrity data */ |
299 bio_end_io_t *bip_end_io; /* saved I/O completion fn */ |
323 bio_end_io_t *bip_end_io; /* saved I/O completion fn */ |
300 |
324 |
301 int bip_error; /* saved I/O error */ |
|
302 unsigned int bip_size; |
325 unsigned int bip_size; |
303 |
326 |
304 unsigned short bip_pool; /* pool the ivec came from */ |
327 unsigned short bip_pool; /* pool the ivec came from */ |
305 unsigned short bip_vcnt; /* # of integrity bio_vecs */ |
328 unsigned short bip_vcnt; /* # of integrity bio_vecs */ |
306 unsigned short bip_idx; /* current bip_vec index */ |
329 unsigned short bip_idx; /* current bip_vec index */ |
330 int error; |
353 int error; |
331 }; |
354 }; |
332 extern struct bio_pair *bio_split(struct bio *bi, int first_sectors); |
355 extern struct bio_pair *bio_split(struct bio *bi, int first_sectors); |
333 extern void bio_pair_release(struct bio_pair *dbio); |
356 extern void bio_pair_release(struct bio_pair *dbio); |
334 |
357 |
335 extern struct bio_set *bioset_create(int, int); |
358 extern struct bio_set *bioset_create(unsigned int, unsigned int); |
336 extern void bioset_free(struct bio_set *); |
359 extern void bioset_free(struct bio_set *); |
337 |
360 |
338 extern struct bio *bio_alloc(gfp_t, int); |
361 extern struct bio *bio_alloc(gfp_t, int); |
339 extern struct bio *bio_kmalloc(gfp_t, int); |
362 extern struct bio *bio_kmalloc(gfp_t, int); |
340 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *); |
363 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *); |
375 struct rq_map_data *, struct sg_iovec *, |
398 struct rq_map_data *, struct sg_iovec *, |
376 int, int, gfp_t); |
399 int, int, gfp_t); |
377 extern int bio_uncopy_user(struct bio *); |
400 extern int bio_uncopy_user(struct bio *); |
378 void zero_fill_bio(struct bio *bio); |
401 void zero_fill_bio(struct bio *bio); |
379 extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *); |
402 extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *); |
|
403 extern void bvec_free_bs(struct bio_set *, struct bio_vec *, unsigned int); |
380 extern unsigned int bvec_nr_vecs(unsigned short idx); |
404 extern unsigned int bvec_nr_vecs(unsigned short idx); |
381 |
405 |
382 /* |
406 /* |
383 * Allow queuer to specify a completion CPU for this bio |
407 * Allow queuer to specify a completion CPU for this bio |
384 */ |
408 */ |
393 * These memory pools in turn all allocate from the bio_slab |
417 * These memory pools in turn all allocate from the bio_slab |
394 * and the bvec_slabs[]. |
418 * and the bvec_slabs[]. |
395 */ |
419 */ |
396 #define BIO_POOL_SIZE 2 |
420 #define BIO_POOL_SIZE 2 |
397 #define BIOVEC_NR_POOLS 6 |
421 #define BIOVEC_NR_POOLS 6 |
|
422 #define BIOVEC_MAX_IDX (BIOVEC_NR_POOLS - 1) |
398 |
423 |
399 struct bio_set { |
424 struct bio_set { |
|
425 struct kmem_cache *bio_slab; |
|
426 unsigned int front_pad; |
|
427 |
400 mempool_t *bio_pool; |
428 mempool_t *bio_pool; |
401 #if defined(CONFIG_BLK_DEV_INTEGRITY) |
429 #if defined(CONFIG_BLK_DEV_INTEGRITY) |
402 mempool_t *bio_integrity_pool; |
430 mempool_t *bio_integrity_pool; |
403 #endif |
431 #endif |
404 mempool_t *bvec_pools[BIOVEC_NR_POOLS]; |
432 mempool_t *bvec_pool; |
405 }; |
433 }; |
406 |
434 |
407 struct biovec_slab { |
435 struct biovec_slab { |
408 int nr_vecs; |
436 int nr_vecs; |
409 char *name; |
437 char *name; |
410 struct kmem_cache *slab; |
438 struct kmem_cache *slab; |
411 }; |
439 }; |
412 |
440 |
413 extern struct bio_set *fs_bio_set; |
441 extern struct bio_set *fs_bio_set; |
|
442 extern struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly; |
414 |
443 |
415 /* |
444 /* |
416 * a small number of entries is fine, not going to be performance critical. |
445 * a small number of entries is fine, not going to be performance critical. |
417 * basically we just need to survive |
446 * basically we just need to survive |
418 */ |
447 */ |
419 #define BIO_SPLIT_ENTRIES 2 |
448 #define BIO_SPLIT_ENTRIES 2 |
420 |
449 |
421 #ifdef CONFIG_HIGHMEM |
450 #ifdef CONFIG_HIGHMEM |
422 /* |
451 /* |
423 * remember to add offset! and never ever reenable interrupts between a |
452 * remember never ever reenable interrupts between a bvec_kmap_irq and |
424 * bvec_kmap_irq and bvec_kunmap_irq!! |
453 * bvec_kunmap_irq! |
425 * |
454 * |
426 * This function MUST be inlined - it plays with the CPU interrupt flags. |
455 * This function MUST be inlined - it plays with the CPU interrupt flags. |
427 */ |
456 */ |
428 static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags) |
457 static __always_inline char *bvec_kmap_irq(struct bio_vec *bvec, |
|
458 unsigned long *flags) |
429 { |
459 { |
430 unsigned long addr; |
460 unsigned long addr; |
431 |
461 |
432 /* |
462 /* |
433 * might not be a highmem page, but the preempt/irq count |
463 * might not be a highmem page, but the preempt/irq count |
439 BUG_ON(addr & ~PAGE_MASK); |
469 BUG_ON(addr & ~PAGE_MASK); |
440 |
470 |
441 return (char *) addr + bvec->bv_offset; |
471 return (char *) addr + bvec->bv_offset; |
442 } |
472 } |
443 |
473 |
444 static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags) |
474 static __always_inline void bvec_kunmap_irq(char *buffer, |
|
475 unsigned long *flags) |
445 { |
476 { |
446 unsigned long ptr = (unsigned long) buffer & PAGE_MASK; |
477 unsigned long ptr = (unsigned long) buffer & PAGE_MASK; |
447 |
478 |
448 kunmap_atomic((void *) ptr, KM_BIO_SRC_IRQ); |
479 kunmap_atomic((void *) ptr, KM_BIO_SRC_IRQ); |
449 local_irq_restore(*flags); |
480 local_irq_restore(*flags); |
498 extern int bio_integrity_prep(struct bio *); |
529 extern int bio_integrity_prep(struct bio *); |
499 extern void bio_integrity_endio(struct bio *, int); |
530 extern void bio_integrity_endio(struct bio *, int); |
500 extern void bio_integrity_advance(struct bio *, unsigned int); |
531 extern void bio_integrity_advance(struct bio *, unsigned int); |
501 extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); |
532 extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int); |
502 extern void bio_integrity_split(struct bio *, struct bio_pair *, int); |
533 extern void bio_integrity_split(struct bio *, struct bio_pair *, int); |
503 extern int bio_integrity_clone(struct bio *, struct bio *, struct bio_set *); |
534 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t, struct bio_set *); |
504 extern int bioset_integrity_create(struct bio_set *, int); |
535 extern int bioset_integrity_create(struct bio_set *, int); |
505 extern void bioset_integrity_free(struct bio_set *); |
536 extern void bioset_integrity_free(struct bio_set *); |
506 extern void bio_integrity_init_slab(void); |
537 extern void bio_integrity_init_slab(void); |
507 |
538 |
508 #else /* CONFIG_BLK_DEV_INTEGRITY */ |
539 #else /* CONFIG_BLK_DEV_INTEGRITY */ |
509 |
540 |
510 #define bio_integrity(a) (0) |
541 #define bio_integrity(a) (0) |
511 #define bioset_integrity_create(a, b) (0) |
542 #define bioset_integrity_create(a, b) (0) |
512 #define bio_integrity_prep(a) (0) |
543 #define bio_integrity_prep(a) (0) |
513 #define bio_integrity_enabled(a) (0) |
544 #define bio_integrity_enabled(a) (0) |
514 #define bio_integrity_clone(a, b, c) (0) |
545 #define bio_integrity_clone(a, b, c,d ) (0) |
515 #define bioset_integrity_free(a) do { } while (0) |
546 #define bioset_integrity_free(a) do { } while (0) |
516 #define bio_integrity_free(a, b) do { } while (0) |
547 #define bio_integrity_free(a, b) do { } while (0) |
517 #define bio_integrity_endio(a, b) do { } while (0) |
548 #define bio_integrity_endio(a, b) do { } while (0) |
518 #define bio_integrity_advance(a, b) do { } while (0) |
549 #define bio_integrity_advance(a, b) do { } while (0) |
519 #define bio_integrity_trim(a, b, c) do { } while (0) |
550 #define bio_integrity_trim(a, b, c) do { } while (0) |