1 // SPDX-License-Identifier: GPL-2.0
12 #include "async-thread.h"
13 #include "dev-replace.h"
15 #include "file-item.h"
16 #include "raid-stripe-tree.h"
29 /* Is this a data path I/O that needs storage layer checksum and repair? */
32 	return bbio->inode && is_data_inode(bbio->inode);  in is_data_bbio()
37 	return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE;  in bbio_has_ordered_extent()
48 	bbio->fs_info = fs_info;  in btrfs_bio_init()
49 	bbio->end_io = end_io;  in btrfs_bio_init()
50 	bbio->private = private;  in btrfs_bio_init()
51 	atomic_set(&bbio->pending_ios, 1);  in btrfs_bio_init()
52 	WRITE_ONCE(bbio->status, BLK_STS_OK);  in btrfs_bio_init()
82 	bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT, GFP_NOFS,  in btrfs_split_bio()
89 	bbio->inode = orig_bbio->inode;  in btrfs_split_bio()
90 	bbio->file_offset = orig_bbio->file_offset;  in btrfs_split_bio()
91 	orig_bbio->file_offset += map_length;  in btrfs_split_bio()
93 		refcount_inc(&orig_bbio->ordered->refs);  in btrfs_split_bio()
94 		bbio->ordered = orig_bbio->ordered;  in btrfs_split_bio()
96 	atomic_inc(&orig_bbio->pending_ios);  in btrfs_split_bio()
104 		btrfs_put_ordered_extent(bbio->ordered);  in btrfs_cleanup_bio()
105 	bio_put(&bbio->bio);  in btrfs_cleanup_bio()
111 		struct btrfs_ordered_extent *ordered = bbio->ordered;  in __btrfs_bio_end_io()
113 		bbio->end_io(bbio);  in __btrfs_bio_end_io()
116 		bbio->end_io(bbio);  in __btrfs_bio_end_io()
122 	bbio->bio.bi_status = status;  in btrfs_bio_end_io()
123 	if (bbio->bio.bi_pool == &btrfs_clone_bioset) {  in btrfs_bio_end_io()
124 		struct btrfs_bio *orig_bbio = bbio->private;  in btrfs_bio_end_io()
135 		cmpxchg(&bbio->status, BLK_STS_OK, status);  in btrfs_bio_end_io()
137 	if (atomic_dec_and_test(&bbio->pending_ios)) {  in btrfs_bio_end_io()
140 			bbio->bio.bi_status = READ_ONCE(bbio->status);  in btrfs_bio_end_io()
147 	if (cur_mirror == fbio->num_copies)  in next_repair_mirror()
148 		return cur_mirror + 1 - fbio->num_copies;  in next_repair_mirror()
155 		return fbio->num_copies;  in prev_repair_mirror()
156 	return cur_mirror - 1;  in prev_repair_mirror()
161 	if (atomic_dec_and_test(&fbio->repair_count)) {  in btrfs_repair_done()
162 		btrfs_bio_end_io(fbio->bbio, fbio->bbio->bio.bi_status);  in btrfs_repair_done()
170 	struct btrfs_failed_bio *fbio = repair_bbio->private;  in btrfs_end_repair_bio()
171 	struct btrfs_inode *inode = repair_bbio->inode;  in btrfs_end_repair_bio()
172 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in btrfs_end_repair_bio()
173 	struct bio_vec *bv = bio_first_bvec_all(&repair_bbio->bio);  in btrfs_end_repair_bio()
174 	int mirror = repair_bbio->mirror_num;  in btrfs_end_repair_bio()  local
177 	 * We can only trigger this for data bio, which doesn't support larger  in btrfs_end_repair_bio()
180 	ASSERT(folio_order(page_folio(bv->bv_page)) == 0);  in btrfs_end_repair_bio()
182 	if (repair_bbio->bio.bi_status ||  in btrfs_end_repair_bio()
184 		bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ);  in btrfs_end_repair_bio()
185 		repair_bbio->bio.bi_iter = repair_bbio->saved_iter;  in btrfs_end_repair_bio()
187 		mirror = next_repair_mirror(fbio, mirror);  in btrfs_end_repair_bio()
188 		if (mirror == fbio->bbio->mirror_num) {  in btrfs_end_repair_bio()
189 			btrfs_debug(fs_info, "no mirror left");  in btrfs_end_repair_bio()
190 			fbio->bbio->bio.bi_status = BLK_STS_IOERR;  in btrfs_end_repair_bio()
194 		btrfs_submit_bbio(repair_bbio, mirror);  in btrfs_end_repair_bio()
199 		mirror = prev_repair_mirror(fbio, mirror);  in btrfs_end_repair_bio()
201 				  repair_bbio->file_offset, fs_info->sectorsize,  in btrfs_end_repair_bio()
202 				  repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT,  in btrfs_end_repair_bio()
203 				  page_folio(bv->bv_page), bv->bv_offset, mirror);  in btrfs_end_repair_bio()
204 	} while (mirror != fbio->bbio->mirror_num);  in btrfs_end_repair_bio()
208 	bio_put(&repair_bbio->bio);  in btrfs_end_repair_bio()
212  * Try to kick off a repair read to the next available mirror for a bad sector.
214  * This primarily tries to recover good data to serve the actual read request,
215  * but also tries to write the good data back to the bad mirror(s) when a
223 	struct btrfs_inode *inode = failed_bbio->inode;  in repair_one_sector()
224 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in repair_one_sector()
225 	const u32 sectorsize = fs_info->sectorsize;  in repair_one_sector()
226 	const u64 logical = (failed_bbio->saved_iter.bi_sector << SECTOR_SHIFT);  in repair_one_sector()
230 	int mirror;  in repair_one_sector()  local
233 		    failed_bbio->file_offset + bio_offset);  in repair_one_sector()
238 		failed_bbio->bio.bi_status = BLK_STS_IOERR;  in repair_one_sector()
244 		fbio->bbio = failed_bbio;  in repair_one_sector()
245 		fbio->num_copies = num_copies;  in repair_one_sector()
246 		atomic_set(&fbio->repair_count, 1);  in repair_one_sector()
249 	atomic_inc(&fbio->repair_count);  in repair_one_sector()
253 	repair_bio->bi_iter.bi_sector = failed_bbio->saved_iter.bi_sector;  in repair_one_sector()
254 	__bio_add_page(repair_bio, bv->bv_page, bv->bv_len, bv->bv_offset);  in repair_one_sector()
258 	repair_bbio->inode = failed_bbio->inode;  in repair_one_sector()
259 	repair_bbio->file_offset = failed_bbio->file_offset + bio_offset;  in repair_one_sector()
261 	mirror = next_repair_mirror(fbio, failed_bbio->mirror_num);  in repair_one_sector()
262 	btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror);  in repair_one_sector()
263 	btrfs_submit_bbio(repair_bbio, mirror);  in repair_one_sector()
269 	struct btrfs_inode *inode = bbio->inode;  in btrfs_check_read_bio()
270 	struct btrfs_fs_info *fs_info = inode->root->fs_info;  in btrfs_check_read_bio()
271 	u32 sectorsize = fs_info->sectorsize;  in btrfs_check_read_bio()
272 	struct bvec_iter *iter = &bbio->saved_iter;  in btrfs_check_read_bio()
273 	blk_status_t status = bbio->bio.bi_status;  in btrfs_check_read_bio()
277 	/* Read-repair requires the inode field to be set by the submitter. */  in btrfs_check_read_bio()
284 	if (bbio->bio.bi_pool == &btrfs_repair_bioset) {  in btrfs_check_read_bio()
290 	bbio->bio.bi_status = BLK_STS_OK;  in btrfs_check_read_bio()
292 	while (iter->bi_size) {  in btrfs_check_read_bio()
293 		struct bio_vec bv = bio_iter_iovec(&bbio->bio, *iter);  in btrfs_check_read_bio()
299 		bio_advance_iter_single(&bbio->bio, iter, sectorsize);  in btrfs_check_read_bio()
303 	if (bbio->csum != bbio->csum_inline)  in btrfs_check_read_bio()
304 		kfree(bbio->csum);  in btrfs_check_read_bio()
309 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_check_read_bio()
314 	if (!dev || !dev->bdev)  in btrfs_log_dev_io_error()
316 	if (bio->bi_status != BLK_STS_IOERR && bio->bi_status != BLK_STS_TARGET)  in btrfs_log_dev_io_error()
321 	else if (!(bio->bi_opf & REQ_RAHEAD))  in btrfs_log_dev_io_error()
323 	if (bio->bi_opf & REQ_PREFLUSH)  in btrfs_log_dev_io_error()
330 	if (bio->bi_opf & REQ_META)  in btrfs_end_io_wq()
331 		return fs_info->endio_meta_workers;  in btrfs_end_io_wq()
332 	return fs_info->endio_workers;  in btrfs_end_io_wq()
341 		btrfs_check_read_bio(bbio, bbio->bio.bi_private);  in btrfs_end_bio_work()
343 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_end_bio_work()
349 	struct btrfs_device *dev = bio->bi_private;  in btrfs_simple_end_io()
350 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_simple_end_io()
354 	if (bio->bi_status)  in btrfs_simple_end_io()
358 		INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work);  in btrfs_simple_end_io()
359 		queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work);  in btrfs_simple_end_io()
361 		if (bio_is_zone_append(bio) && !bio->bi_status)  in btrfs_simple_end_io()
363 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_simple_end_io()
369 	struct btrfs_io_context *bioc = bio->bi_private;  in btrfs_raid56_end_io()
372 	btrfs_bio_counter_dec(bioc->fs_info);  in btrfs_raid56_end_io()
373 	bbio->mirror_num = bioc->mirror_num;  in btrfs_raid56_end_io()
377 		btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_raid56_end_io()
384 	struct btrfs_io_stripe *stripe = bio->bi_private;  in btrfs_orig_write_end_io()
385 	struct btrfs_io_context *bioc = stripe->bioc;  in btrfs_orig_write_end_io()
388 	btrfs_bio_counter_dec(bioc->fs_info);  in btrfs_orig_write_end_io()
390 	if (bio->bi_status) {  in btrfs_orig_write_end_io()
391 		atomic_inc(&bioc->error);  in btrfs_orig_write_end_io()
392 		btrfs_log_dev_io_error(bio, stripe->dev);  in btrfs_orig_write_end_io()
399 	if (atomic_read(&bioc->error) > bioc->max_errors)  in btrfs_orig_write_end_io()
400 		bio->bi_status = BLK_STS_IOERR;  in btrfs_orig_write_end_io()
402 		bio->bi_status = BLK_STS_OK;  in btrfs_orig_write_end_io()
404 	if (bio_is_zone_append(bio) && !bio->bi_status)  in btrfs_orig_write_end_io()
405 		stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_orig_write_end_io()
407 	btrfs_bio_end_io(bbio, bbio->bio.bi_status);  in btrfs_orig_write_end_io()
413 	struct btrfs_io_stripe *stripe = bio->bi_private;  in btrfs_clone_write_end_io()
415 	if (bio->bi_status) {  in btrfs_clone_write_end_io()
416 		atomic_inc(&stripe->bioc->error);  in btrfs_clone_write_end_io()
417 		btrfs_log_dev_io_error(bio, stripe->dev);  in btrfs_clone_write_end_io()
419 		stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_clone_write_end_io()
423 	bio_endio(stripe->bioc->orig_bio);  in btrfs_clone_write_end_io()
429 	if (!dev || !dev->bdev ||  in btrfs_submit_dev_bio()
430 	    test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||  in btrfs_submit_dev_bio()
432 	     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {  in btrfs_submit_dev_bio()
437 	bio_set_dev(bio, dev->bdev);  in btrfs_submit_dev_bio()
444 		u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_dev_bio()
445 		u64 zone_start = round_down(physical, dev->fs_info->zone_size);  in btrfs_submit_dev_bio()
448 		bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT;  in btrfs_submit_dev_bio()
450 	btrfs_debug_in_rcu(dev->fs_info,  in btrfs_submit_dev_bio()
452 		__func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector,  in btrfs_submit_dev_bio()
453 		(unsigned long)dev->bdev->bd_dev, btrfs_dev_name(dev),  in btrfs_submit_dev_bio()
454 		dev->devid, bio->bi_iter.bi_size);  in btrfs_submit_dev_bio()
460 	if (dev->fs_devices->collect_fs_stats && bio_op(bio) == REQ_OP_READ && dev->fs_info)  in btrfs_submit_dev_bio()
461 		percpu_counter_add(&dev->fs_info->stats_read_blocks,  in btrfs_submit_dev_bio()
462 				   bio->bi_iter.bi_size >> dev->fs_info->sectorsize_bits);  in btrfs_submit_dev_bio()
464 	if (bio->bi_opf & REQ_BTRFS_CGROUP_PUNT)  in btrfs_submit_dev_bio()
472 	struct bio *orig_bio = bioc->orig_bio, *bio;  in btrfs_submit_mirrored_bio()
476 	/* Reuse the bio embedded into the btrfs_bio for the last mirror */  in btrfs_submit_mirrored_bio()
477 	if (dev_nr == bioc->num_stripes - 1) {  in btrfs_submit_mirrored_bio()
479 		bio->bi_end_io = btrfs_orig_write_end_io;  in btrfs_submit_mirrored_bio()
483 		bio->bi_end_io = btrfs_clone_write_end_io;  in btrfs_submit_mirrored_bio()
486 	bio->bi_private = &bioc->stripes[dev_nr];  in btrfs_submit_mirrored_bio()
487 	bio->bi_iter.bi_sector = bioc->stripes[dev_nr].physical >> SECTOR_SHIFT;  in btrfs_submit_mirrored_bio()
488 	bioc->stripes[dev_nr].bioc = bioc;  in btrfs_submit_mirrored_bio()
489 	bioc->size = bio->bi_iter.bi_size;  in btrfs_submit_mirrored_bio()
490 	btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio);  in btrfs_submit_mirrored_bio()
497 		/* Single mirror read/write fast path. */  in btrfs_submit_bio()
498 		btrfs_bio(bio)->mirror_num = mirror_num;  in btrfs_submit_bio()
499 		bio->bi_iter.bi_sector = smap->physical >> SECTOR_SHIFT;  in btrfs_submit_bio()
501 			btrfs_bio(bio)->orig_physical = smap->physical;  in btrfs_submit_bio()
502 		bio->bi_private = smap->dev;  in btrfs_submit_bio()
503 		bio->bi_end_io = btrfs_simple_end_io;  in btrfs_submit_bio()
504 		btrfs_submit_dev_bio(smap->dev, bio);  in btrfs_submit_bio()
505 	} else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {  in btrfs_submit_bio()
507 		bio->bi_private = bioc;  in btrfs_submit_bio()
508 		bio->bi_end_io = btrfs_raid56_end_io;  in btrfs_submit_bio()
515 		int total_devs = bioc->num_stripes;  in btrfs_submit_bio()
517 		bioc->orig_bio = bio;  in btrfs_submit_bio()
525 	if (bbio->bio.bi_opf & REQ_META)  in btrfs_bio_csum()
556 	ret = btrfs_bio_csum(async->bbio);  in run_one_async_start()
558 		async->bbio->bio.bi_status = ret;  in run_one_async_start()
575 	struct bio *bio = &async->bbio->bio;  in run_one_async_done()
583 	if (bio->bi_status) {  in run_one_async_done()
584 		btrfs_bio_end_io(async->bbio, async->bbio->bio.bi_status);  in run_one_async_done()
593 	bio->bi_opf |= REQ_BTRFS_CGROUP_PUNT;  in run_one_async_done()
594 	btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num);  in run_one_async_done()
602 	struct btrfs_fs_devices *fs_devices = bbio->fs_info->fs_devices;  in should_async_write()
603 	enum btrfs_offload_csum_mode csum_mode = READ_ONCE(fs_devices->offload_csum_mode);  in should_async_write()
612 	if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &bbio->fs_info->flags))  in should_async_write()
619 	if (op_is_sync(bbio->bio.bi_opf))  in should_async_write()
623 	if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(bbio->fs_info))  in should_async_write()
638 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_wq_submit_bio()
645 	async->bbio = bbio;  in btrfs_wq_submit_bio()
646 	async->bioc = bioc;  in btrfs_wq_submit_bio()
647 	async->smap = *smap;  in btrfs_wq_submit_bio()
648 	async->mirror_num = mirror_num;  in btrfs_wq_submit_bio()
650 	btrfs_init_work(&async->work, run_one_async_start, run_one_async_done);  in btrfs_wq_submit_bio()
651 	btrfs_queue_work(fs_info->workers, &async->work);  in btrfs_wq_submit_bio()
660 	map_length = min(map_length, bbio->fs_info->max_zone_append_size);  in btrfs_append_map_length()
661 	sector_offset = bio_split_rw_at(&bbio->bio, &bbio->fs_info->limits,  in btrfs_append_map_length()
669 		return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, bbio->fs_info->sectorsize);  in btrfs_append_map_length()
676 	struct btrfs_inode *inode = bbio->inode;  in btrfs_submit_chunk()
677 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_submit_chunk()
678 	struct bio *bio = &bbio->bio;  in btrfs_submit_chunk()
679 	u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_chunk()
680 	u64 length = bio->bi_iter.bi_size;  in btrfs_submit_chunk()
688 	if (!bbio->inode || btrfs_is_data_reloc_root(inode->root))  in btrfs_submit_chunk()
716 		bio = &bbio->bio;  in btrfs_submit_chunk()
721 	 * data reads.  in btrfs_submit_chunk()
724 		bbio->saved_iter = bio->bi_iter;  in btrfs_submit_chunk()
732 			bio->bi_opf &= ~REQ_OP_WRITE;  in btrfs_submit_chunk()
733 			bio->bi_opf |= REQ_OP_ZONE_APPEND;  in btrfs_submit_chunk()
736 		if (is_data_bbio(bbio) && bioc && bioc->use_rst) {  in btrfs_submit_chunk()
744 			list_add_tail(&bioc->rst_ordered_entry, &bbio->ordered->bioc_list);  in btrfs_submit_chunk()
749 		 * point, so they are handled as part of the no-checksum case.  in btrfs_submit_chunk()
751 		if (inode && !(inode->flags & BTRFS_INODE_NODATASUM) &&  in btrfs_submit_chunk()
752 		    !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) &&  in btrfs_submit_chunk()
753 		    !btrfs_is_data_reloc_root(inode->root)) {  in btrfs_submit_chunk()
763 			    inode->flags & BTRFS_INODE_NODATASUM)) {  in btrfs_submit_chunk()
781 		struct btrfs_bio *remaining = bbio->private;  in btrfs_submit_chunk()
783 		ASSERT(bbio->bio.bi_pool == &btrfs_clone_bioset);  in btrfs_submit_chunk()
796 	/* If bbio->inode is not populated, its file_offset must be 0. */  in btrfs_submit_bbio()
797 	ASSERT(bbio->inode || bbio->file_offset == 0);  in btrfs_submit_bbio()
822 	ASSERT(!(fs_info->sb->s_flags & SB_RDONLY));  in btrfs_repair_io_failure()
838 	if (!smap.dev->bdev ||  in btrfs_repair_io_failure()
839 	    !test_bit(BTRFS_DEV_STATE_WRITEABLE, &smap.dev->dev_state)) {  in btrfs_repair_io_failure()
840 		ret = -EIO;  in btrfs_repair_io_failure()
844 	bio_init(&bio, smap.dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC);  in btrfs_repair_io_failure()
871  * If @dev_replace is true, the write would be submitted to dev-replace target.
875 	struct btrfs_fs_info *fs_info = bbio->fs_info;  in btrfs_submit_repair_write()
876 	u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT;  in btrfs_submit_repair_write()
877 	u64 length = bbio->bio.bi_iter.bi_size;  in btrfs_submit_repair_write()
883 	ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE);  in btrfs_submit_repair_write()
884 	ASSERT(!bbio->inode);  in btrfs_submit_repair_write()
892 		ASSERT(smap.dev == fs_info->dev_replace.srcdev);  in btrfs_submit_repair_write()
893 		smap.dev = fs_info->dev_replace.tgtdev;  in btrfs_submit_repair_write()
895 	btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num);  in btrfs_submit_repair_write()
908 		return -ENOMEM;  in btrfs_bioset_init()
927 	return -ENOMEM;  in btrfs_bioset_init()