| /* |
| * Copyright (C) 2007 Oracle. All rights reserved. |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public |
| * License v2 as published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public |
| * License along with this program; if not, write to the |
| * Free Software Foundation, Inc., 59 Temple Place - Suite 330, |
| * Boston, MA 021110-1307, USA. |
| */ |
| #include <linux/sched.h> |
| #include <linux/sched/signal.h> |
| #include <linux/pagemap.h> |
| #include <linux/writeback.h> |
| #include <linux/blkdev.h> |
| #include <linux/sort.h> |
| #include <linux/rcupdate.h> |
| #include <linux/kthread.h> |
| #include <linux/slab.h> |
| #include <linux/ratelimit.h> |
| #include <linux/percpu_counter.h> |
| #include "hash.h" |
| #include "tree-log.h" |
| #include "disk-io.h" |
| #include "print-tree.h" |
| #include "volumes.h" |
| #include "raid56.h" |
| #include "locking.h" |
| #include "free-space-cache.h" |
| #include "free-space-tree.h" |
| #include "math.h" |
| #include "sysfs.h" |
| #include "qgroup.h" |
| |
| #undef SCRAMBLE_DELAYED_REFS |
| |
| /* |
| * control flags for do_chunk_alloc's force field |
| * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk |
| * if we really need one. |
| * |
| * CHUNK_ALLOC_LIMITED means to only try and allocate one |
| * if we have very few chunks already allocated. This is |
| * used as part of the clustering code to help make sure |
| * we have a good pool of storage to cluster in, without |
| * filling the FS with empty chunks |
| * |
| * CHUNK_ALLOC_FORCE means it must try to allocate one |
| * |
| */ |
| enum { |
| CHUNK_ALLOC_NO_FORCE = 0, |
| CHUNK_ALLOC_LIMITED = 1, |
| CHUNK_ALLOC_FORCE = 2, |
| }; |
| |
| static int update_block_group(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, u64 bytenr, |
| u64 num_bytes, int alloc); |
| static int __btrfs_free_extent(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, u64 parent, |
| u64 root_objectid, u64 owner_objectid, |
| u64 owner_offset, int refs_to_drop, |
| struct btrfs_delayed_extent_op *extra_op); |
| static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, |
| struct extent_buffer *leaf, |
| struct btrfs_extent_item *ei); |
| static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| u64 parent, u64 root_objectid, |
| u64 flags, u64 owner, u64 offset, |
| struct btrfs_key *ins, int ref_mod); |
| static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| u64 parent, u64 root_objectid, |
| u64 flags, struct btrfs_disk_key *key, |
| int level, struct btrfs_key *ins); |
| static int do_chunk_alloc(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, u64 flags, |
| int force); |
| static int find_next_key(struct btrfs_path *path, int level, |
| struct btrfs_key *key); |
| static void dump_space_info(struct btrfs_fs_info *fs_info, |
| struct btrfs_space_info *info, u64 bytes, |
| int dump_block_groups); |
| static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache, |
| u64 ram_bytes, u64 num_bytes, int delalloc); |
| static int btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache, |
| u64 num_bytes, int delalloc); |
| static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, |
| u64 num_bytes); |
| static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, |
| struct btrfs_space_info *space_info, |
| u64 orig_bytes, |
| enum btrfs_reserve_flush_enum flush, |
| bool system_chunk); |
| static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info, |
| struct btrfs_space_info *space_info, |
| u64 num_bytes); |
| static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info, |
| struct btrfs_space_info *space_info, |
| u64 num_bytes); |
| |
| static noinline int |
| block_group_cache_done(struct btrfs_block_group_cache *cache) |
| { |
| smp_mb(); |
| return cache->cached == BTRFS_CACHE_FINISHED || |
| cache->cached == BTRFS_CACHE_ERROR; |
| } |
| |
| static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits) |
| { |
| return (cache->flags & bits) == bits; |
| } |
| |
| void btrfs_get_block_group(struct btrfs_block_group_cache *cache) |
| { |
| atomic_inc(&cache->count); |
| } |
| |
| void btrfs_put_block_group(struct btrfs_block_group_cache *cache) |
| { |
| if (atomic_dec_and_test(&cache->count)) { |
| WARN_ON(cache->pinned > 0); |
| WARN_ON(cache->reserved > 0); |
| |
| /* |
| * If not empty, someone is still holding mutex of |
| * full_stripe_lock, which can only be released by caller. |
| * And it will definitely cause use-after-free when caller |
| * tries to release full stripe lock. |
| * |
| * No better way to resolve, but only to warn. |
| */ |
| WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root)); |
| kfree(cache->free_space_ctl); |
| kfree(cache); |
| } |
| } |
| |
| /* |
| * this adds the block group to the fs_info rb tree for the block group |
| * cache |
| */ |
| static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, |
| struct btrfs_block_group_cache *block_group) |
| { |
| struct rb_node **p; |
| struct rb_node *parent = NULL; |
| struct btrfs_block_group_cache *cache; |
| |
| spin_lock(&info->block_group_cache_lock); |
| p = &info->block_group_cache_tree.rb_node; |
| |
| while (*p) { |
| parent = *p; |
| cache = rb_entry(parent, struct btrfs_block_group_cache, |
| cache_node); |
| if (block_group->key.objectid < cache->key.objectid) { |
| p = &(*p)->rb_left; |
| } else if (block_group->key.objectid > cache->key.objectid) { |
| p = &(*p)->rb_right; |
| } else { |
| spin_unlock(&info->block_group_cache_lock); |
| return -EEXIST; |
| } |
| } |
| |
| rb_link_node(&block_group->cache_node, parent, p); |
| rb_insert_color(&block_group->cache_node, |
| &info->block_group_cache_tree); |
| |
| if (info->first_logical_byte > block_group->key.objectid) |
| info->first_logical_byte = block_group->key.objectid; |
| |
| spin_unlock(&info->block_group_cache_lock); |
| |
| return 0; |
| } |
| |
| /* |
| * This will return the block group at or after bytenr if contains is 0, else |
| * it will return the block group that contains the bytenr |
| */ |
| static struct btrfs_block_group_cache * |
| block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr, |
| int contains) |
| { |
| struct btrfs_block_group_cache *cache, *ret = NULL; |
| struct rb_node *n; |
| u64 end, start; |
| |
| spin_lock(&info->block_group_cache_lock); |
| n = info->block_group_cache_tree.rb_node; |
| |
| while (n) { |
| cache = rb_entry(n, struct btrfs_block_group_cache, |
| cache_node); |
| end = cache->key.objectid + cache->key.offset - 1; |
| start = cache->key.objectid; |
| |
| if (bytenr < start) { |
| if (!contains && (!ret || start < ret->key.objectid)) |
| ret = cache; |
| n = n->rb_left; |
| } else if (bytenr > start) { |
| if (contains && bytenr <= end) { |
| ret = cache; |
| break; |
| } |
| n = n->rb_right; |
| } else { |
| ret = cache; |
| break; |
| } |
| } |
| if (ret) { |
| btrfs_get_block_group(ret); |
| if (bytenr == 0 && info->first_logical_byte > ret->key.objectid) |
| info->first_logical_byte = ret->key.objectid; |
| } |
| spin_unlock(&info->block_group_cache_lock); |
| |
| return ret; |
| } |
| |
| static int add_excluded_extent(struct btrfs_fs_info *fs_info, |
| u64 start, u64 num_bytes) |
| { |
| u64 end = start + num_bytes - 1; |
| set_extent_bits(&fs_info->freed_extents[0], |
| start, end, EXTENT_UPTODATE); |
| set_extent_bits(&fs_info->freed_extents[1], |
| start, end, EXTENT_UPTODATE); |
| return 0; |
| } |
| |
| static void free_excluded_extents(struct btrfs_fs_info *fs_info, |
| struct btrfs_block_group_cache *cache) |
| { |
| u64 start, end; |
| |
| start = cache->key.objectid; |
| end = start + cache->key.offset - 1; |
| |
| clear_extent_bits(&fs_info->freed_extents[0], |
| start, end, EXTENT_UPTODATE); |
| clear_extent_bits(&fs_info->freed_extents[1], |
| start, end, EXTENT_UPTODATE); |
| } |
| |
| static int exclude_super_stripes(struct btrfs_fs_info *fs_info, |
| struct btrfs_block_group_cache *cache) |
| { |
| u64 bytenr; |
| u64 *logical; |
| int stripe_len; |
| int i, nr, ret; |
| |
| if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) { |
| stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid; |
| cache->bytes_super += stripe_len; |
| ret = add_excluded_extent(fs_info, cache->key.objectid, |
| stripe_len); |
| if (ret) |
| return ret; |
| } |
| |
| for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { |
| bytenr = btrfs_sb_offset(i); |
| ret = btrfs_rmap_block(fs_info, cache->key.objectid, |
| bytenr, 0, &logical, &nr, &stripe_len); |
| if (ret) |
| return ret; |
| |
| while (nr--) { |
| u64 start, len; |
| |
| if (logical[nr] > cache->key.objectid + |
| cache->key.offset) |
| continue; |
| |
| if (logical[nr] + stripe_len <= cache->key.objectid) |
| continue; |
| |
| start = logical[nr]; |
| if (start < cache->key.objectid) { |
| start = cache->key.objectid; |
| len = (logical[nr] + stripe_len) - start; |
| } else { |
| len = min_t(u64, stripe_len, |
| cache->key.objectid + |
| cache->key.offset - start); |
| } |
| |
| cache->bytes_super += len; |
| ret = add_excluded_extent(fs_info, start, len); |
| if (ret) { |
| kfree(logical); |
| return ret; |
| } |
| } |
| |
| kfree(logical); |
| } |
| return 0; |
| } |
| |
| static struct btrfs_caching_control * |
| get_caching_control(struct btrfs_block_group_cache *cache) |
| { |
| struct btrfs_caching_control *ctl; |
| |
| spin_lock(&cache->lock); |
| if (!cache->caching_ctl) { |
| spin_unlock(&cache->lock); |
| return NULL; |
| } |
| |
| ctl = cache->caching_ctl; |
| refcount_inc(&ctl->count); |
| spin_unlock(&cache->lock); |
| return ctl; |
| } |
| |
| static void put_caching_control(struct btrfs_caching_control *ctl) |
| { |
| if (refcount_dec_and_test(&ctl->count)) |
| kfree(ctl); |
| } |
| |
| #ifdef CONFIG_BTRFS_DEBUG |
| static void fragment_free_space(struct btrfs_block_group_cache *block_group) |
| { |
| struct btrfs_fs_info *fs_info = block_group->fs_info; |
| u64 start = block_group->key.objectid; |
| u64 len = block_group->key.offset; |
| u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ? |
| fs_info->nodesize : fs_info->sectorsize; |
| u64 step = chunk << 1; |
| |
| while (len > chunk) { |
| btrfs_remove_free_space(block_group, start, chunk); |
| start += step; |
| if (len < step) |
| len = 0; |
| else |
| len -= step; |
| } |
| } |
| #endif |
| |
| /* |
| * this is only called by cache_block_group, since we could have freed extents |
| * we need to check the pinned_extents for any extents that can't be used yet |
| * since their free space will be released as soon as the transaction commits. |
| */ |
| u64 add_new_free_space(struct btrfs_block_group_cache *block_group, |
| struct btrfs_fs_info *info, u64 start, u64 end) |
| { |
| u64 extent_start, extent_end, size, total_added = 0; |
| int ret; |
| |
| while (start < end) { |
| ret = find_first_extent_bit(info->pinned_extents, start, |
| &extent_start, &extent_end, |
| EXTENT_DIRTY | EXTENT_UPTODATE, |
| NULL); |
| if (ret) |
| break; |
| |
| if (extent_start <= start) { |
| start = extent_end + 1; |
| } else if (extent_start > start && extent_start < end) { |
| size = extent_start - start; |
| total_added += size; |
| ret = btrfs_add_free_space(block_group, start, |
| size); |
| BUG_ON(ret); /* -ENOMEM or logic error */ |
| start = extent_end + 1; |
| } else { |
| break; |
| } |
| } |
| |
| if (start < end) { |
| size = end - start; |
| total_added += size; |
| ret = btrfs_add_free_space(block_group, start, size); |
| BUG_ON(ret); /* -ENOMEM or logic error */ |
| } |
| |
| return total_added; |
| } |
| |
| static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) |
| { |
| struct btrfs_block_group_cache *block_group = caching_ctl->block_group; |
| struct btrfs_fs_info *fs_info = block_group->fs_info; |
| struct btrfs_root *extent_root = fs_info->extent_root; |
| struct btrfs_path *path; |
| struct extent_buffer *leaf; |
| struct btrfs_key key; |
| u64 total_found = 0; |
| u64 last = 0; |
| u32 nritems; |
| int ret; |
| bool wakeup = true; |
| |
| path = btrfs_alloc_path(); |
| if (!path) |
| return -ENOMEM; |
| |
| last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET); |
| |
| #ifdef CONFIG_BTRFS_DEBUG |
| /* |
| * If we're fragmenting we don't want to make anybody think we can |
| * allocate from this block group until we've had a chance to fragment |
| * the free space. |
| */ |
| if (btrfs_should_fragment_free_space(block_group)) |
| wakeup = false; |
| #endif |
| /* |
| * We don't want to deadlock with somebody trying to allocate a new |
| * extent for the extent root while also trying to search the extent |
| * root to add free space. So we skip locking and search the commit |
| * root, since its read-only |
| */ |
| path->skip_locking = 1; |
| path->search_commit_root = 1; |
| path->reada = READA_FORWARD; |
| |
| key.objectid = last; |
| key.offset = 0; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| |
| next: |
| ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); |
| if (ret < 0) |
| goto out; |
| |
| leaf = path->nodes[0]; |
| nritems = btrfs_header_nritems(leaf); |
| |
| while (1) { |
| if (btrfs_fs_closing(fs_info) > 1) { |
| last = (u64)-1; |
| break; |
| } |
| |
| if (path->slots[0] < nritems) { |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| } else { |
| ret = find_next_key(path, 0, &key); |
| if (ret) |
| break; |
| |
| if (need_resched() || |
| rwsem_is_contended(&fs_info->commit_root_sem)) { |
| if (wakeup) |
| caching_ctl->progress = last; |
| btrfs_release_path(path); |
| up_read(&fs_info->commit_root_sem); |
| mutex_unlock(&caching_ctl->mutex); |
| cond_resched(); |
| mutex_lock(&caching_ctl->mutex); |
| down_read(&fs_info->commit_root_sem); |
| goto next; |
| } |
| |
| ret = btrfs_next_leaf(extent_root, path); |
| if (ret < 0) |
| goto out; |
| if (ret) |
| break; |
| leaf = path->nodes[0]; |
| nritems = btrfs_header_nritems(leaf); |
| continue; |
| } |
| |
| if (key.objectid < last) { |
| key.objectid = last; |
| key.offset = 0; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| |
| if (wakeup) |
| caching_ctl->progress = last; |
| btrfs_release_path(path); |
| goto next; |
| } |
| |
| if (key.objectid < block_group->key.objectid) { |
| path->slots[0]++; |
| continue; |
| } |
| |
| if (key.objectid >= block_group->key.objectid + |
| block_group->key.offset) |
| break; |
| |
| if (key.type == BTRFS_EXTENT_ITEM_KEY || |
| key.type == BTRFS_METADATA_ITEM_KEY) { |
| total_found += add_new_free_space(block_group, |
| fs_info, last, |
| key.objectid); |
| if (key.type == BTRFS_METADATA_ITEM_KEY) |
| last = key.objectid + |
| fs_info->nodesize; |
| else |
| last = key.objectid + key.offset; |
| |
| if (total_found > CACHING_CTL_WAKE_UP) { |
| total_found = 0; |
| if (wakeup) |
| wake_up(&caching_ctl->wait); |
| } |
| } |
| path->slots[0]++; |
| } |
| ret = 0; |
| |
| total_found += add_new_free_space(block_group, fs_info, last, |
| block_group->key.objectid + |
| block_group->key.offset); |
| caching_ctl->progress = (u64)-1; |
| |
| out: |
| btrfs_free_path(path); |
| return ret; |
| } |
| |
| static noinline void caching_thread(struct btrfs_work *work) |
| { |
| struct btrfs_block_group_cache *block_group; |
| struct btrfs_fs_info *fs_info; |
| struct btrfs_caching_control *caching_ctl; |
| struct btrfs_root *extent_root; |
| int ret; |
| |
| caching_ctl = container_of(work, struct btrfs_caching_control, work); |
| block_group = caching_ctl->block_group; |
| fs_info = block_group->fs_info; |
| extent_root = fs_info->extent_root; |
| |
| mutex_lock(&caching_ctl->mutex); |
| down_read(&fs_info->commit_root_sem); |
| |
| if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) |
| ret = load_free_space_tree(caching_ctl); |
| else |
| ret = load_extent_tree_free(caching_ctl); |
| |
| spin_lock(&block_group->lock); |
| block_group->caching_ctl = NULL; |
| block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED; |
| spin_unlock(&block_group->lock); |
| |
| #ifdef CONFIG_BTRFS_DEBUG |
| if (btrfs_should_fragment_free_space(block_group)) { |
| u64 bytes_used; |
| |
| spin_lock(&block_group->space_info->lock); |
| spin_lock(&block_group->lock); |
| bytes_used = block_group->key.offset - |
| btrfs_block_group_used(&block_group->item); |
| block_group->space_info->bytes_used += bytes_used >> 1; |
| spin_unlock(&block_group->lock); |
| spin_unlock(&block_group->space_info->lock); |
| fragment_free_space(block_group); |
| } |
| #endif |
| |
| caching_ctl->progress = (u64)-1; |
| |
| up_read(&fs_info->commit_root_sem); |
| free_excluded_extents(fs_info, block_group); |
| mutex_unlock(&caching_ctl->mutex); |
| |
| wake_up(&caching_ctl->wait); |
| |
| put_caching_control(caching_ctl); |
| btrfs_put_block_group(block_group); |
| } |
| |
| static int cache_block_group(struct btrfs_block_group_cache *cache, |
| int load_cache_only) |
| { |
| DEFINE_WAIT(wait); |
| struct btrfs_fs_info *fs_info = cache->fs_info; |
| struct btrfs_caching_control *caching_ctl; |
| int ret = 0; |
| |
| caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS); |
| if (!caching_ctl) |
| return -ENOMEM; |
| |
| INIT_LIST_HEAD(&caching_ctl->list); |
| mutex_init(&caching_ctl->mutex); |
| init_waitqueue_head(&caching_ctl->wait); |
| caching_ctl->block_group = cache; |
| caching_ctl->progress = cache->key.objectid; |
| refcount_set(&caching_ctl->count, 1); |
| btrfs_init_work(&caching_ctl->work, btrfs_cache_helper, |
| caching_thread, NULL, NULL); |
| |
| spin_lock(&cache->lock); |
| /* |
| * This should be a rare occasion, but this could happen I think in the |
| * case where one thread starts to load the space cache info, and then |
| * some other thread starts a transaction commit which tries to do an |
| * allocation while the other thread is still loading the space cache |
| * info. The previous loop should have kept us from choosing this block |
| * group, but if we've moved to the state where we will wait on caching |
| * block groups we need to first check if we're doing a fast load here, |
| * so we can wait for it to finish, otherwise we could end up allocating |
| * from a block group who's cache gets evicted for one reason or |
| * another. |
| */ |
| while (cache->cached == BTRFS_CACHE_FAST) { |
| struct btrfs_caching_control *ctl; |
| |
| ctl = cache->caching_ctl; |
| refcount_inc(&ctl->count); |
| prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE); |
| spin_unlock(&cache->lock); |
| |
| schedule(); |
| |
| finish_wait(&ctl->wait, &wait); |
| put_caching_control(ctl); |
| spin_lock(&cache->lock); |
| } |
| |
| if (cache->cached != BTRFS_CACHE_NO) { |
| spin_unlock(&cache->lock); |
| kfree(caching_ctl); |
| return 0; |
| } |
| WARN_ON(cache->caching_ctl); |
| cache->caching_ctl = caching_ctl; |
| cache->cached = BTRFS_CACHE_FAST; |
| spin_unlock(&cache->lock); |
| |
| if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) { |
| mutex_lock(&caching_ctl->mutex); |
| ret = load_free_space_cache(fs_info, cache); |
| |
| spin_lock(&cache->lock); |
| if (ret == 1) { |
| cache->caching_ctl = NULL; |
| cache->cached = BTRFS_CACHE_FINISHED; |
| cache->last_byte_to_unpin = (u64)-1; |
| caching_ctl->progress = (u64)-1; |
| } else { |
| if (load_cache_only) { |
| cache->caching_ctl = NULL; |
| cache->cached = BTRFS_CACHE_NO; |
| } else { |
| cache->cached = BTRFS_CACHE_STARTED; |
| cache->has_caching_ctl = 1; |
| } |
| } |
| spin_unlock(&cache->lock); |
| #ifdef CONFIG_BTRFS_DEBUG |
| if (ret == 1 && |
| btrfs_should_fragment_free_space(cache)) { |
| u64 bytes_used; |
| |
| spin_lock(&cache->space_info->lock); |
| spin_lock(&cache->lock); |
| bytes_used = cache->key.offset - |
| btrfs_block_group_used(&cache->item); |
| cache->space_info->bytes_used += bytes_used >> 1; |
| spin_unlock(&cache->lock); |
| spin_unlock(&cache->space_info->lock); |
| fragment_free_space(cache); |
| } |
| #endif |
| mutex_unlock(&caching_ctl->mutex); |
| |
| wake_up(&caching_ctl->wait); |
| if (ret == 1) { |
| put_caching_control(caching_ctl); |
| free_excluded_extents(fs_info, cache); |
| return 0; |
| } |
| } else { |
| /* |
| * We're either using the free space tree or no caching at all. |
| * Set cached to the appropriate value and wakeup any waiters. |
| */ |
| spin_lock(&cache->lock); |
| if (load_cache_only) { |
| cache->caching_ctl = NULL; |
| cache->cached = BTRFS_CACHE_NO; |
| } else { |
| cache->cached = BTRFS_CACHE_STARTED; |
| cache->has_caching_ctl = 1; |
| } |
| spin_unlock(&cache->lock); |
| wake_up(&caching_ctl->wait); |
| } |
| |
| if (load_cache_only) { |
| put_caching_control(caching_ctl); |
| return 0; |
| } |
| |
| down_write(&fs_info->commit_root_sem); |
| refcount_inc(&caching_ctl->count); |
| list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups); |
| up_write(&fs_info->commit_root_sem); |
| |
| btrfs_get_block_group(cache); |
| |
| btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); |
| |
| return ret; |
| } |
| |
| /* |
| * return the block group that starts at or after bytenr |
| */ |
| static struct btrfs_block_group_cache * |
| btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr) |
| { |
| return block_group_cache_tree_search(info, bytenr, 0); |
| } |
| |
| /* |
| * return the block group that contains the given bytenr |
| */ |
| struct btrfs_block_group_cache *btrfs_lookup_block_group( |
| struct btrfs_fs_info *info, |
| u64 bytenr) |
| { |
| return block_group_cache_tree_search(info, bytenr, 1); |
| } |
| |
| static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info, |
| u64 flags) |
| { |
| struct list_head *head = &info->space_info; |
| struct btrfs_space_info *found; |
| |
| flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; |
| |
| rcu_read_lock(); |
| list_for_each_entry_rcu(found, head, list) { |
| if (found->flags & flags) { |
| rcu_read_unlock(); |
| return found; |
| } |
| } |
| rcu_read_unlock(); |
| return NULL; |
| } |
| |
| static void add_pinned_bytes(struct btrfs_fs_info *fs_info, s64 num_bytes, |
| u64 owner, u64 root_objectid) |
| { |
| struct btrfs_space_info *space_info; |
| u64 flags; |
| |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID) |
| flags = BTRFS_BLOCK_GROUP_SYSTEM; |
| else |
| flags = BTRFS_BLOCK_GROUP_METADATA; |
| } else { |
| flags = BTRFS_BLOCK_GROUP_DATA; |
| } |
| |
| space_info = __find_space_info(fs_info, flags); |
| ASSERT(space_info); |
| percpu_counter_add(&space_info->total_bytes_pinned, num_bytes); |
| } |
| |
| /* |
| * after adding space to the filesystem, we need to clear the full flags |
| * on all the space infos. |
| */ |
| void btrfs_clear_space_info_full(struct btrfs_fs_info *info) |
| { |
| struct list_head *head = &info->space_info; |
| struct btrfs_space_info *found; |
| |
| rcu_read_lock(); |
| list_for_each_entry_rcu(found, head, list) |
| found->full = 0; |
| rcu_read_unlock(); |
| } |
| |
| /* simple helper to search for an existing data extent at a given offset */ |
| int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len) |
| { |
| int ret; |
| struct btrfs_key key; |
| struct btrfs_path *path; |
| |
| path = btrfs_alloc_path(); |
| if (!path) |
| return -ENOMEM; |
| |
| key.objectid = start; |
| key.offset = len; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0); |
| btrfs_free_path(path); |
| return ret; |
| } |
| |
| /* |
| * helper function to lookup reference count and flags of a tree block. |
| * |
| * the head node for delayed ref is used to store the sum of all the |
| * reference count modifications queued up in the rbtree. the head |
| * node may also store the extent flags to set. This way you can check |
| * to see what the reference count and extent flags would be if all of |
| * the delayed refs are not processed. |
| */ |
| int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, u64 bytenr, |
| u64 offset, int metadata, u64 *refs, u64 *flags) |
| { |
| struct btrfs_delayed_ref_head *head; |
| struct btrfs_delayed_ref_root *delayed_refs; |
| struct btrfs_path *path; |
| struct btrfs_extent_item *ei; |
| struct extent_buffer *leaf; |
| struct btrfs_key key; |
| u32 item_size; |
| u64 num_refs; |
| u64 extent_flags; |
| int ret; |
| |
| /* |
| * If we don't have skinny metadata, don't bother doing anything |
| * different |
| */ |
| if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) { |
| offset = fs_info->nodesize; |
| metadata = 0; |
| } |
| |
| path = btrfs_alloc_path(); |
| if (!path) |
| return -ENOMEM; |
| |
| if (!trans) { |
| path->skip_locking = 1; |
| path->search_commit_root = 1; |
| } |
| |
| search_again: |
| key.objectid = bytenr; |
| key.offset = offset; |
| if (metadata) |
| key.type = BTRFS_METADATA_ITEM_KEY; |
| else |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| |
| ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 0); |
| if (ret < 0) |
| goto out_free; |
| |
| if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) { |
| if (path->slots[0]) { |
| path->slots[0]--; |
| btrfs_item_key_to_cpu(path->nodes[0], &key, |
| path->slots[0]); |
| if (key.objectid == bytenr && |
| key.type == BTRFS_EXTENT_ITEM_KEY && |
| key.offset == fs_info->nodesize) |
| ret = 0; |
| } |
| } |
| |
| if (ret == 0) { |
| leaf = path->nodes[0]; |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| if (item_size >= sizeof(*ei)) { |
| ei = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_item); |
| num_refs = btrfs_extent_refs(leaf, ei); |
| extent_flags = btrfs_extent_flags(leaf, ei); |
| } else { |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| struct btrfs_extent_item_v0 *ei0; |
| BUG_ON(item_size != sizeof(*ei0)); |
| ei0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_item_v0); |
| num_refs = btrfs_extent_refs_v0(leaf, ei0); |
| /* FIXME: this isn't correct for data */ |
| extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; |
| #else |
| BUG(); |
| #endif |
| } |
| BUG_ON(num_refs == 0); |
| } else { |
| num_refs = 0; |
| extent_flags = 0; |
| ret = 0; |
| } |
| |
| if (!trans) |
| goto out; |
| |
| delayed_refs = &trans->transaction->delayed_refs; |
| spin_lock(&delayed_refs->lock); |
| head = btrfs_find_delayed_ref_head(delayed_refs, bytenr); |
| if (head) { |
| if (!mutex_trylock(&head->mutex)) { |
| refcount_inc(&head->node.refs); |
| spin_unlock(&delayed_refs->lock); |
| |
| btrfs_release_path(path); |
| |
| /* |
| * Mutex was contended, block until it's released and try |
| * again |
| */ |
| mutex_lock(&head->mutex); |
| mutex_unlock(&head->mutex); |
| btrfs_put_delayed_ref(&head->node); |
| goto search_again; |
| } |
| spin_lock(&head->lock); |
| if (head->extent_op && head->extent_op->update_flags) |
| extent_flags |= head->extent_op->flags_to_set; |
| else |
| BUG_ON(num_refs == 0); |
| |
| num_refs += head->node.ref_mod; |
| spin_unlock(&head->lock); |
| mutex_unlock(&head->mutex); |
| } |
| spin_unlock(&delayed_refs->lock); |
| out: |
| WARN_ON(num_refs == 0); |
| if (refs) |
| *refs = num_refs; |
| if (flags) |
| *flags = extent_flags; |
| out_free: |
| btrfs_free_path(path); |
| return ret; |
| } |
| |
| /* |
| * Back reference rules. Back refs have three main goals: |
| * |
| * 1) differentiate between all holders of references to an extent so that |
| * when a reference is dropped we can make sure it was a valid reference |
| * before freeing the extent. |
| * |
| * 2) Provide enough information to quickly find the holders of an extent |
| * if we notice a given block is corrupted or bad. |
| * |
| * 3) Make it easy to migrate blocks for FS shrinking or storage pool |
| * maintenance. This is actually the same as #2, but with a slightly |
| * different use case. |
| * |
| * There are two kinds of back refs. The implicit back refs is optimized |
| * for pointers in non-shared tree blocks. For a given pointer in a block, |
| * back refs of this kind provide information about the block's owner tree |
| * and the pointer's key. These information allow us to find the block by |
| * b-tree searching. The full back refs is for pointers in tree blocks not |
| * referenced by their owner trees. The location of tree block is recorded |
| * in the back refs. Actually the full back refs is generic, and can be |
| * used in all cases the implicit back refs is used. The major shortcoming |
| * of the full back refs is its overhead. Every time a tree block gets |
| * COWed, we have to update back refs entry for all pointers in it. |
| * |
| * For a newly allocated tree block, we use implicit back refs for |
| * pointers in it. This means most tree related operations only involve |
| * implicit back refs. For a tree block created in old transaction, the |
| * only way to drop a reference to it is COW it. So we can detect the |
| * event that tree block loses its owner tree's reference and do the |
| * back refs conversion. |
| * |
| * When a tree block is COWed through a tree, there are four cases: |
| * |
| * The reference count of the block is one and the tree is the block's |
| * owner tree. Nothing to do in this case. |
| * |
| * The reference count of the block is one and the tree is not the |
| * block's owner tree. In this case, full back refs is used for pointers |
| * in the block. Remove these full back refs, add implicit back refs for |
| * every pointers in the new block. |
| * |
| * The reference count of the block is greater than one and the tree is |
| * the block's owner tree. In this case, implicit back refs is used for |
| * pointers in the block. Add full back refs for every pointers in the |
| * block, increase lower level extents' reference counts. The original |
| * implicit back refs are entailed to the new block. |
| * |
| * The reference count of the block is greater than one and the tree is |
| * not the block's owner tree. Add implicit back refs for every pointer in |
| * the new block, increase lower level extents' reference count. |
| * |
| * Back Reference Key composing: |
| * |
| * The key objectid corresponds to the first byte in the extent, |
| * The key type is used to differentiate between types of back refs. |
| * There are different meanings of the key offset for different types |
| * of back refs. |
| * |
| * File extents can be referenced by: |
| * |
| * - multiple snapshots, subvolumes, or different generations in one subvol |
| * - different files inside a single subvolume |
| * - different offsets inside a file (bookend extents in file.c) |
| * |
| * The extent ref structure for the implicit back refs has fields for: |
| * |
| * - Objectid of the subvolume root |
| * - objectid of the file holding the reference |
| * - original offset in the file |
| * - how many bookend extents |
| * |
| * The key offset for the implicit back refs is hash of the first |
| * three fields. |
| * |
| * The extent ref structure for the full back refs has field for: |
| * |
| * - number of pointers in the tree leaf |
| * |
| * The key offset for the implicit back refs is the first byte of |
| * the tree leaf |
| * |
| * When a file extent is allocated, The implicit back refs is used. |
| * the fields are filled in: |
| * |
| * (root_key.objectid, inode objectid, offset in file, 1) |
| * |
| * When a file extent is removed file truncation, we find the |
| * corresponding implicit back refs and check the following fields: |
| * |
| * (btrfs_header_owner(leaf), inode objectid, offset in file) |
| * |
| * Btree extents can be referenced by: |
| * |
| * - Different subvolumes |
| * |
| * Both the implicit back refs and the full back refs for tree blocks |
| * only consist of key. The key offset for the implicit back refs is |
| * objectid of block's owner tree. The key offset for the full back refs |
| * is the first byte of parent block. |
| * |
| * When implicit back refs is used, information about the lowest key and |
| * level of the tree block are required. These information are stored in |
| * tree block info structure. |
| */ |
| |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| static int convert_extent_item_v0(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 owner, u32 extra_size) |
| { |
| struct btrfs_root *root = fs_info->extent_root; |
| struct btrfs_extent_item *item; |
| struct btrfs_extent_item_v0 *ei0; |
| struct btrfs_extent_ref_v0 *ref0; |
| struct btrfs_tree_block_info *bi; |
| struct extent_buffer *leaf; |
| struct btrfs_key key; |
| struct btrfs_key found_key; |
| u32 new_size = sizeof(*item); |
| u64 refs; |
| int ret; |
| |
| leaf = path->nodes[0]; |
| BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0)); |
| |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| ei0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_item_v0); |
| refs = btrfs_extent_refs_v0(leaf, ei0); |
| |
| if (owner == (u64)-1) { |
| while (1) { |
| if (path->slots[0] >= btrfs_header_nritems(leaf)) { |
| ret = btrfs_next_leaf(root, path); |
| if (ret < 0) |
| return ret; |
| BUG_ON(ret > 0); /* Corruption */ |
| leaf = path->nodes[0]; |
| } |
| btrfs_item_key_to_cpu(leaf, &found_key, |
| path->slots[0]); |
| BUG_ON(key.objectid != found_key.objectid); |
| if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) { |
| path->slots[0]++; |
| continue; |
| } |
| ref0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_ref_v0); |
| owner = btrfs_ref_objectid_v0(leaf, ref0); |
| break; |
| } |
| } |
| btrfs_release_path(path); |
| |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) |
| new_size += sizeof(*bi); |
| |
| new_size -= sizeof(*ei0); |
| ret = btrfs_search_slot(trans, root, &key, path, |
| new_size + extra_size, 1); |
| if (ret < 0) |
| return ret; |
| BUG_ON(ret); /* Corruption */ |
| |
| btrfs_extend_item(fs_info, path, new_size); |
| |
| leaf = path->nodes[0]; |
| item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| btrfs_set_extent_refs(leaf, item, refs); |
| /* FIXME: get real generation */ |
| btrfs_set_extent_generation(leaf, item, 0); |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| btrfs_set_extent_flags(leaf, item, |
| BTRFS_EXTENT_FLAG_TREE_BLOCK | |
| BTRFS_BLOCK_FLAG_FULL_BACKREF); |
| bi = (struct btrfs_tree_block_info *)(item + 1); |
| /* FIXME: get first key of the block */ |
| memzero_extent_buffer(leaf, (unsigned long)bi, sizeof(*bi)); |
| btrfs_set_tree_block_level(leaf, bi, (int)owner); |
| } else { |
| btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA); |
| } |
| btrfs_mark_buffer_dirty(leaf); |
| return 0; |
| } |
| #endif |
| |
| /* |
| * is_data == BTRFS_REF_TYPE_BLOCK, tree block type is required, |
| * is_data == BTRFS_REF_TYPE_DATA, data type is requried, |
| * is_data == BTRFS_REF_TYPE_ANY, either type is OK. |
| */ |
| int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, |
| struct btrfs_extent_inline_ref *iref, |
| enum btrfs_inline_ref_type is_data) |
| { |
| int type = btrfs_extent_inline_ref_type(eb, iref); |
| u64 offset = btrfs_extent_inline_ref_offset(eb, iref); |
| |
| if (type == BTRFS_TREE_BLOCK_REF_KEY || |
| type == BTRFS_SHARED_BLOCK_REF_KEY || |
| type == BTRFS_SHARED_DATA_REF_KEY || |
| type == BTRFS_EXTENT_DATA_REF_KEY) { |
| if (is_data == BTRFS_REF_TYPE_BLOCK) { |
| if (type == BTRFS_TREE_BLOCK_REF_KEY) |
| return type; |
| if (type == BTRFS_SHARED_BLOCK_REF_KEY) { |
| ASSERT(eb->fs_info); |
| /* |
| * Every shared one has parent tree |
| * block, which must be aligned to |
| * nodesize. |
| */ |
| if (offset && |
| IS_ALIGNED(offset, eb->fs_info->nodesize)) |
| return type; |
| } |
| } else if (is_data == BTRFS_REF_TYPE_DATA) { |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) |
| return type; |
| if (type == BTRFS_SHARED_DATA_REF_KEY) { |
| ASSERT(eb->fs_info); |
| /* |
| * Every shared one has parent tree |
| * block, which must be aligned to |
| * nodesize. |
| */ |
| if (offset && |
| IS_ALIGNED(offset, eb->fs_info->nodesize)) |
| return type; |
| } |
| } else { |
| ASSERT(is_data == BTRFS_REF_TYPE_ANY); |
| return type; |
| } |
| } |
| |
| btrfs_print_leaf((struct extent_buffer *)eb); |
| btrfs_err(eb->fs_info, "eb %llu invalid extent inline ref type %d", |
| eb->start, type); |
| WARN_ON(1); |
| |
| return BTRFS_REF_TYPE_INVALID; |
| } |
| |
| static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset) |
| { |
| u32 high_crc = ~(u32)0; |
| u32 low_crc = ~(u32)0; |
| __le64 lenum; |
| |
| lenum = cpu_to_le64(root_objectid); |
| high_crc = btrfs_crc32c(high_crc, &lenum, sizeof(lenum)); |
| lenum = cpu_to_le64(owner); |
| low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); |
| lenum = cpu_to_le64(offset); |
| low_crc = btrfs_crc32c(low_crc, &lenum, sizeof(lenum)); |
| |
| return ((u64)high_crc << 31) ^ (u64)low_crc; |
| } |
| |
| static u64 hash_extent_data_ref_item(struct extent_buffer *leaf, |
| struct btrfs_extent_data_ref *ref) |
| { |
| return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref), |
| btrfs_extent_data_ref_objectid(leaf, ref), |
| btrfs_extent_data_ref_offset(leaf, ref)); |
| } |
| |
| static int match_extent_data_ref(struct extent_buffer *leaf, |
| struct btrfs_extent_data_ref *ref, |
| u64 root_objectid, u64 owner, u64 offset) |
| { |
| if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid || |
| btrfs_extent_data_ref_objectid(leaf, ref) != owner || |
| btrfs_extent_data_ref_offset(leaf, ref) != offset) |
| return 0; |
| return 1; |
| } |
| |
| static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 parent, |
| u64 root_objectid, |
| u64 owner, u64 offset) |
| { |
| struct btrfs_root *root = fs_info->extent_root; |
| struct btrfs_key key; |
| struct btrfs_extent_data_ref *ref; |
| struct extent_buffer *leaf; |
| u32 nritems; |
| int ret; |
| int recow; |
| int err = -ENOENT; |
| |
| key.objectid = bytenr; |
| if (parent) { |
| key.type = BTRFS_SHARED_DATA_REF_KEY; |
| key.offset = parent; |
| } else { |
| key.type = BTRFS_EXTENT_DATA_REF_KEY; |
| key.offset = hash_extent_data_ref(root_objectid, |
| owner, offset); |
| } |
| again: |
| recow = 0; |
| ret = btrfs_search_slot(trans, root, &key, path, -1, 1); |
| if (ret < 0) { |
| err = ret; |
| goto fail; |
| } |
| |
| if (parent) { |
| if (!ret) |
| return 0; |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| key.type = BTRFS_EXTENT_REF_V0_KEY; |
| btrfs_release_path(path); |
| ret = btrfs_search_slot(trans, root, &key, path, -1, 1); |
| if (ret < 0) { |
| err = ret; |
| goto fail; |
| } |
| if (!ret) |
| return 0; |
| #endif |
| goto fail; |
| } |
| |
| leaf = path->nodes[0]; |
| nritems = btrfs_header_nritems(leaf); |
| while (1) { |
| if (path->slots[0] >= nritems) { |
| ret = btrfs_next_leaf(root, path); |
| if (ret < 0) |
| err = ret; |
| if (ret) |
| goto fail; |
| |
| leaf = path->nodes[0]; |
| nritems = btrfs_header_nritems(leaf); |
| recow = 1; |
| } |
| |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| if (key.objectid != bytenr || |
| key.type != BTRFS_EXTENT_DATA_REF_KEY) |
| goto fail; |
| |
| ref = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_data_ref); |
| |
| if (match_extent_data_ref(leaf, ref, root_objectid, |
| owner, offset)) { |
| if (recow) { |
| btrfs_release_path(path); |
| goto again; |
| } |
| err = 0; |
| break; |
| } |
| path->slots[0]++; |
| } |
| fail: |
| return err; |
| } |
| |
| static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 parent, |
| u64 root_objectid, u64 owner, |
| u64 offset, int refs_to_add) |
| { |
| struct btrfs_root *root = fs_info->extent_root; |
| struct btrfs_key key; |
| struct extent_buffer *leaf; |
| u32 size; |
| u32 num_refs; |
| int ret; |
| |
| key.objectid = bytenr; |
| if (parent) { |
| key.type = BTRFS_SHARED_DATA_REF_KEY; |
| key.offset = parent; |
| size = sizeof(struct btrfs_shared_data_ref); |
| } else { |
| key.type = BTRFS_EXTENT_DATA_REF_KEY; |
| key.offset = hash_extent_data_ref(root_objectid, |
| owner, offset); |
| size = sizeof(struct btrfs_extent_data_ref); |
| } |
| |
| ret = btrfs_insert_empty_item(trans, root, path, &key, size); |
| if (ret && ret != -EEXIST) |
| goto fail; |
| |
| leaf = path->nodes[0]; |
| if (parent) { |
| struct btrfs_shared_data_ref *ref; |
| ref = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_shared_data_ref); |
| if (ret == 0) { |
| btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add); |
| } else { |
| num_refs = btrfs_shared_data_ref_count(leaf, ref); |
| num_refs += refs_to_add; |
| btrfs_set_shared_data_ref_count(leaf, ref, num_refs); |
| } |
| } else { |
| struct btrfs_extent_data_ref *ref; |
| while (ret == -EEXIST) { |
| ref = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_data_ref); |
| if (match_extent_data_ref(leaf, ref, root_objectid, |
| owner, offset)) |
| break; |
| btrfs_release_path(path); |
| key.offset++; |
| ret = btrfs_insert_empty_item(trans, root, path, &key, |
| size); |
| if (ret && ret != -EEXIST) |
| goto fail; |
| |
| leaf = path->nodes[0]; |
| } |
| ref = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_data_ref); |
| if (ret == 0) { |
| btrfs_set_extent_data_ref_root(leaf, ref, |
| root_objectid); |
| btrfs_set_extent_data_ref_objectid(leaf, ref, owner); |
| btrfs_set_extent_data_ref_offset(leaf, ref, offset); |
| btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add); |
| } else { |
| num_refs = btrfs_extent_data_ref_count(leaf, ref); |
| num_refs += refs_to_add; |
| btrfs_set_extent_data_ref_count(leaf, ref, num_refs); |
| } |
| } |
| btrfs_mark_buffer_dirty(leaf); |
| ret = 0; |
| fail: |
| btrfs_release_path(path); |
| return ret; |
| } |
| |
| static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| int refs_to_drop, int *last_ref) |
| { |
| struct btrfs_key key; |
| struct btrfs_extent_data_ref *ref1 = NULL; |
| struct btrfs_shared_data_ref *ref2 = NULL; |
| struct extent_buffer *leaf; |
| u32 num_refs = 0; |
| int ret = 0; |
| |
| leaf = path->nodes[0]; |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| |
| if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { |
| ref1 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_data_ref); |
| num_refs = btrfs_extent_data_ref_count(leaf, ref1); |
| } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { |
| ref2 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_shared_data_ref); |
| num_refs = btrfs_shared_data_ref_count(leaf, ref2); |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { |
| struct btrfs_extent_ref_v0 *ref0; |
| ref0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_ref_v0); |
| num_refs = btrfs_ref_count_v0(leaf, ref0); |
| #endif |
| } else { |
| BUG(); |
| } |
| |
| BUG_ON(num_refs < refs_to_drop); |
| num_refs -= refs_to_drop; |
| |
| if (num_refs == 0) { |
| ret = btrfs_del_item(trans, fs_info->extent_root, path); |
| *last_ref = 1; |
| } else { |
| if (key.type == BTRFS_EXTENT_DATA_REF_KEY) |
| btrfs_set_extent_data_ref_count(leaf, ref1, num_refs); |
| else if (key.type == BTRFS_SHARED_DATA_REF_KEY) |
| btrfs_set_shared_data_ref_count(leaf, ref2, num_refs); |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| else { |
| struct btrfs_extent_ref_v0 *ref0; |
| ref0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_ref_v0); |
| btrfs_set_ref_count_v0(leaf, ref0, num_refs); |
| } |
| #endif |
| btrfs_mark_buffer_dirty(leaf); |
| } |
| return ret; |
| } |
| |
| static noinline u32 extent_data_ref_count(struct btrfs_path *path, |
| struct btrfs_extent_inline_ref *iref) |
| { |
| struct btrfs_key key; |
| struct extent_buffer *leaf; |
| struct btrfs_extent_data_ref *ref1; |
| struct btrfs_shared_data_ref *ref2; |
| u32 num_refs = 0; |
| int type; |
| |
| leaf = path->nodes[0]; |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| if (iref) { |
| /* |
| * If type is invalid, we should have bailed out earlier than |
| * this call. |
| */ |
| type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA); |
| ASSERT(type != BTRFS_REF_TYPE_INVALID); |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) { |
| ref1 = (struct btrfs_extent_data_ref *)(&iref->offset); |
| num_refs = btrfs_extent_data_ref_count(leaf, ref1); |
| } else { |
| ref2 = (struct btrfs_shared_data_ref *)(iref + 1); |
| num_refs = btrfs_shared_data_ref_count(leaf, ref2); |
| } |
| } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { |
| ref1 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_data_ref); |
| num_refs = btrfs_extent_data_ref_count(leaf, ref1); |
| } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { |
| ref2 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_shared_data_ref); |
| num_refs = btrfs_shared_data_ref_count(leaf, ref2); |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) { |
| struct btrfs_extent_ref_v0 *ref0; |
| ref0 = btrfs_item_ptr(leaf, path->slots[0], |
| struct btrfs_extent_ref_v0); |
| num_refs = btrfs_ref_count_v0(leaf, ref0); |
| #endif |
| } else { |
| WARN_ON(1); |
| } |
| return num_refs; |
| } |
| |
| static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 parent, |
| u64 root_objectid) |
| { |
| struct btrfs_root *root = fs_info->extent_root; |
| struct btrfs_key key; |
| int ret; |
| |
| key.objectid = bytenr; |
| if (parent) { |
| key.type = BTRFS_SHARED_BLOCK_REF_KEY; |
| key.offset = parent; |
| } else { |
| key.type = BTRFS_TREE_BLOCK_REF_KEY; |
| key.offset = root_objectid; |
| } |
| |
| ret = btrfs_search_slot(trans, root, &key, path, -1, 1); |
| if (ret > 0) |
| ret = -ENOENT; |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| if (ret == -ENOENT && parent) { |
| btrfs_release_path(path); |
| key.type = BTRFS_EXTENT_REF_V0_KEY; |
| ret = btrfs_search_slot(trans, root, &key, path, -1, 1); |
| if (ret > 0) |
| ret = -ENOENT; |
| } |
| #endif |
| return ret; |
| } |
| |
| static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 parent, |
| u64 root_objectid) |
| { |
| struct btrfs_key key; |
| int ret; |
| |
| key.objectid = bytenr; |
| if (parent) { |
| key.type = BTRFS_SHARED_BLOCK_REF_KEY; |
| key.offset = parent; |
| } else { |
| key.type = BTRFS_TREE_BLOCK_REF_KEY; |
| key.offset = root_objectid; |
| } |
| |
| ret = btrfs_insert_empty_item(trans, fs_info->extent_root, |
| path, &key, 0); |
| btrfs_release_path(path); |
| return ret; |
| } |
| |
| static inline int extent_ref_type(u64 parent, u64 owner) |
| { |
| int type; |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| if (parent > 0) |
| type = BTRFS_SHARED_BLOCK_REF_KEY; |
| else |
| type = BTRFS_TREE_BLOCK_REF_KEY; |
| } else { |
| if (parent > 0) |
| type = BTRFS_SHARED_DATA_REF_KEY; |
| else |
| type = BTRFS_EXTENT_DATA_REF_KEY; |
| } |
| return type; |
| } |
| |
| static int find_next_key(struct btrfs_path *path, int level, |
| struct btrfs_key *key) |
| |
| { |
| for (; level < BTRFS_MAX_LEVEL; level++) { |
| if (!path->nodes[level]) |
| break; |
| if (path->slots[level] + 1 >= |
| btrfs_header_nritems(path->nodes[level])) |
| continue; |
| if (level == 0) |
| btrfs_item_key_to_cpu(path->nodes[level], key, |
| path->slots[level] + 1); |
| else |
| btrfs_node_key_to_cpu(path->nodes[level], key, |
| path->slots[level] + 1); |
| return 0; |
| } |
| return 1; |
| } |
| |
| /* |
| * look for inline back ref. if back ref is found, *ref_ret is set |
| * to the address of inline back ref, and 0 is returned. |
| * |
| * if back ref isn't found, *ref_ret is set to the address where it |
| * should be inserted, and -ENOENT is returned. |
| * |
| * if insert is true and there are too many inline back refs, the path |
| * points to the extent item, and -EAGAIN is returned. |
| * |
| * NOTE: inline back refs are ordered in the same way that back ref |
| * items in the tree are ordered. |
| */ |
| static noinline_for_stack |
| int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| struct btrfs_extent_inline_ref **ref_ret, |
| u64 bytenr, u64 num_bytes, |
| u64 parent, u64 root_objectid, |
| u64 owner, u64 offset, int insert) |
| { |
| struct btrfs_root *root = fs_info->extent_root; |
| struct btrfs_key key; |
| struct extent_buffer *leaf; |
| struct btrfs_extent_item *ei; |
| struct btrfs_extent_inline_ref *iref; |
| u64 flags; |
| u64 item_size; |
| unsigned long ptr; |
| unsigned long end; |
| int extra_size; |
| int type; |
| int want; |
| int ret; |
| int err = 0; |
| bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); |
| int needed; |
| |
| key.objectid = bytenr; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| key.offset = num_bytes; |
| |
| want = extent_ref_type(parent, owner); |
| if (insert) { |
| extra_size = btrfs_extent_inline_ref_size(want); |
| path->keep_locks = 1; |
| } else |
| extra_size = -1; |
| |
| /* |
| * Owner is our parent level, so we can just add one to get the level |
| * for the block we are interested in. |
| */ |
| if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) { |
| key.type = BTRFS_METADATA_ITEM_KEY; |
| key.offset = owner; |
| } |
| |
| again: |
| ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1); |
| if (ret < 0) { |
| err = ret; |
| goto out; |
| } |
| |
| /* |
| * We may be a newly converted file system which still has the old fat |
| * extent entries for metadata, so try and see if we have one of those. |
| */ |
| if (ret > 0 && skinny_metadata) { |
| skinny_metadata = false; |
| if (path->slots[0]) { |
| path->slots[0]--; |
| btrfs_item_key_to_cpu(path->nodes[0], &key, |
| path->slots[0]); |
| if (key.objectid == bytenr && |
| key.type == BTRFS_EXTENT_ITEM_KEY && |
| key.offset == num_bytes) |
| ret = 0; |
| } |
| if (ret) { |
| key.objectid = bytenr; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| key.offset = num_bytes; |
| btrfs_release_path(path); |
| goto again; |
| } |
| } |
| |
| if (ret && !insert) { |
| err = -ENOENT; |
| goto out; |
| } else if (WARN_ON(ret)) { |
| err = -EIO; |
| goto out; |
| } |
| |
| leaf = path->nodes[0]; |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| if (item_size < sizeof(*ei)) { |
| if (!insert) { |
| err = -ENOENT; |
| goto out; |
| } |
| ret = convert_extent_item_v0(trans, fs_info, path, owner, |
| extra_size); |
| if (ret < 0) { |
| err = ret; |
| goto out; |
| } |
| leaf = path->nodes[0]; |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| } |
| #endif |
| BUG_ON(item_size < sizeof(*ei)); |
| |
| ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| flags = btrfs_extent_flags(leaf, ei); |
| |
| ptr = (unsigned long)(ei + 1); |
| end = (unsigned long)ei + item_size; |
| |
| if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) { |
| ptr += sizeof(struct btrfs_tree_block_info); |
| BUG_ON(ptr > end); |
| } |
| |
| if (owner >= BTRFS_FIRST_FREE_OBJECTID) |
| needed = BTRFS_REF_TYPE_DATA; |
| else |
| needed = BTRFS_REF_TYPE_BLOCK; |
| |
| err = -ENOENT; |
| while (1) { |
| if (ptr >= end) { |
| WARN_ON(ptr > end); |
| break; |
| } |
| iref = (struct btrfs_extent_inline_ref *)ptr; |
| type = btrfs_get_extent_inline_ref_type(leaf, iref, needed); |
| if (type == BTRFS_REF_TYPE_INVALID) { |
| err = -EINVAL; |
| goto out; |
| } |
| |
| if (want < type) |
| break; |
| if (want > type) { |
| ptr += btrfs_extent_inline_ref_size(type); |
| continue; |
| } |
| |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) { |
| struct btrfs_extent_data_ref *dref; |
| dref = (struct btrfs_extent_data_ref *)(&iref->offset); |
| if (match_extent_data_ref(leaf, dref, root_objectid, |
| owner, offset)) { |
| err = 0; |
| break; |
| } |
| if (hash_extent_data_ref_item(leaf, dref) < |
| hash_extent_data_ref(root_objectid, owner, offset)) |
| break; |
| } else { |
| u64 ref_offset; |
| ref_offset = btrfs_extent_inline_ref_offset(leaf, iref); |
| if (parent > 0) { |
| if (parent == ref_offset) { |
| err = 0; |
| break; |
| } |
| if (ref_offset < parent) |
| break; |
| } else { |
| if (root_objectid == ref_offset) { |
| err = 0; |
| break; |
| } |
| if (ref_offset < root_objectid) |
| break; |
| } |
| } |
| ptr += btrfs_extent_inline_ref_size(type); |
| } |
| if (err == -ENOENT && insert) { |
| if (item_size + extra_size >= |
| BTRFS_MAX_EXTENT_ITEM_SIZE(root)) { |
| err = -EAGAIN; |
| goto out; |
| } |
| /* |
| * To add new inline back ref, we have to make sure |
| * there is no corresponding back ref item. |
| * For simplicity, we just do not add new inline back |
| * ref if there is any kind of item for this block |
| */ |
| if (find_next_key(path, 0, &key) == 0 && |
| key.objectid == bytenr && |
| key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) { |
| err = -EAGAIN; |
| goto out; |
| } |
| } |
| *ref_ret = (struct btrfs_extent_inline_ref *)ptr; |
| out: |
| if (insert) { |
| path->keep_locks = 0; |
| btrfs_unlock_up_safe(path, 1); |
| } |
| return err; |
| } |
| |
| /* |
| * helper to add new inline back ref |
| */ |
| static noinline_for_stack |
| void setup_inline_extent_backref(struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| struct btrfs_extent_inline_ref *iref, |
| u64 parent, u64 root_objectid, |
| u64 owner, u64 offset, int refs_to_add, |
| struct btrfs_delayed_extent_op *extent_op) |
| { |
| struct extent_buffer *leaf; |
| struct btrfs_extent_item *ei; |
| unsigned long ptr; |
| unsigned long end; |
| unsigned long item_offset; |
| u64 refs; |
| int size; |
| int type; |
| |
| leaf = path->nodes[0]; |
| ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| item_offset = (unsigned long)iref - (unsigned long)ei; |
| |
| type = extent_ref_type(parent, owner); |
| size = btrfs_extent_inline_ref_size(type); |
| |
| btrfs_extend_item(fs_info, path, size); |
| |
| ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| refs = btrfs_extent_refs(leaf, ei); |
| refs += refs_to_add; |
| btrfs_set_extent_refs(leaf, ei, refs); |
| if (extent_op) |
| __run_delayed_extent_op(extent_op, leaf, ei); |
| |
| ptr = (unsigned long)ei + item_offset; |
| end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]); |
| if (ptr < end - size) |
| memmove_extent_buffer(leaf, ptr + size, ptr, |
| end - size - ptr); |
| |
| iref = (struct btrfs_extent_inline_ref *)ptr; |
| btrfs_set_extent_inline_ref_type(leaf, iref, type); |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) { |
| struct btrfs_extent_data_ref *dref; |
| dref = (struct btrfs_extent_data_ref *)(&iref->offset); |
| btrfs_set_extent_data_ref_root(leaf, dref, root_objectid); |
| btrfs_set_extent_data_ref_objectid(leaf, dref, owner); |
| btrfs_set_extent_data_ref_offset(leaf, dref, offset); |
| btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add); |
| } else if (type == BTRFS_SHARED_DATA_REF_KEY) { |
| struct btrfs_shared_data_ref *sref; |
| sref = (struct btrfs_shared_data_ref *)(iref + 1); |
| btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add); |
| btrfs_set_extent_inline_ref_offset(leaf, iref, parent); |
| } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) { |
| btrfs_set_extent_inline_ref_offset(leaf, iref, parent); |
| } else { |
| btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); |
| } |
| btrfs_mark_buffer_dirty(leaf); |
| } |
| |
| static int lookup_extent_backref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| struct btrfs_extent_inline_ref **ref_ret, |
| u64 bytenr, u64 num_bytes, u64 parent, |
| u64 root_objectid, u64 owner, u64 offset) |
| { |
| int ret; |
| |
| ret = lookup_inline_extent_backref(trans, fs_info, path, ref_ret, |
| bytenr, num_bytes, parent, |
| root_objectid, owner, offset, 0); |
| if (ret != -ENOENT) |
| return ret; |
| |
| btrfs_release_path(path); |
| *ref_ret = NULL; |
| |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| ret = lookup_tree_block_ref(trans, fs_info, path, bytenr, |
| parent, root_objectid); |
| } else { |
| ret = lookup_extent_data_ref(trans, fs_info, path, bytenr, |
| parent, root_objectid, owner, |
| offset); |
| } |
| return ret; |
| } |
| |
| /* |
| * helper to update/remove inline back ref |
| */ |
| static noinline_for_stack |
| void update_inline_extent_backref(struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| struct btrfs_extent_inline_ref *iref, |
| int refs_to_mod, |
| struct btrfs_delayed_extent_op *extent_op, |
| int *last_ref) |
| { |
| struct extent_buffer *leaf; |
| struct btrfs_extent_item *ei; |
| struct btrfs_extent_data_ref *dref = NULL; |
| struct btrfs_shared_data_ref *sref = NULL; |
| unsigned long ptr; |
| unsigned long end; |
| u32 item_size; |
| int size; |
| int type; |
| u64 refs; |
| |
| leaf = path->nodes[0]; |
| ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| refs = btrfs_extent_refs(leaf, ei); |
| WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0); |
| refs += refs_to_mod; |
| btrfs_set_extent_refs(leaf, ei, refs); |
| if (extent_op) |
| __run_delayed_extent_op(extent_op, leaf, ei); |
| |
| /* |
| * If type is invalid, we should have bailed out after |
| * lookup_inline_extent_backref(). |
| */ |
| type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_ANY); |
| ASSERT(type != BTRFS_REF_TYPE_INVALID); |
| |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) { |
| dref = (struct btrfs_extent_data_ref *)(&iref->offset); |
| refs = btrfs_extent_data_ref_count(leaf, dref); |
| } else if (type == BTRFS_SHARED_DATA_REF_KEY) { |
| sref = (struct btrfs_shared_data_ref *)(iref + 1); |
| refs = btrfs_shared_data_ref_count(leaf, sref); |
| } else { |
| refs = 1; |
| BUG_ON(refs_to_mod != -1); |
| } |
| |
| BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod); |
| refs += refs_to_mod; |
| |
| if (refs > 0) { |
| if (type == BTRFS_EXTENT_DATA_REF_KEY) |
| btrfs_set_extent_data_ref_count(leaf, dref, refs); |
| else |
| btrfs_set_shared_data_ref_count(leaf, sref, refs); |
| } else { |
| *last_ref = 1; |
| size = btrfs_extent_inline_ref_size(type); |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| ptr = (unsigned long)iref; |
| end = (unsigned long)ei + item_size; |
| if (ptr + size < end) |
| memmove_extent_buffer(leaf, ptr, ptr + size, |
| end - ptr - size); |
| item_size -= size; |
| btrfs_truncate_item(fs_info, path, item_size, 1); |
| } |
| btrfs_mark_buffer_dirty(leaf); |
| } |
| |
| static noinline_for_stack |
| int insert_inline_extent_backref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 num_bytes, u64 parent, |
| u64 root_objectid, u64 owner, |
| u64 offset, int refs_to_add, |
| struct btrfs_delayed_extent_op *extent_op) |
| { |
| struct btrfs_extent_inline_ref *iref; |
| int ret; |
| |
| ret = lookup_inline_extent_backref(trans, fs_info, path, &iref, |
| bytenr, num_bytes, parent, |
| root_objectid, owner, offset, 1); |
| if (ret == 0) { |
| BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID); |
| update_inline_extent_backref(fs_info, path, iref, |
| refs_to_add, extent_op, NULL); |
| } else if (ret == -ENOENT) { |
| setup_inline_extent_backref(fs_info, path, iref, parent, |
| root_objectid, owner, offset, |
| refs_to_add, extent_op); |
| ret = 0; |
| } |
| return ret; |
| } |
| |
| static int insert_extent_backref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| u64 bytenr, u64 parent, u64 root_objectid, |
| u64 owner, u64 offset, int refs_to_add) |
| { |
| int ret; |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| BUG_ON(refs_to_add != 1); |
| ret = insert_tree_block_ref(trans, fs_info, path, bytenr, |
| parent, root_objectid); |
| } else { |
| ret = insert_extent_data_ref(trans, fs_info, path, bytenr, |
| parent, root_objectid, |
| owner, offset, refs_to_add); |
| } |
| return ret; |
| } |
| |
| static int remove_extent_backref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_path *path, |
| struct btrfs_extent_inline_ref *iref, |
| int refs_to_drop, int is_data, int *last_ref) |
| { |
| int ret = 0; |
| |
| BUG_ON(!is_data && refs_to_drop != 1); |
| if (iref) { |
| update_inline_extent_backref(fs_info, path, iref, |
| -refs_to_drop, NULL, last_ref); |
| } else if (is_data) { |
| ret = remove_extent_data_ref(trans, fs_info, path, refs_to_drop, |
| last_ref); |
| } else { |
| *last_ref = 1; |
| ret = btrfs_del_item(trans, fs_info->extent_root, path); |
| } |
| return ret; |
| } |
| |
| #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len)) |
| static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, |
| u64 *discarded_bytes) |
| { |
| int j, ret = 0; |
| u64 bytes_left, end; |
| u64 aligned_start = ALIGN(start, 1 << 9); |
| |
| if (WARN_ON(start != aligned_start)) { |
| len -= aligned_start - start; |
| len = round_down(len, 1 << 9); |
| start = aligned_start; |
| } |
| |
| *discarded_bytes = 0; |
| |
| if (!len) |
| return 0; |
| |
| end = start + len; |
| bytes_left = len; |
| |
| /* Skip any superblocks on this device. */ |
| for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) { |
| u64 sb_start = btrfs_sb_offset(j); |
| u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE; |
| u64 size = sb_start - start; |
| |
| if (!in_range(sb_start, start, bytes_left) && |
| !in_range(sb_end, start, bytes_left) && |
| !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE)) |
| continue; |
| |
| /* |
| * Superblock spans beginning of range. Adjust start and |
| * try again. |
| */ |
| if (sb_start <= start) { |
| start += sb_end - start; |
| if (start > end) { |
| bytes_left = 0; |
| break; |
| } |
| bytes_left = end - start; |
| continue; |
| } |
| |
| if (size) { |
| ret = blkdev_issue_discard(bdev, start >> 9, size >> 9, |
| GFP_NOFS, 0); |
| if (!ret) |
| *discarded_bytes += size; |
| else if (ret != -EOPNOTSUPP) |
| return ret; |
| } |
| |
| start = sb_end; |
| if (start > end) { |
| bytes_left = 0; |
| break; |
| } |
| bytes_left = end - start; |
| } |
| |
| if (bytes_left) { |
| ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9, |
| GFP_NOFS, 0); |
| if (!ret) |
| *discarded_bytes += bytes_left; |
| } |
| return ret; |
| } |
| |
| int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, |
| u64 num_bytes, u64 *actual_bytes) |
| { |
| int ret; |
| u64 discarded_bytes = 0; |
| struct btrfs_bio *bbio = NULL; |
| |
| |
| /* |
| * Avoid races with device replace and make sure our bbio has devices |
| * associated to its stripes that don't go away while we are discarding. |
| */ |
| btrfs_bio_counter_inc_blocked(fs_info); |
| /* Tell the block device(s) that the sectors can be discarded */ |
| ret = btrfs_map_block(fs_info, BTRFS_MAP_DISCARD, bytenr, &num_bytes, |
| &bbio, 0); |
| /* Error condition is -ENOMEM */ |
| if (!ret) { |
| struct btrfs_bio_stripe *stripe = bbio->stripes; |
| int i; |
| |
| |
| for (i = 0; i < bbio->num_stripes; i++, stripe++) { |
| u64 bytes; |
| if (!stripe->dev->can_discard) |
| continue; |
| |
| ret = btrfs_issue_discard(stripe->dev->bdev, |
| stripe->physical, |
| stripe->length, |
| &bytes); |
| if (!ret) |
| discarded_bytes += bytes; |
| else if (ret != -EOPNOTSUPP) |
| break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */ |
| |
| /* |
| * Just in case we get back EOPNOTSUPP for some reason, |
| * just ignore the return value so we don't screw up |
| * people calling discard_extent. |
| */ |
| ret = 0; |
| } |
| btrfs_put_bbio(bbio); |
| } |
| btrfs_bio_counter_dec(fs_info); |
| |
| if (actual_bytes) |
| *actual_bytes = discarded_bytes; |
| |
| |
| if (ret == -EOPNOTSUPP) |
| ret = 0; |
| return ret; |
| } |
| |
| /* Can return -ENOMEM */ |
| int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| u64 bytenr, u64 num_bytes, u64 parent, |
| u64 root_objectid, u64 owner, u64 offset) |
| { |
| int old_ref_mod, new_ref_mod; |
| int ret; |
| |
| BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID && |
| root_objectid == BTRFS_TREE_LOG_OBJECTID); |
| |
| if (owner < BTRFS_FIRST_FREE_OBJECTID) { |
| ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr, |
| num_bytes, parent, |
| root_objectid, (int)owner, |
| BTRFS_ADD_DELAYED_REF, NULL, |
| &old_ref_mod, &new_ref_mod); |
| } else { |
| ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr, |
| num_bytes, parent, |
| root_objectid, owner, offset, |
| 0, BTRFS_ADD_DELAYED_REF, |
| &old_ref_mod, &new_ref_mod); |
| } |
| |
| if (ret == 0 && old_ref_mod < 0 && new_ref_mod >= 0) |
| add_pinned_bytes(fs_info, -num_bytes, owner, root_objectid); |
| |
| return ret; |
| } |
| |
| static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, |
| u64 parent, u64 root_objectid, |
| u64 owner, u64 offset, int refs_to_add, |
| struct btrfs_delayed_extent_op *extent_op) |
| { |
| struct btrfs_path *path; |
| struct extent_buffer *leaf; |
| struct btrfs_extent_item *item; |
| struct btrfs_key key; |
| u64 bytenr = node->bytenr; |
| u64 num_bytes = node->num_bytes; |
| u64 refs; |
| int ret; |
| |
| path = btrfs_alloc_path(); |
| if (!path) |
| return -ENOMEM; |
| |
| path->reada = READA_FORWARD; |
| path->leave_spinning = 1; |
| /* this will setup the path even if it fails to insert the back ref */ |
| ret = insert_inline_extent_backref(trans, fs_info, path, bytenr, |
| num_bytes, parent, root_objectid, |
| owner, offset, |
| refs_to_add, extent_op); |
| if ((ret < 0 && ret != -EAGAIN) || !ret) |
| goto out; |
| |
| /* |
| * Ok we had -EAGAIN which means we didn't have space to insert and |
| * inline extent ref, so just update the reference count and add a |
| * normal backref. |
| */ |
| leaf = path->nodes[0]; |
| btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); |
| item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| refs = btrfs_extent_refs(leaf, item); |
| btrfs_set_extent_refs(leaf, item, refs + refs_to_add); |
| if (extent_op) |
| __run_delayed_extent_op(extent_op, leaf, item); |
| |
| btrfs_mark_buffer_dirty(leaf); |
| btrfs_release_path(path); |
| |
| path->reada = READA_FORWARD; |
| path->leave_spinning = 1; |
| /* now insert the actual backref */ |
| ret = insert_extent_backref(trans, fs_info, path, bytenr, parent, |
| root_objectid, owner, offset, refs_to_add); |
| if (ret) |
| btrfs_abort_transaction(trans, ret); |
| out: |
| btrfs_free_path(path); |
| return ret; |
| } |
| |
| static int run_delayed_data_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, |
| struct btrfs_delayed_extent_op *extent_op, |
| int insert_reserved) |
| { |
| int ret = 0; |
| struct btrfs_delayed_data_ref *ref; |
| struct btrfs_key ins; |
| u64 parent = 0; |
| u64 ref_root = 0; |
| u64 flags = 0; |
| |
| ins.objectid = node->bytenr; |
| ins.offset = node->num_bytes; |
| ins.type = BTRFS_EXTENT_ITEM_KEY; |
| |
| ref = btrfs_delayed_node_to_data_ref(node); |
| trace_run_delayed_data_ref(fs_info, node, ref, node->action); |
| |
| if (node->type == BTRFS_SHARED_DATA_REF_KEY) |
| parent = ref->parent; |
| ref_root = ref->root; |
| |
| if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { |
| if (extent_op) |
| flags |= extent_op->flags_to_set; |
| ret = alloc_reserved_file_extent(trans, fs_info, |
| parent, ref_root, flags, |
| ref->objectid, ref->offset, |
| &ins, node->ref_mod); |
| } else if (node->action == BTRFS_ADD_DELAYED_REF) { |
| ret = __btrfs_inc_extent_ref(trans, fs_info, node, parent, |
| ref_root, ref->objectid, |
| ref->offset, node->ref_mod, |
| extent_op); |
| } else if (node->action == BTRFS_DROP_DELAYED_REF) { |
| ret = __btrfs_free_extent(trans, fs_info, node, parent, |
| ref_root, ref->objectid, |
| ref->offset, node->ref_mod, |
| extent_op); |
| } else { |
| BUG(); |
| } |
| return ret; |
| } |
| |
| static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, |
| struct extent_buffer *leaf, |
| struct btrfs_extent_item *ei) |
| { |
| u64 flags = btrfs_extent_flags(leaf, ei); |
| if (extent_op->update_flags) { |
| flags |= extent_op->flags_to_set; |
| btrfs_set_extent_flags(leaf, ei, flags); |
| } |
| |
| if (extent_op->update_key) { |
| struct btrfs_tree_block_info *bi; |
| BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)); |
| bi = (struct btrfs_tree_block_info *)(ei + 1); |
| btrfs_set_tree_block_key(leaf, bi, &extent_op->key); |
| } |
| } |
| |
| static int run_delayed_extent_op(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, |
| struct btrfs_delayed_extent_op *extent_op) |
| { |
| struct btrfs_key key; |
| struct btrfs_path *path; |
| struct btrfs_extent_item *ei; |
| struct extent_buffer *leaf; |
| u32 item_size; |
| int ret; |
| int err = 0; |
| int metadata = !extent_op->is_data; |
| |
| if (trans->aborted) |
| return 0; |
| |
| if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) |
| metadata = 0; |
| |
| path = btrfs_alloc_path(); |
| if (!path) |
| return -ENOMEM; |
| |
| key.objectid = node->bytenr; |
| |
| if (metadata) { |
| key.type = BTRFS_METADATA_ITEM_KEY; |
| key.offset = extent_op->level; |
| } else { |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| key.offset = node->num_bytes; |
| } |
| |
| again: |
| path->reada = READA_FORWARD; |
| path->leave_spinning = 1; |
| ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 1); |
| if (ret < 0) { |
| err = ret; |
| goto out; |
| } |
| if (ret > 0) { |
| if (metadata) { |
| if (path->slots[0] > 0) { |
| path->slots[0]--; |
| btrfs_item_key_to_cpu(path->nodes[0], &key, |
| path->slots[0]); |
| if (key.objectid == node->bytenr && |
| key.type == BTRFS_EXTENT_ITEM_KEY && |
| key.offset == node->num_bytes) |
| ret = 0; |
| } |
| if (ret > 0) { |
| btrfs_release_path(path); |
| metadata = 0; |
| |
| key.objectid = node->bytenr; |
| key.offset = node->num_bytes; |
| key.type = BTRFS_EXTENT_ITEM_KEY; |
| goto again; |
| } |
| } else { |
| err = -EIO; |
| goto out; |
| } |
| } |
| |
| leaf = path->nodes[0]; |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| #ifdef BTRFS_COMPAT_EXTENT_TREE_V0 |
| if (item_size < sizeof(*ei)) { |
| ret = convert_extent_item_v0(trans, fs_info, path, (u64)-1, 0); |
| if (ret < 0) { |
| err = ret; |
| goto out; |
| } |
| leaf = path->nodes[0]; |
| item_size = btrfs_item_size_nr(leaf, path->slots[0]); |
| } |
| #endif |
| BUG_ON(item_size < sizeof(*ei)); |
| ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); |
| __run_delayed_extent_op(extent_op, leaf, ei); |
| |
| btrfs_mark_buffer_dirty(leaf); |
| out: |
| btrfs_free_path(path); |
| return err; |
| } |
| |
| static int run_delayed_tree_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, |
| struct btrfs_delayed_extent_op *extent_op, |
| int insert_reserved) |
| { |
| int ret = 0; |
| struct btrfs_delayed_tree_ref *ref; |
| struct btrfs_key ins; |
| u64 parent = 0; |
| u64 ref_root = 0; |
| bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); |
| |
| ref = btrfs_delayed_node_to_tree_ref(node); |
| trace_run_delayed_tree_ref(fs_info, node, ref, node->action); |
| |
| if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) |
| parent = ref->parent; |
| ref_root = ref->root; |
| |
| ins.objectid = node->bytenr; |
| if (skinny_metadata) { |
| ins.offset = ref->level; |
| ins.type = BTRFS_METADATA_ITEM_KEY; |
| } else { |
| ins.offset = node->num_bytes; |
| ins.type = BTRFS_EXTENT_ITEM_KEY; |
| } |
| |
| if (node->ref_mod != 1) { |
| btrfs_err(fs_info, |
| "btree block(%llu) has %d references rather than 1: action %d ref_root %llu parent %llu", |
| node->bytenr, node->ref_mod, node->action, ref_root, |
| parent); |
| return -EIO; |
| } |
| if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { |
| BUG_ON(!extent_op || !extent_op->update_flags); |
| ret = alloc_reserved_tree_block(trans, fs_info, |
| parent, ref_root, |
| extent_op->flags_to_set, |
| &extent_op->key, |
| ref->level, &ins); |
| } else if (node->action == BTRFS_ADD_DELAYED_REF) { |
| ret = __btrfs_inc_extent_ref(trans, fs_info, node, |
| parent, ref_root, |
| ref->level, 0, 1, |
| extent_op); |
| } else if (node->action == BTRFS_DROP_DELAYED_REF) { |
| ret = __btrfs_free_extent(trans, fs_info, node, |
| parent, ref_root, |
| ref->level, 0, 1, extent_op); |
| } else { |
| BUG(); |
| } |
| return ret; |
| } |
| |
| /* helper function to actually process a single delayed ref entry */ |
| static int run_one_delayed_ref(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| struct btrfs_delayed_ref_node *node, |
| struct btrfs_delayed_extent_op *extent_op, |
| int insert_reserved) |
| { |
| int ret = 0; |
| |
| if (trans->aborted) { |
| if (insert_reserved) |
| btrfs_pin_extent(fs_info, node->bytenr, |
| node->num_bytes, 1); |
| return 0; |
| } |
| |
| if (btrfs_delayed_ref_is_head(node)) { |
| struct btrfs_delayed_ref_head *head; |
| /* |
| * we've hit the end of the chain and we were supposed |
| * to insert this extent into the tree. But, it got |
| * deleted before we ever needed to insert it, so all |
| * we have to do is clean up the accounting |
| */ |
| BUG_ON(extent_op); |
| head = btrfs_delayed_node_to_head(node); |
| trace_run_delayed_ref_head(fs_info, node, head, node->action); |
| |
| if (head->total_ref_mod < 0) { |
| struct btrfs_block_group_cache *cache; |
| |
| cache = btrfs_lookup_block_group(fs_info, node->bytenr); |
| ASSERT(cache); |
| percpu_counter_add(&cache->space_info->total_bytes_pinned, |
| -node->num_bytes); |
| btrfs_put_block_group(cache); |
| } |
| |
| if (insert_reserved) { |
| btrfs_pin_extent(fs_info, node->bytenr, |
| node->num_bytes, 1); |
| if (head->is_data) { |
| ret = btrfs_del_csums(trans, fs_info, |
| node->bytenr, |
| node->num_bytes); |
| } |
| } |
| |
| /* Also free its reserved qgroup space */ |
| btrfs_qgroup_free_delayed_ref(fs_info, head->qgroup_ref_root, |
| head->qgroup_reserved); |
| return ret; |
| } |
| |
| if (node->type == BTRFS_TREE_BLOCK_REF_KEY || |
| node->type == BTRFS_SHARED_BLOCK_REF_KEY) |
| ret = run_delayed_tree_ref(trans, fs_info, node, extent_op, |
| insert_reserved); |
| else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || |
| node->type == BTRFS_SHARED_DATA_REF_KEY) |
| ret = run_delayed_data_ref(trans, fs_info, node, extent_op, |
| insert_reserved); |
| else |
| BUG(); |
| return ret; |
| } |
| |
| static inline struct btrfs_delayed_ref_node * |
| select_delayed_ref(struct btrfs_delayed_ref_head *head) |
| { |
| struct btrfs_delayed_ref_node *ref; |
| |
| if (list_empty(&head->ref_list)) |
| return NULL; |
| |
| /* |
| * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first. |
| * This is to prevent a ref count from going down to zero, which deletes |
| * the extent item from the extent tree, when there still are references |
| * to add, which would fail because they would not find the extent item. |
| */ |
| if (!list_empty(&head->ref_add_list)) |
| return list_first_entry(&head->ref_add_list, |
| struct btrfs_delayed_ref_node, add_list); |
| |
| ref = list_first_entry(&head->ref_list, struct btrfs_delayed_ref_node, |
| list); |
| ASSERT(list_empty(&ref->add_list)); |
| return ref; |
| } |
| |
| /* |
| * Returns 0 on success or if called with an already aborted transaction. |
| * Returns -ENOMEM or -EIO on failure and will abort the transaction. |
| */ |
| static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, |
| unsigned long nr) |
| { |
| struct btrfs_delayed_ref_root *delayed_refs; |
| struct btrfs_delayed_ref_node *ref; |
| struct btrfs_delayed_ref_head *locked_ref = NULL; |
| struct btrfs_delayed_extent_op *extent_op; |
| ktime_t start = ktime_get(); |
| int ret; |
| unsigned long count = 0; |
| unsigned long actual_count = 0; |
| int must_insert_reserved = 0; |
| |
| delayed_refs = &trans->transaction->delayed_refs; |
| while (1) { |
| if (!locked_ref) { |
| if (count >= nr) |
| break; |
| |
| spin_lock(&delayed_refs->lock); |
| locked_ref = btrfs_select_ref_head(trans); |
| if (!locked_ref) { |
| spin_unlock(&delayed_refs->lock); |
| break; |
| } |
| |
| /* grab the lock that says we are going to process |
| * all the refs for this head */ |
| ret = btrfs_delayed_ref_lock(trans, locked_ref); |
| spin_unlock(&delayed_refs->lock); |
| /* |
| * we may have dropped the spin lock to get the head |
| * mutex lock, and that might have given someone else |
| * time to free the head. If that's true, it has been |
| * removed from our list and we can move on. |
| */ |
| if (ret == -EAGAIN) { |
| locked_ref = NULL; |
| count++; |
| continue; |
| } |
| } |
| |
| /* |
| * We need to try and merge add/drops of the same ref since we |
| * can run into issues with relocate dropping the implicit ref |
| * and then it being added back again before the drop can |
| * finish. If we merged anything we need to re-loop so we can |
| * get a good ref. |
| * Or we can get node references of the same type that weren't |
| * merged when created due to bumps in the tree mod seq, and |
| * we need to merge them to prevent adding an inline extent |
| * backref before dropping it (triggering a BUG_ON at |
| * insert_inline_extent_backref()). |
| */ |
| spin_lock(&locked_ref->lock); |
| btrfs_merge_delayed_refs(trans, fs_info, delayed_refs, |
| locked_ref); |
| |
| /* |
| * locked_ref is the head node, so we have to go one |
| * node back for any delayed ref updates |
| */ |
| ref = select_delayed_ref(locked_ref); |
| |
| if (ref && ref->seq && |
| btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) { |
| spin_unlock(&locked_ref->lock); |
| spin_lock(&delayed_refs->lock); |
| locked_ref->processing = 0; |
| delayed_refs->num_heads_ready++; |
| spin_unlock(&delayed_refs->lock); |
| btrfs_delayed_ref_unlock(locked_ref); |
| locked_ref = NULL; |
| cond_resched(); |
| count++; |
| continue; |
| } |
| |
| /* |
| * record the must insert reserved flag before we |
| * drop the spin lock. |
| */ |
| must_insert_reserved = locked_ref->must_insert_reserved; |
| locked_ref->must_insert_reserved = 0; |
| |
| extent_op = locked_ref->extent_op; |
| locked_ref->extent_op = NULL; |
| |
| if (!ref) { |
| |
| |
| /* All delayed refs have been processed, Go ahead |
| * and send the head node to run_one_delayed_ref, |
| * so that any accounting fixes can happen |
| */ |
| ref = &locked_ref->node; |
| |
| if (extent_op && must_insert_reserved) { |
| btrfs_free_delayed_extent_op(extent_op); |
| extent_op = NULL; |
| } |
| |
| if (extent_op) { |
| spin_unlock(&locked_ref->lock); |
| ret = run_delayed_extent_op(trans, fs_info, |
| ref, extent_op); |
| btrfs_free_delayed_extent_op(extent_op); |
| |
| if (ret) { |
| /* |
| * Need to reset must_insert_reserved if |
| * there was an error so the abort stuff |
| * can cleanup the reserved space |
| * properly. |
| */ |
| if (must_insert_reserved) |
| locked_ref->must_insert_reserved = 1; |
| spin_lock(&delayed_refs->lock); |
| locked_ref->processing = 0; |
| delayed_refs->num_heads_ready++; |
| spin_unlock(&delayed_refs->lock); |
| btrfs_debug(fs_info, |
| "run_delayed_extent_op returned %d", |
| ret); |
| btrfs_delayed_ref_unlock(locked_ref); |
| return ret; |
| } |
| continue; |
| } |
| |
| /* |
| * Need to drop our head ref lock and re-acquire the |
| * delayed ref lock and then re-check to make sure |
| * nobody got added. |
| */ |
| spin_unlock(&locked_ref->lock); |
| spin_lock(&delayed_refs->lock); |
| spin_lock(&locked_ref->lock); |
| if (!list_empty(&locked_ref->ref_list) || |
| locked_ref->extent_op) { |
| spin_unlock(&locked_ref->lock); |
| spin_unlock(&delayed_refs->lock); |
| continue; |
| } |
| ref->in_tree = 0; |
| delayed_refs->num_heads--; |
| rb_erase(&locked_ref->href_node, |
| &delayed_refs->href_root); |
| spin_unlock(&delayed_refs->lock); |
| } else { |
| actual_count++; |
| ref->in_tree = 0; |
| list_del(&ref->list); |
| if (!list_empty(&ref->add_list)) |
| list_del(&ref->add_list); |
| } |
| atomic_dec(&delayed_refs->num_entries); |
| |
| if (!btrfs_delayed_ref_is_head(ref)) { |
| /* |
| * when we play the delayed ref, also correct the |
| * ref_mod on head |
| */ |
| switch (ref->action) { |
| case BTRFS_ADD_DELAYED_REF: |
| case BTRFS_ADD_DELAYED_EXTENT: |
| locked_ref->node.ref_mod -= ref->ref_mod; |
| break; |
| case BTRFS_DROP_DELAYED_REF: |
| locked_ref->node.ref_mod += ref->ref_mod; |
| break; |
| default: |
| WARN_ON(1); |
| } |
| } |
| spin_unlock(&locked_ref->lock); |
| |
| ret = run_one_delayed_ref(trans, fs_info, ref, extent_op, |
| must_insert_reserved); |
| |
| btrfs_free_delayed_extent_op(extent_op); |
| if (ret) { |
| spin_lock(&delayed_refs->lock); |
| locked_ref->processing = 0; |
| delayed_refs->num_heads_ready++; |
| spin_unlock(&delayed_refs->lock); |
| btrfs_delayed_ref_unlock(locked_ref); |
| btrfs_put_delayed_ref(ref); |
| btrfs_debug(fs_info, "run_one_delayed_ref returned %d", |
| ret); |
| return ret; |
| } |
| |
| /* |
| * If this node is a head, that means all the refs in this head |
| * have been dealt with, and we will pick the next head to deal |
| * with, so we must unlock the head and drop it from the cluster |
| * list before we release it. |
| */ |
| if (btrfs_delayed_ref_is_head(ref)) { |
| if (locked_ref->is_data && |
| locked_ref->total_ref_mod < 0) { |
| spin_lock(&delayed_refs->lock); |
| delayed_refs->pending_csums -= ref->num_bytes; |
| spin_unlock(&delayed_refs->lock); |
| } |
| btrfs_delayed_ref_unlock(locked_ref); |
| locked_ref = NULL; |
| } |
| btrfs_put_delayed_ref(ref); |
| count++; |
| cond_resched(); |
| } |
| |
| /* |
| * We don't want to include ref heads since we can have empty ref heads |
| * and those will drastically skew our runtime down since we just do |
| * accounting, no actual extent tree updates. |
| */ |
| if (actual_count > 0) { |
| u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start)); |
| u64 avg; |
| |
| /* |
| * We weigh the current average higher than our current runtime |
| * to avoid large swings in the average. |
| */ |
| spin_lock(&delayed_refs->lock); |
| avg = fs_info->avg_delayed_ref_runtime * 3 + runtime; |
| fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */ |
| spin_unlock(&delayed_refs->lock); |
| } |
| return 0; |
| } |
| |
| #ifdef SCRAMBLE_DELAYED_REFS |
| /* |
| * Normally delayed refs get processed in ascending bytenr order. This |
| * correlates in most cases to the order added. To expose dependencies on this |
| * order, we start to process the tree in the middle instead of the beginning |
| */ |
| static u64 find_middle(struct rb_root *root) |
| { |
| struct rb_node *n = root->rb_node; |
| struct btrfs_delayed_ref_node *entry; |
| int alt = 1; |
| u64 middle; |
| u64 first = 0, last = 0; |
| |
| n = rb_first(root); |
| if (n) { |
| entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); |
| first = entry->bytenr; |
| } |
| n = rb_last(root); |
| if (n) { |
| entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); |
| last = entry->bytenr; |
| } |
| n = root->rb_node; |
| |
| while (n) { |
| entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); |
| WARN_ON(!entry->in_tree); |
| |
| middle = entry->bytenr; |
| |
| if (alt) |
| n = n->rb_left; |
| else |
| n = n->rb_right; |
| |
| alt = 1 - alt; |
| } |
| return middle; |
| } |
| #endif |
| |
| static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads) |
| { |
| u64 num_bytes; |
| |
| num_bytes = heads * (sizeof(struct btrfs_extent_item) + |
| sizeof(struct btrfs_extent_inline_ref)); |
| if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) |
| num_bytes += heads * sizeof(struct btrfs_tree_block_info); |
| |
| /* |
| * We don't ever fill up leaves all the way so multiply by 2 just to be |
| * closer to what we're really going to want to use. |
| */ |
| return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info)); |
| } |
| |
| /* |
| * Takes the number of bytes to be csumm'ed and figures out how many leaves it |
| * would require to store the csums for that many bytes. |
| */ |
| u64 btrfs_csum_bytes_to_leaves(struct btrfs_fs_info *fs_info, u64 csum_bytes) |
| { |
| u64 csum_size; |
| u64 num_csums_per_leaf; |
| u64 num_csums; |
| |
| csum_size = BTRFS_MAX_ITEM_SIZE(fs_info); |
| num_csums_per_leaf = div64_u64(csum_size, |
| (u64)btrfs_super_csum_size(fs_info->super_copy)); |
| num_csums = div64_u64(csum_bytes, fs_info->sectorsize); |
| num_csums += num_csums_per_leaf - 1; |
| num_csums = div64_u64(num_csums, num_csums_per_leaf); |
| return num_csums; |
| } |
| |
| int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info) |
| { |
| struct btrfs_block_rsv *global_rsv; |
| u64 num_heads = trans->transaction->delayed_refs.num_heads_ready; |
| u64 csum_bytes = trans->transaction->delayed_refs.pending_csums; |
| u64 num_dirty_bgs = trans->transaction->num_dirty_bgs; |
| u64 num_bytes, num_dirty_bgs_bytes; |
| int ret = 0; |
| |
| num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1); |
| num_heads = heads_to_leaves(fs_info, num_heads); |
| if (num_heads > 1) |
| num_bytes += (num_heads - 1) * fs_info->nodesize; |
| num_bytes <<= 1; |
| num_bytes += btrfs_csum_bytes_to_leaves(fs_info, csum_bytes) * |
| fs_info->nodesize; |
| num_dirty_bgs_bytes = btrfs_calc_trans_metadata_size(fs_info, |
| num_dirty_bgs); |
| global_rsv = &fs_info->global_block_rsv; |
| |
| /* |
| * If we can't allocate any more chunks lets make sure we have _lots_ of |
| * wiggle room since running delayed refs can create more delayed refs. |
| */ |
| if (global_rsv->space_info->full) { |
| num_dirty_bgs_bytes <<= 1; |
| num_bytes <<= 1; |
| } |
| |
| spin_lock(&global_rsv->lock); |
| if (global_rsv->reserved <= num_bytes + num_dirty_bgs_bytes) |
| ret = 1; |
| spin_unlock(&global_rsv->lock); |
| return ret; |
| } |
| |
| int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info) |
| { |
| u64 num_entries = |
| atomic_read(&trans->transaction->delayed_refs.num_entries); |
| u64 avg_runtime; |
| u64 val; |
| |
| smp_mb(); |
| avg_runtime = fs_info->avg_delayed_ref_runtime; |
| val = num_entries * avg_runtime; |
| if (val >= NSEC_PER_SEC) |
| return 1; |
| if (val >= NSEC_PER_SEC / 2) |
| return 2; |
| |
| return btrfs_check_space_for_delayed_refs(trans, fs_info); |
| } |
| |
| struct async_delayed_refs { |
| struct btrfs_root *root; |
| u64 transid; |
| int count; |
| int error; |
| int sync; |
| struct completion wait; |
| struct btrfs_work work; |
| }; |
| |
| static inline struct async_delayed_refs * |
| to_async_delayed_refs(struct btrfs_work *work) |
| { |
| return container_of(work, struct async_delayed_refs, work); |
| } |
| |
| static void delayed_ref_async_start(struct btrfs_work *work) |
| { |
| struct async_delayed_refs *async = to_async_delayed_refs(work); |
| struct btrfs_trans_handle *trans; |
| struct btrfs_fs_info *fs_info = async->root->fs_info; |
| int ret; |
| |
| /* if the commit is already started, we don't need to wait here */ |
| if (btrfs_transaction_blocked(fs_info)) |
| goto done; |
| |
| trans = btrfs_join_transaction(async->root); |
| if (IS_ERR(trans)) { |
| async->error = PTR_ERR(trans); |
| goto done; |
| } |
| |
| /* |
| * trans->sync means that when we call end_transaction, we won't |
| * wait on delayed refs |
| */ |
| trans->sync = true; |
| |
| /* Don't bother flushing if we got into a different transaction */ |
| if (trans->transid > async->transid) |
| goto end; |
| |
| ret = btrfs_run_delayed_refs(trans, fs_info, async->count); |
| if (ret) |
| async->error = ret; |
| end: |
| ret = btrfs_end_transaction(trans); |
| if (ret && !async->error) |
| async->error = ret; |
| done: |
| if (async->sync) |
| complete(&async->wait); |
| else |
| kfree(async); |
| } |
| |
| int btrfs_async_run_delayed_refs(struct btrfs_fs_info *fs_info, |
| unsigned long count, u64 transid, int wait) |
| { |
| struct async_delayed_refs *async; |
| int ret; |
| |
| async = kmalloc(sizeof(*async), GFP_NOFS); |
| if (!async) |
| return -ENOMEM; |
| |
| async->root = fs_info->tree_root; |
| async->count = count; |
| async->error = 0; |
| async->transid = transid; |
| if (wait) |
| async->sync = 1; |
| else |
| async->sync = 0; |
| init_completion(&async->wait); |
| |
| btrfs_init_work(&async->work, btrfs_extent_refs_helper, |
| delayed_ref_async_start, NULL, NULL); |
| |
| btrfs_queue_work(fs_info->extent_workers, &async->work); |
| |
| if (wait) { |
| wait_for_completion(&async->wait); |
| ret = async->error; |
| kfree(async); |
| return ret; |
| } |
| return 0; |
| } |
| |
| /* |
| * this starts processing the delayed reference count updates and |
| * extent insertions we have queued up so far. count can be |
| * 0, which means to process everything in the tree at the start |
| * of the run (but not newly added entries), or it can be some target |
| * number you'd like to process. |
| * |
| * Returns 0 on success or if called with an aborted transaction |
| * Returns <0 on error and aborts the transaction |
| */ |
| int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, |
| struct btrfs_fs_info *fs_info, unsigned long count) |
| { |
| struct rb_node *node; |
| struct btrfs_delayed_ref_root *delayed_refs; |
| struct btrfs_delayed_ref_head *head; |
| int ret; |
| int run_all = count == (unsigned long)-1; |
| bool can_flush_pending_bgs = trans->can_flush_pending_bgs; |
| |
| /* We'll clean this up in btrfs_cleanup_transaction */ |
| if (trans->aborted) |
| return 0; |
| |
| if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags)) |
| return 0; |
| |
| delayed_refs = &trans->transaction->delayed_refs; |
| if |