1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (C) 2007 Oracle. All rights reserved.
4	*/
5
6	#include <crypto/hash.h>
7	#include <linux/kernel.h>
8	#include <linux/bio.h>
9	#include <linux/blk-cgroup.h>
10	#include <linux/file.h>
11	#include <linux/fs.h>
12	#include <linux/fs_struct.h>
13	#include <linux/pagemap.h>
14	#include <linux/highmem.h>
15	#include <linux/time.h>
16	#include <linux/init.h>
17	#include <linux/string.h>
18	#include <linux/backing-dev.h>
19	#include <linux/writeback.h>
20	#include <linux/compat.h>
21	#include <linux/xattr.h>
22	#include <linux/posix_acl.h>
23	#include <linux/falloc.h>
24	#include <linux/slab.h>
25	#include <linux/ratelimit.h>
26	#include <linux/btrfs.h>
27	#include <linux/blkdev.h>
28	#include <linux/posix_acl_xattr.h>
29	#include <linux/uio.h>
30	#include <linux/magic.h>
31	#include <linux/iversion.h>
32	#include <linux/swap.h>
33	#include <linux/migrate.h>
34	#include <linux/sched/mm.h>
35	#include <linux/iomap.h>
36	#include <linux/unaligned.h>
37	#include <linux/fsverity.h>
38	#include "misc.h"
39	#include "ctree.h"
40	#include "disk-io.h"
41	#include "transaction.h"
42	#include "btrfs_inode.h"
43	#include "ordered-data.h"
44	#include "xattr.h"
45	#include "tree-log.h"
46	#include "bio.h"
47	#include "compression.h"
48	#include "locking.h"
49	#include "props.h"
50	#include "qgroup.h"
51	#include "delalloc-space.h"
52	#include "block-group.h"
53	#include "space-info.h"
54	#include "zoned.h"
55	#include "subpage.h"
56	#include "inode-item.h"
57	#include "fs.h"
58	#include "accessors.h"
59	#include "extent-tree.h"
60	#include "root-tree.h"
61	#include "defrag.h"
62	#include "dir-item.h"
63	#include "file-item.h"
64	#include "uuid-tree.h"
65	#include "ioctl.h"
66	#include "file.h"
67	#include "acl.h"
68	#include "relocation.h"
69	#include "verity.h"
70	#include "super.h"
71	#include "orphan.h"
72	#include "backref.h"
73	#include "raid-stripe-tree.h"
74	#include "fiemap.h"
75	#include "delayed-inode.h"
76
77	#define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0)
78	#define COW_FILE_RANGE_NO_INLINE (1UL << 1)
79
80	struct btrfs_iget_args {
81	u64 ino;
82	struct btrfs_root *root;
83	};
84
85	struct btrfs_rename_ctx {
86	/* Output field. Stores the index number of the old directory entry. */
87	u64 index;
88	};
89
90	/*
91	* Used by data_reloc_print_warning_inode() to pass needed info for filename
92	* resolution and output of error message.
93	*/
94	struct data_reloc_warn {
95	struct btrfs_path path;
96	struct btrfs_fs_info *fs_info;
97	u64 extent_item_size;
98	u64 logical;
99	int mirror_num;
100	};
101
102	/*
103	* For the file_extent_tree, we want to hold the inode lock when we lookup and
104	* update the disk_i_size, but lockdep will complain because our io_tree we hold
105	* the tree lock and get the inode lock when setting delalloc. These two things
106	* are unrelated, so make a class for the file_extent_tree so we don't get the
107	* two locking patterns mixed up.
108	*/
109	static struct lock_class_key file_extent_tree_class;
110
111	static const struct inode_operations btrfs_dir_inode_operations;
112	static const struct inode_operations btrfs_symlink_inode_operations;
113	static const struct inode_operations btrfs_special_inode_operations;
114	static const struct inode_operations btrfs_file_inode_operations;
115	static const struct address_space_operations btrfs_aops;
116	static const struct file_operations btrfs_dir_file_operations;
117
118	static struct kmem_cache *btrfs_inode_cachep;
119
120	static int btrfs_setsize(struct inode *inode, struct iattr *attr);
121	static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback);
122
123	static noinline int run_delalloc_cow(struct btrfs_inode *inode,
124	struct folio *locked_folio, u64 start,
125	u64 end, struct writeback_control *wbc,
126	bool pages_dirty);
127
128	static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes,
129	u64 root, void *warn_ctx)
130	{
131	struct data_reloc_warn *warn = warn_ctx;
132	struct btrfs_fs_info *fs_info = warn->fs_info;
133	struct extent_buffer *eb;
134	struct btrfs_inode_item *inode_item;
135	struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL;
136	struct btrfs_root *local_root;
137	struct btrfs_key key;
138	unsigned int nofs_flag;
139	u32 nlink;
140	int ret;
141
142	local_root = btrfs_get_fs_root(fs_info, objectid: root, check_ref: true);
143	if (IS_ERR(ptr: local_root)) {
144	ret = PTR_ERR(ptr: local_root);
145	goto err;
146	}
147
148	/* This makes the path point to (inum INODE_ITEM ioff). */
149	key.objectid = inum;
150	key.type = BTRFS_INODE_ITEM_KEY;
151	key.offset = 0;
152
153	ret = btrfs_search_slot(NULL, root: local_root, key: &key, p: &warn->path, ins_len: 0, cow: 0);
154	if (ret) {
155	btrfs_put_root(root: local_root);
156	btrfs_release_path(p: &warn->path);
157	goto err;
158	}
159
160	eb = warn->path.nodes[0];
161	inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item);
162	nlink = btrfs_inode_nlink(eb, s: inode_item);
163	btrfs_release_path(p: &warn->path);
164
165	nofs_flag = memalloc_nofs_save();
166	ipath = init_ipath(total_bytes: 4096, fs_root: local_root, path: &warn->path);
167	memalloc_nofs_restore(flags: nofs_flag);
168	if (IS_ERR(ptr: ipath)) {
169	btrfs_put_root(root: local_root);
170	ret = PTR_ERR(ptr: ipath);
171	ipath = NULL;
172	/*
173	* -ENOMEM, not a critical error, just output an generic error
174	* without filename.
175	*/
176	btrfs_warn(fs_info,
177	"checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu",
178	warn->logical, warn->mirror_num, root, inum, offset);
179	return ret;
180	}
181	ret = paths_from_inode(inum, ipath);
182	if (ret < 0) {
183	btrfs_put_root(root: local_root);
184	goto err;
185	}
186
187	/*
188	* We deliberately ignore the bit ipath might have been too small to
189	* hold all of the paths here
190	*/
191	for (int i = 0; i < ipath->fspath->elem_cnt; i++) {
192	btrfs_warn(fs_info,
193	"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)",
194	warn->logical, warn->mirror_num, root, inum, offset,
195	fs_info->sectorsize, nlink,
196	(char *)(unsigned long)ipath->fspath->val[i]);
197	}
198
199	btrfs_put_root(root: local_root);
200	return 0;
201
202	err:
203	btrfs_warn(fs_info,
204	"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d",
205	warn->logical, warn->mirror_num, root, inum, offset, ret);
206
207	return ret;
208	}
209
210	/*
211	* Do extra user-friendly error output (e.g. lookup all the affected files).
212	*
213	* Return true if we succeeded doing the backref lookup.
214	* Return false if such lookup failed, and has to fallback to the old error message.
215	*/
216	static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off,
217	const u8 *csum, const u8 *csum_expected,
218	int mirror_num)
219	{
220	struct btrfs_fs_info *fs_info = inode->root->fs_info;
221	struct btrfs_path path = { 0 };
222	struct btrfs_key found_key = { 0 };
223	struct extent_buffer *eb;
224	struct btrfs_extent_item *ei;
225	const u32 csum_size = fs_info->csum_size;
226	u64 logical;
227	u64 flags;
228	u32 item_size;
229	int ret;
230
231	mutex_lock(&fs_info->reloc_mutex);
232	logical = btrfs_get_reloc_bg_bytenr(fs_info);
233	mutex_unlock(lock: &fs_info->reloc_mutex);
234
235	if (logical == U64_MAX) {
236	btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation");
237	btrfs_warn_rl(fs_info,
238	"csum failed root %lld ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
239	btrfs_root_id(inode->root), btrfs_ino(inode), file_off,
240	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
241	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
242	mirror_num);
243	return;
244	}
245
246	logical += file_off;
247	btrfs_warn_rl(fs_info,
248	"csum failed root %lld ino %llu off %llu logical %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
249	btrfs_root_id(inode->root),
250	btrfs_ino(inode), file_off, logical,
251	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
252	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
253	mirror_num);
254
255	ret = extent_from_logical(fs_info, logical, path: &path, found_key: &found_key, flags: &flags);
256	if (ret < 0) {
257	btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %d",
258	logical, ret);
259	btrfs_release_path(p: &path);
260	return;
261	}
262	eb = path.nodes[0];
263	ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item);
264	item_size = btrfs_item_size(eb, slot: path.slots[0]);
265	if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
266	unsigned long ptr = 0;
267	u64 ref_root;
268	u8 ref_level;
269
270	while (true) {
271	ret = tree_backref_for_extent(ptr: &ptr, eb, key: &found_key, ei,
272	item_size, out_root: &ref_root,
273	out_level: &ref_level);
274	if (ret < 0) {
275	btrfs_warn_rl(fs_info,
276	"failed to resolve tree backref for logical %llu: %d",
277	logical, ret);
278	break;
279	}
280	if (ret > 0)
281	break;
282
283	btrfs_warn_rl(fs_info,
284	"csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu",
285	logical, mirror_num,
286	(ref_level ? "node" : "leaf"),
287	ref_level, ref_root);
288	}
289	btrfs_release_path(p: &path);
290	} else {
291	struct btrfs_backref_walk_ctx ctx = { 0 };
292	struct data_reloc_warn reloc_warn = { 0 };
293
294	btrfs_release_path(p: &path);
295
296	ctx.bytenr = found_key.objectid;
297	ctx.extent_item_pos = logical - found_key.objectid;
298	ctx.fs_info = fs_info;
299
300	reloc_warn.logical = logical;
301	reloc_warn.extent_item_size = found_key.offset;
302	reloc_warn.mirror_num = mirror_num;
303	reloc_warn.fs_info = fs_info;
304
305	iterate_extent_inodes(ctx: &ctx, search_commit_root: true,
306	iterate: data_reloc_print_warning_inode, user_ctx: &reloc_warn);
307	}
308	}
309
310	static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode,
311	u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num)
312	{
313	struct btrfs_root *root = inode->root;
314	const u32 csum_size = root->fs_info->csum_size;
315
316	/* For data reloc tree, it's better to do a backref lookup instead. */
317	if (btrfs_is_data_reloc_root(root))
318	return print_data_reloc_error(inode, file_off: logical_start, csum,
319	csum_expected, mirror_num);
320
321	/* Output without objectid, which is more meaningful */
322	if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) {
323	btrfs_warn_rl(root->fs_info,
324	"csum failed root %lld ino %lld off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
325	btrfs_root_id(root), btrfs_ino(inode),
326	logical_start,
327	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
328	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
329	mirror_num);
330	} else {
331	btrfs_warn_rl(root->fs_info,
332	"csum failed root %llu ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d",
333	btrfs_root_id(root), btrfs_ino(inode),
334	logical_start,
335	BTRFS_CSUM_FMT_VALUE(csum_size, csum),
336	BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected),
337	mirror_num);
338	}
339	}
340
341	/*
342	* Lock inode i_rwsem based on arguments passed.
343	*
344	* ilock_flags can have the following bit set:
345	*
346	* BTRFS_ILOCK_SHARED - acquire a shared lock on the inode
347	* BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt
348	* return -EAGAIN
349	* BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock
350	*/
351	int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags)
352	{
353	if (ilock_flags & BTRFS_ILOCK_SHARED) {
354	if (ilock_flags & BTRFS_ILOCK_TRY) {
355	if (!inode_trylock_shared(inode: &inode->vfs_inode))
356	return -EAGAIN;
357	else
358	return 0;
359	}
360	inode_lock_shared(inode: &inode->vfs_inode);
361	} else {
362	if (ilock_flags & BTRFS_ILOCK_TRY) {
363	if (!inode_trylock(inode: &inode->vfs_inode))
364	return -EAGAIN;
365	else
366	return 0;
367	}
368	inode_lock(inode: &inode->vfs_inode);
369	}
370	if (ilock_flags & BTRFS_ILOCK_MMAP)
371	down_write(sem: &inode->i_mmap_lock);
372	return 0;
373	}
374
375	/*
376	* Unlock inode i_rwsem.
377	*
378	* ilock_flags should contain the same bits set as passed to btrfs_inode_lock()
379	* to decide whether the lock acquired is shared or exclusive.
380	*/
381	void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags)
382	{
383	if (ilock_flags & BTRFS_ILOCK_MMAP)
384	up_write(sem: &inode->i_mmap_lock);
385	if (ilock_flags & BTRFS_ILOCK_SHARED)
386	inode_unlock_shared(inode: &inode->vfs_inode);
387	else
388	inode_unlock(inode: &inode->vfs_inode);
389	}
390
391	/*
392	* Cleanup all submitted ordered extents in specified range to handle errors
393	* from the btrfs_run_delalloc_range() callback.
394	*
395	* NOTE: caller must ensure that when an error happens, it can not call
396	* extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
397	* and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
398	* to be released, which we want to happen only when finishing the ordered
399	* extent (btrfs_finish_ordered_io()).
400	*/
401	static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,
402	u64 offset, u64 bytes)
403	{
404	pgoff_t index = offset >> PAGE_SHIFT;
405	const pgoff_t end_index = (offset + bytes - 1) >> PAGE_SHIFT;
406	struct folio *folio;
407
408	while (index <= end_index) {
409	folio = filemap_get_folio(mapping: inode->vfs_inode.i_mapping, index);
410	if (IS_ERR(ptr: folio)) {
411	index++;
412	continue;
413	}
414
415	index = folio_next_index(folio);
416	/*
417	* Here we just clear all Ordered bits for every page in the
418	* range, then btrfs_mark_ordered_io_finished() will handle
419	* the ordered extent accounting for the range.
420	*/
421	btrfs_folio_clamp_clear_ordered(fs_info: inode->root->fs_info, folio,
422	start: offset, len: bytes);
423	folio_put(folio);
424	}
425
426	return btrfs_mark_ordered_io_finished(inode, NULL, file_offset: offset, num_bytes: bytes, uptodate: false);
427	}
428
429	static int btrfs_dirty_inode(struct btrfs_inode *inode);
430
431	static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
432	struct btrfs_new_inode_args *args)
433	{
434	int ret;
435
436	if (args->default_acl) {
437	ret = __btrfs_set_acl(trans, inode: args->inode, acl: args->default_acl,
438	ACL_TYPE_DEFAULT);
439	if (ret)
440	return ret;
441	}
442	if (args->acl) {
443	ret = __btrfs_set_acl(trans, inode: args->inode, acl: args->acl, ACL_TYPE_ACCESS);
444	if (ret)
445	return ret;
446	}
447	if (!args->default_acl && !args->acl)
448	cache_no_acl(inode: args->inode);
449	return btrfs_xattr_security_init(trans, inode: args->inode, dir: args->dir,
450	qstr: &args->dentry->d_name);
451	}
452
453	/*
454	* this does all the hard work for inserting an inline extent into
455	* the btree. The caller should have done a btrfs_drop_extents so that
456	* no overlapping inline items exist in the btree
457	*/
458	static int insert_inline_extent(struct btrfs_trans_handle *trans,
459	struct btrfs_path *path,
460	struct btrfs_inode *inode, bool extent_inserted,
461	size_t size, size_t compressed_size,
462	int compress_type,
463	struct folio *compressed_folio,
464	bool update_i_size)
465	{
466	struct btrfs_root *root = inode->root;
467	struct extent_buffer *leaf;
468	const u32 sectorsize = trans->fs_info->sectorsize;
469	char *kaddr;
470	unsigned long ptr;
471	struct btrfs_file_extent_item *ei;
472	int ret;
473	size_t cur_size = size;
474	u64 i_size;
475
476	/*
477	* The decompressed size must still be no larger than a sector. Under
478	* heavy race, we can have size == 0 passed in, but that shouldn't be a
479	* big deal and we can continue the insertion.
480	*/
481	ASSERT(size <= sectorsize);
482
483	/*
484	* The compressed size also needs to be no larger than a page.
485	* That's also why we only need one folio as the parameter.
486	*/
487	if (compressed_folio) {
488	ASSERT(compressed_size <= sectorsize);
489	ASSERT(compressed_size <= PAGE_SIZE);
490	} else {
491	ASSERT(compressed_size == 0);
492	}
493
494	if (compressed_size && compressed_folio)
495	cur_size = compressed_size;
496
497	if (!extent_inserted) {
498	struct btrfs_key key;
499	size_t datasize;
500
501	key.objectid = btrfs_ino(inode);
502	key.type = BTRFS_EXTENT_DATA_KEY;
503	key.offset = 0;
504
505	datasize = btrfs_file_extent_calc_inline_size(datasize: cur_size);
506	ret = btrfs_insert_empty_item(trans, root, path, key: &key,
507	data_size: datasize);
508	if (ret)
509	goto fail;
510	}
511	leaf = path->nodes[0];
512	ei = btrfs_item_ptr(leaf, path->slots[0],
513	struct btrfs_file_extent_item);
514	btrfs_set_file_extent_generation(eb: leaf, s: ei, val: trans->transid);
515	btrfs_set_file_extent_type(eb: leaf, s: ei, val: BTRFS_FILE_EXTENT_INLINE);
516	btrfs_set_file_extent_encryption(eb: leaf, s: ei, val: 0);
517	btrfs_set_file_extent_other_encoding(eb: leaf, s: ei, val: 0);
518	btrfs_set_file_extent_ram_bytes(eb: leaf, s: ei, val: size);
519	ptr = btrfs_file_extent_inline_start(e: ei);
520
521	if (compress_type != BTRFS_COMPRESS_NONE) {
522	kaddr = kmap_local_folio(folio: compressed_folio, offset: 0);
523	write_extent_buffer(eb: leaf, src: kaddr, start: ptr, len: compressed_size);
524	kunmap_local(kaddr);
525
526	btrfs_set_file_extent_compression(eb: leaf, s: ei,
527	val: compress_type);
528	} else {
529	struct folio *folio;
530
531	folio = filemap_get_folio(mapping: inode->vfs_inode.i_mapping, index: 0);
532	ASSERT(!IS_ERR(folio));
533	btrfs_set_file_extent_compression(eb: leaf, s: ei, val: 0);
534	kaddr = kmap_local_folio(folio, offset: 0);
535	write_extent_buffer(eb: leaf, src: kaddr, start: ptr, len: size);
536	kunmap_local(kaddr);
537	folio_put(folio);
538	}
539	btrfs_release_path(p: path);
540
541	/*
542	* We align size to sectorsize for inline extents just for simplicity
543	* sake.
544	*/
545	ret = btrfs_inode_set_file_extent_range(inode, start: 0,
546	ALIGN(size, root->fs_info->sectorsize));
547	if (ret)
548	goto fail;
549
550	/*
551	* We're an inline extent, so nobody can extend the file past i_size
552	* without locking a page we already have locked.
553	*
554	* We must do any i_size and inode updates before we unlock the pages.
555	* Otherwise we could end up racing with unlink.
556	*/
557	i_size = i_size_read(inode: &inode->vfs_inode);
558	if (update_i_size && size > i_size) {
559	i_size_write(inode: &inode->vfs_inode, i_size: size);
560	i_size = size;
561	}
562	inode->disk_i_size = i_size;
563
564	fail:
565	return ret;
566	}
567
568	static bool can_cow_file_range_inline(struct btrfs_inode *inode,
569	u64 offset, u64 size,
570	size_t compressed_size)
571	{
572	struct btrfs_fs_info *fs_info = inode->root->fs_info;
573	u64 data_len = (compressed_size ?: size);
574
575	/* Inline extents must start at offset 0. */
576	if (offset != 0)
577	return false;
578
579	/*
580	* Even for bs > ps cases, cow_file_range_inline() can only accept a
581	* single folio.
582	*
583	* This can be problematic and cause access beyond page boundary if a
584	* page sized folio is passed into that function.
585	* And encoded write is doing exactly that.
586	* So here limits the inlined extent size to PAGE_SIZE.
587	*/
588	if (size > PAGE_SIZE || compressed_size > PAGE_SIZE)
589	return false;
590
591	/* Inline extents are limited to sectorsize. */
592	if (size > fs_info->sectorsize)
593	return false;
594
595	/* We do not allow a non-compressed extent to be as large as block size. */
596	if (data_len >= fs_info->sectorsize)
597	return false;
598
599	/* We cannot exceed the maximum inline data size. */
600	if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(info: fs_info))
601	return false;
602
603	/* We cannot exceed the user specified max_inline size. */
604	if (data_len > fs_info->max_inline)
605	return false;
606
607	/* Inline extents must be the entirety of the file. */
608	if (size < i_size_read(inode: &inode->vfs_inode))
609	return false;
610
611	/* Encrypted file cannot be inlined. */
612	if (IS_ENCRYPTED(&inode->vfs_inode))
613	return false;
614
615	return true;
616	}
617
618	/*
619	* conditionally insert an inline extent into the file. This
620	* does the checks required to make sure the data is small enough
621	* to fit as an inline extent.
622	*
623	* If being used directly, you must have already checked we're allowed to cow
624	* the range by getting true from can_cow_file_range_inline().
625	*/
626	static noinline int __cow_file_range_inline(struct btrfs_inode *inode,
627	u64 size, size_t compressed_size,
628	int compress_type,
629	struct folio *compressed_folio,
630	bool update_i_size)
631	{
632	struct btrfs_drop_extents_args drop_args = { 0 };
633	struct btrfs_root *root = inode->root;
634	struct btrfs_fs_info *fs_info = root->fs_info;
635	struct btrfs_trans_handle *trans = NULL;
636	u64 data_len = (compressed_size ?: size);
637	int ret;
638	struct btrfs_path *path;
639
640	path = btrfs_alloc_path();
641	if (!path) {
642	ret = -ENOMEM;
643	goto out;
644	}
645
646	trans = btrfs_join_transaction(root);
647	if (IS_ERR(ptr: trans)) {
648	ret = PTR_ERR(ptr: trans);
649	trans = NULL;
650	goto out;
651	}
652	trans->block_rsv = &inode->block_rsv;
653
654	drop_args.path = path;
655	drop_args.start = 0;
656	drop_args.end = fs_info->sectorsize;
657	drop_args.drop_cache = true;
658	drop_args.replace_extent = true;
659	drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(datasize: data_len);
660	ret = btrfs_drop_extents(trans, root, inode, args: &drop_args);
661	if (unlikely(ret)) {
662	btrfs_abort_transaction(trans, ret);
663	goto out;
664	}
665
666	ret = insert_inline_extent(trans, path, inode, extent_inserted: drop_args.extent_inserted,
667	size, compressed_size, compress_type,
668	compressed_folio, update_i_size);
669	if (unlikely(ret && ret != -ENOSPC)) {
670	btrfs_abort_transaction(trans, ret);
671	goto out;
672	} else if (ret == -ENOSPC) {
673	ret = 1;
674	goto out;
675	}
676
677	btrfs_update_inode_bytes(inode, add_bytes: size, del_bytes: drop_args.bytes_found);
678	ret = btrfs_update_inode(trans, inode);
679	if (unlikely(ret && ret != -ENOSPC)) {
680	btrfs_abort_transaction(trans, ret);
681	goto out;
682	} else if (ret == -ENOSPC) {
683	ret = 1;
684	goto out;
685	}
686
687	btrfs_set_inode_full_sync(inode);
688	out:
689	/*
690	* Don't forget to free the reserved space, as for inlined extent
691	* it won't count as data extent, free them directly here.
692	* And at reserve time, it's always aligned to page size, so
693	* just free one page here.
694	*
695	* If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need
696	* to keep the data reservation.
697	*/
698	if (ret <= 0)
699	btrfs_qgroup_free_data(inode, NULL, start: 0, len: fs_info->sectorsize, NULL);
700	btrfs_free_path(p: path);
701	if (trans)
702	btrfs_end_transaction(trans);
703	return ret;
704	}
705
706	static noinline int cow_file_range_inline(struct btrfs_inode *inode,
707	struct folio *locked_folio,
708	u64 offset, u64 end,
709	size_t compressed_size,
710	int compress_type,
711	struct folio *compressed_folio,
712	bool update_i_size)
713	{
714	struct extent_state *cached = NULL;
715	unsigned long clear_flags = EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
716	EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING | EXTENT_LOCKED;
717	u64 size = min_t(u64, i_size_read(&inode->vfs_inode), end + 1);
718	int ret;
719
720	if (!can_cow_file_range_inline(inode, offset, size, compressed_size))
721	return 1;
722
723	btrfs_lock_extent(tree: &inode->io_tree, start: offset, end, cached: &cached);
724	ret = __cow_file_range_inline(inode, size, compressed_size,
725	compress_type, compressed_folio,
726	update_i_size);
727	if (ret > 0) {
728	btrfs_unlock_extent(tree: &inode->io_tree, start: offset, end, cached: &cached);
729	return ret;
730	}
731
732	/*
733	* In the successful case (ret == 0 here), cow_file_range will return 1.
734	*
735	* Quite a bit further up the callstack in extent_writepage(), ret == 1
736	* is treated as a short circuited success and does not unlock the folio,
737	* so we must do it here.
738	*
739	* In the failure case, the locked_folio does get unlocked by
740	* btrfs_folio_end_all_writers, which asserts that it is still locked
741	* at that point, so we must *not* unlock it here.
742	*
743	* The other two callsites in compress_file_range do not have a
744	* locked_folio, so they are not relevant to this logic.
745	*/
746	if (ret == 0)
747	locked_folio = NULL;
748
749	extent_clear_unlock_delalloc(inode, start: offset, end, locked_folio, cached: &cached,
750	bits_to_clear: clear_flags, page_ops: PAGE_UNLOCK |
751	PAGE_START_WRITEBACK | PAGE_END_WRITEBACK);
752	return ret;
753	}
754
755	struct async_extent {
756	u64 start;
757	u64 ram_size;
758	u64 compressed_size;
759	struct folio **folios;
760	unsigned long nr_folios;
761	int compress_type;
762	struct list_head list;
763	};
764
765	struct async_chunk {
766	struct btrfs_inode *inode;
767	struct folio *locked_folio;
768	u64 start;
769	u64 end;
770	blk_opf_t write_flags;
771	struct list_head extents;
772	struct cgroup_subsys_state *blkcg_css;
773	struct btrfs_work work;
774	struct async_cow *async_cow;
775	};
776
777	struct async_cow {
778	atomic_t num_chunks;
779	struct async_chunk chunks[];
780	};
781
782	static noinline int add_async_extent(struct async_chunk *cow,
783	u64 start, u64 ram_size,
784	u64 compressed_size,
785	struct folio **folios,
786	unsigned long nr_folios,
787	int compress_type)
788	{
789	struct async_extent *async_extent;
790
791	async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
792	if (!async_extent)
793	return -ENOMEM;
794	async_extent->start = start;
795	async_extent->ram_size = ram_size;
796	async_extent->compressed_size = compressed_size;
797	async_extent->folios = folios;
798	async_extent->nr_folios = nr_folios;
799	async_extent->compress_type = compress_type;
800	list_add_tail(new: &async_extent->list, head: &cow->extents);
801	return 0;
802	}
803
804	/*
805	* Check if the inode needs to be submitted to compression, based on mount
806	* options, defragmentation, properties or heuristics.
807	*/
808	static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
809	u64 end)
810	{
811	struct btrfs_fs_info *fs_info = inode->root->fs_info;
812
813	if (!btrfs_inode_can_compress(inode)) {
814	DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode));
815	return 0;
816	}
817
818	/* Defrag ioctl takes precedence over mount options and properties. */
819	if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS)
820	return 0;
821	if (BTRFS_COMPRESS_NONE < inode->defrag_compress &&
822	inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES)
823	return 1;
824	/* force compress */
825	if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
826	return 1;
827	/* bad compression ratios */
828	if (inode->flags & BTRFS_INODE_NOCOMPRESS)
829	return 0;
830	if (btrfs_test_opt(fs_info, COMPRESS) ||
831	inode->flags & BTRFS_INODE_COMPRESS ||
832	inode->prop_compress)
833	return btrfs_compress_heuristic(inode, start, end);
834	return 0;
835	}
836
837	static inline void inode_should_defrag(struct btrfs_inode *inode,
838	u64 start, u64 end, u64 num_bytes, u32 small_write)
839	{
840	/* If this is a small write inside eof, kick off a defrag */
841	if (num_bytes < small_write &&
842	(start > 0 || end + 1 < inode->disk_i_size))
843	btrfs_add_inode_defrag(inode, extent_thresh: small_write);
844	}
845
846	static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end)
847	{
848	const pgoff_t end_index = end >> PAGE_SHIFT;
849	struct folio *folio;
850	int ret = 0;
851
852	for (pgoff_t index = start >> PAGE_SHIFT; index <= end_index; index++) {
853	folio = filemap_get_folio(mapping: inode->vfs_inode.i_mapping, index);
854	if (IS_ERR(ptr: folio)) {
855	if (!ret)
856	ret = PTR_ERR(ptr: folio);
857	continue;
858	}
859	btrfs_folio_clamp_clear_dirty(fs_info: inode->root->fs_info, folio, start,
860	len: end + 1 - start);
861	folio_put(folio);
862	}
863	return ret;
864	}
865
866	/*
867	* Work queue call back to started compression on a file and pages.
868	*
869	* This is done inside an ordered work queue, and the compression is spread
870	* across many cpus. The actual IO submission is step two, and the ordered work
871	* queue takes care of making sure that happens in the same order things were
872	* put onto the queue by writepages and friends.
873	*
874	* If this code finds it can't get good compression, it puts an entry onto the
875	* work queue to write the uncompressed bytes. This makes sure that both
876	* compressed inodes and uncompressed inodes are written in the same order that
877	* the flusher thread sent them down.
878	*/
879	static void compress_file_range(struct btrfs_work *work)
880	{
881	struct async_chunk *async_chunk =
882	container_of(work, struct async_chunk, work);
883	struct btrfs_inode *inode = async_chunk->inode;
884	struct btrfs_fs_info *fs_info = inode->root->fs_info;
885	struct address_space *mapping = inode->vfs_inode.i_mapping;
886	const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order;
887	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
888	u64 blocksize = fs_info->sectorsize;
889	u64 start = async_chunk->start;
890	u64 end = async_chunk->end;
891	u64 actual_end;
892	u64 i_size;
893	int ret = 0;
894	struct folio **folios = NULL;
895	unsigned long nr_folios;
896	unsigned long total_compressed = 0;
897	unsigned long total_in = 0;
898	unsigned int loff;
899	int i;
900	int compress_type = fs_info->compress_type;
901	int compress_level = fs_info->compress_level;
902
903	if (unlikely(btrfs_is_shutdown(fs_info)))
904	goto cleanup_and_bail_uncompressed;
905
906	inode_should_defrag(inode, start, end, num_bytes: end - start + 1, SZ_16K);
907
908	/*
909	* We need to call clear_page_dirty_for_io on each page in the range.
910	* Otherwise applications with the file mmap'd can wander in and change
911	* the page contents while we are compressing them.
912	*/
913	ret = extent_range_clear_dirty_for_io(inode, start, end);
914
915	/*
916	* All the folios should have been locked thus no failure.
917	*
918	* And even if some folios are missing, btrfs_compress_folios()
919	* would handle them correctly, so here just do an ASSERT() check for
920	* early logic errors.
921	*/
922	ASSERT(ret == 0);
923
924	/*
925	* We need to save i_size before now because it could change in between
926	* us evaluating the size and assigning it. This is because we lock and
927	* unlock the page in truncate and fallocate, and then modify the i_size
928	* later on.
929	*
930	* The barriers are to emulate READ_ONCE, remove that once i_size_read
931	* does that for us.
932	*/
933	barrier();
934	i_size = i_size_read(inode: &inode->vfs_inode);
935	barrier();
936	actual_end = min_t(u64, i_size, end + 1);
937	again:
938	folios = NULL;
939	nr_folios = (end >> min_folio_shift) - (start >> min_folio_shift) + 1;
940	nr_folios = min_t(unsigned long, nr_folios, BTRFS_MAX_COMPRESSED >> min_folio_shift);
941
942	/*
943	* we don't want to send crud past the end of i_size through
944	* compression, that's just a waste of CPU time. So, if the
945	* end of the file is before the start of our current
946	* requested range of bytes, we bail out to the uncompressed
947	* cleanup code that can deal with all of this.
948	*
949	* It isn't really the fastest way to fix things, but this is a
950	* very uncommon corner.
951	*/
952	if (actual_end <= start)
953	goto cleanup_and_bail_uncompressed;
954
955	total_compressed = actual_end - start;
956
957	/*
958	* Skip compression for a small file range(<=blocksize) that
959	* isn't an inline extent, since it doesn't save disk space at all.
960	*/
961	if (total_compressed <= blocksize &&
962	(start > 0 || end + 1 < inode->disk_i_size))
963	goto cleanup_and_bail_uncompressed;
964
965	total_compressed = min_t(unsigned long, total_compressed,
966	BTRFS_MAX_UNCOMPRESSED);
967	total_in = 0;
968	ret = 0;
969
970	/*
971	* We do compression for mount -o compress and when the inode has not
972	* been flagged as NOCOMPRESS. This flag can change at any time if we
973	* discover bad compression ratios.
974	*/
975	if (!inode_need_compress(inode, start, end))
976	goto cleanup_and_bail_uncompressed;
977
978	folios = kcalloc(nr_folios, sizeof(struct folio *), GFP_NOFS);
979	if (!folios) {
980	/*
981	* Memory allocation failure is not a fatal error, we can fall
982	* back to uncompressed code.
983	*/
984	goto cleanup_and_bail_uncompressed;
985	}
986
987	if (0 < inode->defrag_compress && inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) {
988	compress_type = inode->defrag_compress;
989	compress_level = inode->defrag_compress_level;
990	} else if (inode->prop_compress) {
991	compress_type = inode->prop_compress;
992	}
993
994	/* Compression level is applied here. */
995	ret = btrfs_compress_folios(type: compress_type, level: compress_level,
996	inode, start, folios, out_folios: &nr_folios, total_in: &total_in,
997	total_out: &total_compressed);
998	if (ret)
999	goto mark_incompressible;
1000
1001	/*
1002	* Zero the tail end of the last folio, as we might be sending it down
1003	* to disk.
1004	*/
1005	loff = (total_compressed & (min_folio_size - 1));
1006	if (loff)
1007	folio_zero_range(folio: folios[nr_folios - 1], start: loff, length: min_folio_size - loff);
1008
1009	/*
1010	* Try to create an inline extent.
1011	*
1012	* If we didn't compress the entire range, try to create an uncompressed
1013	* inline extent, else a compressed one.
1014	*
1015	* Check cow_file_range() for why we don't even try to create inline
1016	* extent for the subpage case.
1017	*/
1018	if (total_in < actual_end)
1019	ret = cow_file_range_inline(inode, NULL, offset: start, end, compressed_size: 0,
1020	compress_type: BTRFS_COMPRESS_NONE, NULL, update_i_size: false);
1021	else
1022	ret = cow_file_range_inline(inode, NULL, offset: start, end, compressed_size: total_compressed,
1023	compress_type, compressed_folio: folios[0], update_i_size: false);
1024	if (ret <= 0) {
1025	if (ret < 0)
1026	mapping_set_error(mapping, error: -EIO);
1027	goto free_pages;
1028	}
1029
1030	/*
1031	* We aren't doing an inline extent. Round the compressed size up to a
1032	* block size boundary so the allocator does sane things.
1033	*/
1034	total_compressed = ALIGN(total_compressed, blocksize);
1035
1036	/*
1037	* One last check to make sure the compression is really a win, compare
1038	* the page count read with the blocks on disk, compression must free at
1039	* least one sector.
1040	*/
1041	total_in = round_up(total_in, fs_info->sectorsize);
1042	if (total_compressed + blocksize > total_in)
1043	goto mark_incompressible;
1044
1045	/*
1046	* The async work queues will take care of doing actual allocation on
1047	* disk for these compressed pages, and will submit the bios.
1048	*/
1049	ret = add_async_extent(cow: async_chunk, start, ram_size: total_in, compressed_size: total_compressed, folios,
1050	nr_folios, compress_type);
1051	BUG_ON(ret);
1052	if (start + total_in < end) {
1053	start += total_in;
1054	cond_resched();
1055	goto again;
1056	}
1057	return;
1058
1059	mark_incompressible:
1060	if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress)
1061	inode->flags |= BTRFS_INODE_NOCOMPRESS;
1062	cleanup_and_bail_uncompressed:
1063	ret = add_async_extent(cow: async_chunk, start, ram_size: end - start + 1, compressed_size: 0, NULL, nr_folios: 0,
1064	compress_type: BTRFS_COMPRESS_NONE);
1065	BUG_ON(ret);
1066	free_pages:
1067	if (folios) {
1068	for (i = 0; i < nr_folios; i++) {
1069	WARN_ON(folios[i]->mapping);
1070	btrfs_free_compr_folio(folio: folios[i]);
1071	}
1072	kfree(objp: folios);
1073	}
1074	}
1075
1076	static void free_async_extent_pages(struct async_extent *async_extent)
1077	{
1078	int i;
1079
1080	if (!async_extent->folios)
1081	return;
1082
1083	for (i = 0; i < async_extent->nr_folios; i++) {
1084	WARN_ON(async_extent->folios[i]->mapping);
1085	btrfs_free_compr_folio(folio: async_extent->folios[i]);
1086	}
1087	kfree(objp: async_extent->folios);
1088	async_extent->nr_folios = 0;
1089	async_extent->folios = NULL;
1090	}
1091
1092	static void submit_uncompressed_range(struct btrfs_inode *inode,
1093	struct async_extent *async_extent,
1094	struct folio *locked_folio)
1095	{
1096	u64 start = async_extent->start;
1097	u64 end = async_extent->start + async_extent->ram_size - 1;
1098	int ret;
1099	struct writeback_control wbc = {
1100	.sync_mode = WB_SYNC_ALL,
1101	.range_start = start,
1102	.range_end = end,
1103	.no_cgroup_owner = 1,
1104	};
1105
1106	wbc_attach_fdatawrite_inode(wbc: &wbc, inode: &inode->vfs_inode);
1107	ret = run_delalloc_cow(inode, locked_folio, start, end,
1108	wbc: &wbc, pages_dirty: false);
1109	wbc_detach_inode(wbc: &wbc);
1110	if (ret < 0) {
1111	if (locked_folio)
1112	btrfs_folio_end_lock(fs_info: inode->root->fs_info, folio: locked_folio,
1113	start, len: async_extent->ram_size);
1114	btrfs_err_rl(inode->root->fs_info,
1115	"%s failed, root=%llu inode=%llu start=%llu len=%llu: %d",
1116	__func__, btrfs_root_id(inode->root),
1117	btrfs_ino(inode), start, async_extent->ram_size, ret);
1118	}
1119	}
1120
1121	static void submit_one_async_extent(struct async_chunk *async_chunk,
1122	struct async_extent *async_extent,
1123	u64 *alloc_hint)
1124	{
1125	struct btrfs_inode *inode = async_chunk->inode;
1126	struct extent_io_tree *io_tree = &inode->io_tree;
1127	struct btrfs_root *root = inode->root;
1128	struct btrfs_fs_info *fs_info = root->fs_info;
1129	struct btrfs_ordered_extent *ordered;
1130	struct btrfs_file_extent file_extent;
1131	struct btrfs_key ins;
1132	struct folio *locked_folio = NULL;
1133	struct extent_state *cached = NULL;
1134	struct extent_map *em;
1135	int ret = 0;
1136	bool free_pages = false;
1137	u64 start = async_extent->start;
1138	u64 end = async_extent->start + async_extent->ram_size - 1;
1139
1140	if (async_chunk->blkcg_css)
1141	kthread_associate_blkcg(css: async_chunk->blkcg_css);
1142
1143	/*
1144	* If async_chunk->locked_folio is in the async_extent range, we need to
1145	* handle it.
1146	*/
1147	if (async_chunk->locked_folio) {
1148	u64 locked_folio_start = folio_pos(folio: async_chunk->locked_folio);
1149	u64 locked_folio_end = locked_folio_start +
1150	folio_size(folio: async_chunk->locked_folio) - 1;
1151
1152	if (!(start >= locked_folio_end || end <= locked_folio_start))
1153	locked_folio = async_chunk->locked_folio;
1154	}
1155
1156	if (async_extent->compress_type == BTRFS_COMPRESS_NONE) {
1157	ASSERT(!async_extent->folios);
1158	ASSERT(async_extent->nr_folios == 0);
1159	submit_uncompressed_range(inode, async_extent, locked_folio);
1160	free_pages = true;
1161	goto done;
1162	}
1163
1164	ret = btrfs_reserve_extent(root, ram_bytes: async_extent->ram_size,
1165	num_bytes: async_extent->compressed_size,
1166	min_alloc_size: async_extent->compressed_size,
1167	empty_size: 0, hint_byte: *alloc_hint, ins: &ins, is_data: true, delalloc: true);
1168	if (ret) {
1169	/*
1170	* We can't reserve contiguous space for the compressed size.
1171	* Unlikely, but it's possible that we could have enough
1172	* non-contiguous space for the uncompressed size instead. So
1173	* fall back to uncompressed.
1174	*/
1175	submit_uncompressed_range(inode, async_extent, locked_folio);
1176	free_pages = true;
1177	goto done;
1178	}
1179
1180	btrfs_lock_extent(tree: io_tree, start, end, cached: &cached);
1181
1182	/* Here we're doing allocation and writeback of the compressed pages */
1183	file_extent.disk_bytenr = ins.objectid;
1184	file_extent.disk_num_bytes = ins.offset;
1185	file_extent.ram_bytes = async_extent->ram_size;
1186	file_extent.num_bytes = async_extent->ram_size;
1187	file_extent.offset = 0;
1188	file_extent.compression = async_extent->compress_type;
1189
1190	em = btrfs_create_io_em(inode, start, file_extent: &file_extent, type: BTRFS_ORDERED_COMPRESSED);
1191	if (IS_ERR(ptr: em)) {
1192	ret = PTR_ERR(ptr: em);
1193	goto out_free_reserve;
1194	}
1195	btrfs_free_extent_map(em);
1196
1197	ordered = btrfs_alloc_ordered_extent(inode, file_offset: start, file_extent: &file_extent,
1198	flags: 1U << BTRFS_ORDERED_COMPRESSED);
1199	if (IS_ERR(ptr: ordered)) {
1200	btrfs_drop_extent_map_range(inode, start, end, skip_pinned: false);
1201	ret = PTR_ERR(ptr: ordered);
1202	goto out_free_reserve;
1203	}
1204	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
1205
1206	/* Clear dirty, set writeback and unlock the pages. */
1207	extent_clear_unlock_delalloc(inode, start, end,
1208	NULL, cached: &cached, bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC,
1209	page_ops: PAGE_UNLOCK | PAGE_START_WRITEBACK);
1210	btrfs_submit_compressed_write(ordered,
1211	compressed_folios: async_extent->folios, /* compressed_folios */
1212	nr_folios: async_extent->nr_folios,
1213	write_flags: async_chunk->write_flags, writeback: true);
1214	*alloc_hint = ins.objectid + ins.offset;
1215	done:
1216	if (async_chunk->blkcg_css)
1217	kthread_associate_blkcg(NULL);
1218	if (free_pages)
1219	free_async_extent_pages(async_extent);
1220	kfree(objp: async_extent);
1221	return;
1222
1223	out_free_reserve:
1224	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
1225	btrfs_free_reserved_extent(fs_info, start: ins.objectid, len: ins.offset, is_delalloc: true);
1226	mapping_set_error(mapping: inode->vfs_inode.i_mapping, error: -EIO);
1227	extent_clear_unlock_delalloc(inode, start, end,
1228	NULL, cached: &cached,
1229	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC |
1230	EXTENT_DELALLOC_NEW |
1231	EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
1232	page_ops: PAGE_UNLOCK | PAGE_START_WRITEBACK |
1233	PAGE_END_WRITEBACK);
1234	free_async_extent_pages(async_extent);
1235	if (async_chunk->blkcg_css)
1236	kthread_associate_blkcg(NULL);
1237	btrfs_debug(fs_info,
1238	"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d",
1239	btrfs_root_id(root), btrfs_ino(inode), start,
1240	async_extent->ram_size, ret);
1241	kfree(objp: async_extent);
1242	}
1243
1244	u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
1245	u64 num_bytes)
1246	{
1247	struct extent_map_tree *em_tree = &inode->extent_tree;
1248	struct extent_map *em;
1249	u64 alloc_hint = 0;
1250
1251	read_lock(&em_tree->lock);
1252	em = btrfs_search_extent_mapping(tree: em_tree, start, len: num_bytes);
1253	if (em) {
1254	/*
1255	* if block start isn't an actual block number then find the
1256	* first block in this inode and use that as a hint. If that
1257	* block is also bogus then just don't worry about it.
1258	*/
1259	if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) {
1260	btrfs_free_extent_map(em);
1261	em = btrfs_search_extent_mapping(tree: em_tree, start: 0, len: 0);
1262	if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
1263	alloc_hint = btrfs_extent_map_block_start(em);
1264	if (em)
1265	btrfs_free_extent_map(em);
1266	} else {
1267	alloc_hint = btrfs_extent_map_block_start(em);
1268	btrfs_free_extent_map(em);
1269	}
1270	}
1271	read_unlock(&em_tree->lock);
1272
1273	return alloc_hint;
1274	}
1275
1276	/*
1277	* when extent_io.c finds a delayed allocation range in the file,
1278	* the call backs end up in this code. The basic idea is to
1279	* allocate extents on disk for the range, and create ordered data structs
1280	* in ram to track those extents.
1281	*
1282	* locked_folio is the folio that writepage had locked already. We use
1283	* it to make sure we don't do extra locks or unlocks.
1284	*
1285	* When this function fails, it unlocks all folios except @locked_folio.
1286	*
1287	* When this function successfully creates an inline extent, it returns 1 and
1288	* unlocks all folios including locked_folio and starts I/O on them.
1289	* (In reality inline extents are limited to a single block, so locked_folio is
1290	* the only folio handled anyway).
1291	*
1292	* When this function succeed and creates a normal extent, the folio locking
1293	* status depends on the passed in flags:
1294	*
1295	* - If COW_FILE_RANGE_KEEP_LOCKED flag is set, all folios are kept locked.
1296	* - Else all folios except for @locked_folio are unlocked.
1297	*
1298	* When a failure happens in the second or later iteration of the
1299	* while-loop, the ordered extents created in previous iterations are cleaned up.
1300	*/
1301	static noinline int cow_file_range(struct btrfs_inode *inode,
1302	struct folio *locked_folio, u64 start,
1303	u64 end, u64 *done_offset,
1304	unsigned long flags)
1305	{
1306	struct btrfs_root *root = inode->root;
1307	struct btrfs_fs_info *fs_info = root->fs_info;
1308	struct extent_state *cached = NULL;
1309	u64 alloc_hint = 0;
1310	u64 orig_start = start;
1311	u64 num_bytes;
1312	u64 cur_alloc_size = 0;
1313	u64 min_alloc_size;
1314	u64 blocksize = fs_info->sectorsize;
1315	struct btrfs_key ins;
1316	struct extent_map *em;
1317	unsigned clear_bits;
1318	unsigned long page_ops;
1319	int ret = 0;
1320
1321	if (unlikely(btrfs_is_shutdown(fs_info))) {
1322	ret = -EIO;
1323	goto out_unlock;
1324	}
1325
1326	if (btrfs_is_free_space_inode(inode)) {
1327	ret = -EINVAL;
1328	goto out_unlock;
1329	}
1330
1331	num_bytes = ALIGN(end - start + 1, blocksize);
1332	num_bytes = max(blocksize, num_bytes);
1333	ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
1334
1335	inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
1336
1337	if (!(flags & COW_FILE_RANGE_NO_INLINE)) {
1338	/* lets try to make an inline extent */
1339	ret = cow_file_range_inline(inode, locked_folio, offset: start, end, compressed_size: 0,
1340	compress_type: BTRFS_COMPRESS_NONE, NULL, update_i_size: false);
1341	if (ret <= 0) {
1342	/*
1343	* We succeeded, return 1 so the caller knows we're done
1344	* with this page and already handled the IO.
1345	*
1346	* If there was an error then cow_file_range_inline() has
1347	* already done the cleanup.
1348	*/
1349	if (ret == 0)
1350	ret = 1;
1351	goto done;
1352	}
1353	}
1354
1355	alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes);
1356
1357	/*
1358	* We're not doing compressed IO, don't unlock the first page (which
1359	* the caller expects to stay locked), don't clear any dirty bits and
1360	* don't set any writeback bits.
1361	*
1362	* Do set the Ordered (Private2) bit so we know this page was properly
1363	* setup for writepage.
1364	*/
1365	page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK);
1366	page_ops |= PAGE_SET_ORDERED;
1367
1368	/*
1369	* Relocation relies on the relocated extents to have exactly the same
1370	* size as the original extents. Normally writeback for relocation data
1371	* extents follows a NOCOW path because relocation preallocates the
1372	* extents. However, due to an operation such as scrub turning a block
1373	* group to RO mode, it may fallback to COW mode, so we must make sure
1374	* an extent allocated during COW has exactly the requested size and can
1375	* not be split into smaller extents, otherwise relocation breaks and
1376	* fails during the stage where it updates the bytenr of file extent
1377	* items.
1378	*/
1379	if (btrfs_is_data_reloc_root(root))
1380	min_alloc_size = num_bytes;
1381	else
1382	min_alloc_size = fs_info->sectorsize;
1383
1384	while (num_bytes > 0) {
1385	struct btrfs_ordered_extent *ordered;
1386	struct btrfs_file_extent file_extent;
1387
1388	ret = btrfs_reserve_extent(root, ram_bytes: num_bytes, num_bytes,
1389	min_alloc_size, empty_size: 0, hint_byte: alloc_hint,
1390	ins: &ins, is_data: true, delalloc: true);
1391	if (ret == -EAGAIN) {
1392	/*
1393	* btrfs_reserve_extent only returns -EAGAIN for zoned
1394	* file systems, which is an indication that there are
1395	* no active zones to allocate from at the moment.
1396	*
1397	* If this is the first loop iteration, wait for at
1398	* least one zone to finish before retrying the
1399	* allocation. Otherwise ask the caller to write out
1400	* the already allocated blocks before coming back to
1401	* us, or return -ENOSPC if it can't handle retries.
1402	*/
1403	ASSERT(btrfs_is_zoned(fs_info));
1404	if (start == orig_start) {
1405	wait_on_bit_io(word: &inode->root->fs_info->flags,
1406	bit: BTRFS_FS_NEED_ZONE_FINISH,
1407	TASK_UNINTERRUPTIBLE);
1408	continue;
1409	}
1410	if (done_offset) {
1411	/*
1412	* Move @end to the end of the processed range,
1413	* and exit the loop to unlock the processed extents.
1414	*/
1415	end = start - 1;
1416	ret = 0;
1417	break;
1418	}
1419	ret = -ENOSPC;
1420	}
1421	if (ret < 0)
1422	goto out_unlock;
1423	cur_alloc_size = ins.offset;
1424
1425	file_extent.disk_bytenr = ins.objectid;
1426	file_extent.disk_num_bytes = ins.offset;
1427	file_extent.num_bytes = ins.offset;
1428	file_extent.ram_bytes = ins.offset;
1429	file_extent.offset = 0;
1430	file_extent.compression = BTRFS_COMPRESS_NONE;
1431
1432	/*
1433	* Locked range will be released either during error clean up or
1434	* after the whole range is finished.
1435	*/
1436	btrfs_lock_extent(tree: &inode->io_tree, start, end: start + cur_alloc_size - 1,
1437	cached: &cached);
1438
1439	em = btrfs_create_io_em(inode, start, file_extent: &file_extent,
1440	type: BTRFS_ORDERED_REGULAR);
1441	if (IS_ERR(ptr: em)) {
1442	btrfs_unlock_extent(tree: &inode->io_tree, start,
1443	end: start + cur_alloc_size - 1, cached: &cached);
1444	ret = PTR_ERR(ptr: em);
1445	goto out_reserve;
1446	}
1447	btrfs_free_extent_map(em);
1448
1449	ordered = btrfs_alloc_ordered_extent(inode, file_offset: start, file_extent: &file_extent,
1450	flags: 1U << BTRFS_ORDERED_REGULAR);
1451	if (IS_ERR(ptr: ordered)) {
1452	btrfs_unlock_extent(tree: &inode->io_tree, start,
1453	end: start + cur_alloc_size - 1, cached: &cached);
1454	ret = PTR_ERR(ptr: ordered);
1455	goto out_drop_extent_cache;
1456	}
1457
1458	if (btrfs_is_data_reloc_root(root)) {
1459	ret = btrfs_reloc_clone_csums(ordered);
1460
1461	/*
1462	* Only drop cache here, and process as normal.
1463	*
1464	* We must not allow extent_clear_unlock_delalloc()
1465	* at out_unlock label to free meta of this ordered
1466	* extent, as its meta should be freed by
1467	* btrfs_finish_ordered_io().
1468	*
1469	* So we must continue until @start is increased to
1470	* skip current ordered extent.
1471	*/
1472	if (ret)
1473	btrfs_drop_extent_map_range(inode, start,
1474	end: start + cur_alloc_size - 1,
1475	skip_pinned: false);
1476	}
1477	btrfs_put_ordered_extent(entry: ordered);
1478
1479	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
1480
1481	if (num_bytes < cur_alloc_size)
1482	num_bytes = 0;
1483	else
1484	num_bytes -= cur_alloc_size;
1485	alloc_hint = ins.objectid + ins.offset;
1486	start += cur_alloc_size;
1487	cur_alloc_size = 0;
1488
1489	/*
1490	* btrfs_reloc_clone_csums() error, since start is increased
1491	* extent_clear_unlock_delalloc() at out_unlock label won't
1492	* free metadata of current ordered extent, we're OK to exit.
1493	*/
1494	if (ret)
1495	goto out_unlock;
1496	}
1497	extent_clear_unlock_delalloc(inode, start: orig_start, end, locked_folio, cached: &cached,
1498	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC, page_ops);
1499	done:
1500	if (done_offset)
1501	*done_offset = end;
1502	return ret;
1503
1504	out_drop_extent_cache:
1505	btrfs_drop_extent_map_range(inode, start, end: start + cur_alloc_size - 1, skip_pinned: false);
1506	out_reserve:
1507	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
1508	btrfs_free_reserved_extent(fs_info, start: ins.objectid, len: ins.offset, is_delalloc: true);
1509	out_unlock:
1510	/*
1511	* Now, we have three regions to clean up:
1512	*
1513	* |-------(1)----|---(2)---|-------------(3)----------|
1514	* `- orig_start `- start `- start + cur_alloc_size `- end
1515	*
1516	* We process each region below.
1517	*/
1518
1519	/*
1520	* For the range (1). We have already instantiated the ordered extents
1521	* for this region, thus we need to cleanup those ordered extents.
1522	* EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV
1523	* are also handled by the ordered extents cleanup.
1524	*
1525	* So here we only clear EXTENT_LOCKED and EXTENT_DELALLOC flag, and
1526	* finish the writeback of the involved folios, which will be never submitted.
1527	*/
1528	if (orig_start < start) {
1529	clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC;
1530	page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK;
1531
1532	if (!locked_folio)
1533	mapping_set_error(mapping: inode->vfs_inode.i_mapping, error: ret);
1534
1535	btrfs_cleanup_ordered_extents(inode, offset: orig_start, bytes: start - orig_start);
1536	extent_clear_unlock_delalloc(inode, start: orig_start, end: start - 1,
1537	locked_folio, NULL, bits_to_clear: clear_bits, page_ops);
1538	}
1539
1540	clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1541	EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1542	page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK;
1543
1544	/*
1545	* For the range (2). If we reserved an extent for our delalloc range
1546	* (or a subrange) and failed to create the respective ordered extent,
1547	* then it means that when we reserved the extent we decremented the
1548	* extent's size from the data space_info's bytes_may_use counter and
1549	* incremented the space_info's bytes_reserved counter by the same
1550	* amount. We must make sure extent_clear_unlock_delalloc() does not try
1551	* to decrement again the data space_info's bytes_may_use counter,
1552	* therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV.
1553	*/
1554	if (cur_alloc_size) {
1555	extent_clear_unlock_delalloc(inode, start,
1556	end: start + cur_alloc_size - 1,
1557	locked_folio, cached: &cached, bits_to_clear: clear_bits,
1558	page_ops);
1559	btrfs_qgroup_free_data(inode, NULL, start, len: cur_alloc_size, NULL);
1560	}
1561
1562	/*
1563	* For the range (3). We never touched the region. In addition to the
1564	* clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data
1565	* space_info's bytes_may_use counter, reserved in
1566	* btrfs_check_data_free_space().
1567	*/
1568	if (start + cur_alloc_size < end) {
1569	clear_bits |= EXTENT_CLEAR_DATA_RESV;
1570	extent_clear_unlock_delalloc(inode, start: start + cur_alloc_size,
1571	end, locked_folio,
1572	cached: &cached, bits_to_clear: clear_bits, page_ops);
1573	btrfs_qgroup_free_data(inode, NULL, start: start + cur_alloc_size,
1574	len: end - start - cur_alloc_size + 1, NULL);
1575	}
1576	btrfs_err(fs_info,
1577	"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%llu: %d",
1578	__func__, btrfs_root_id(inode->root),
1579	btrfs_ino(inode), orig_start, end + 1 - orig_start,
1580	start, cur_alloc_size, ret);
1581	return ret;
1582	}
1583
1584	/*
1585	* Phase two of compressed writeback. This is the ordered portion of the code,
1586	* which only gets called in the order the work was queued. We walk all the
1587	* async extents created by compress_file_range and send them down to the disk.
1588	*
1589	* If called with @do_free == true then it'll try to finish the work and free
1590	* the work struct eventually.
1591	*/
1592	static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free)
1593	{
1594	struct async_chunk *async_chunk = container_of(work, struct async_chunk,
1595	work);
1596	struct btrfs_fs_info *fs_info = btrfs_work_owner(work);
1597	struct async_extent *async_extent;
1598	unsigned long nr_pages;
1599	u64 alloc_hint = 0;
1600
1601	if (do_free) {
1602	struct async_cow *async_cow;
1603
1604	btrfs_add_delayed_iput(inode: async_chunk->inode);
1605	if (async_chunk->blkcg_css)
1606	css_put(css: async_chunk->blkcg_css);
1607
1608	async_cow = async_chunk->async_cow;
1609	if (atomic_dec_and_test(v: &async_cow->num_chunks))
1610	kvfree(addr: async_cow);
1611	return;
1612	}
1613
1614	nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>
1615	PAGE_SHIFT;
1616
1617	while (!list_empty(head: &async_chunk->extents)) {
1618	async_extent = list_first_entry(&async_chunk->extents,
1619	struct async_extent, list);
1620	list_del(entry: &async_extent->list);
1621	submit_one_async_extent(async_chunk, async_extent, alloc_hint: &alloc_hint);
1622	}
1623
1624	/* atomic_sub_return implies a barrier */
1625	if (atomic_sub_return(i: nr_pages, v: &fs_info->async_delalloc_pages) <
1626	5 * SZ_1M)
1627	cond_wake_up_nomb(wq: &fs_info->async_submit_wait);
1628	}
1629
1630	static bool run_delalloc_compressed(struct btrfs_inode *inode,
1631	struct folio *locked_folio, u64 start,
1632	u64 end, struct writeback_control *wbc)
1633	{
1634	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1635	struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc);
1636	struct async_cow *ctx;
1637	struct async_chunk *async_chunk;
1638	unsigned long nr_pages;
1639	u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);
1640	int i;
1641	unsigned nofs_flag;
1642	const blk_opf_t write_flags = wbc_to_write_flags(wbc);
1643
1644	nofs_flag = memalloc_nofs_save();
1645	ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL);
1646	memalloc_nofs_restore(flags: nofs_flag);
1647	if (!ctx)
1648	return false;
1649
1650	set_bit(nr: BTRFS_INODE_HAS_ASYNC_EXTENT, addr: &inode->runtime_flags);
1651
1652	async_chunk = ctx->chunks;
1653	atomic_set(v: &ctx->num_chunks, i: num_chunks);
1654
1655	for (i = 0; i < num_chunks; i++) {
1656	u64 cur_end = min(end, start + SZ_512K - 1);
1657
1658	/*
1659	* igrab is called higher up in the call chain, take only the
1660	* lightweight reference for the callback lifetime
1661	*/
1662	ihold(inode: &inode->vfs_inode);
1663	async_chunk[i].async_cow = ctx;
1664	async_chunk[i].inode = inode;
1665	async_chunk[i].start = start;
1666	async_chunk[i].end = cur_end;
1667	async_chunk[i].write_flags = write_flags;
1668	INIT_LIST_HEAD(list: &async_chunk[i].extents);
1669
1670	/*
1671	* The locked_folio comes all the way from writepage and its
1672	* the original folio we were actually given. As we spread
1673	* this large delalloc region across multiple async_chunk
1674	* structs, only the first struct needs a pointer to
1675	* locked_folio.
1676	*
1677	* This way we don't need racey decisions about who is supposed
1678	* to unlock it.
1679	*/
1680	if (locked_folio) {
1681	/*
1682	* Depending on the compressibility, the pages might or
1683	* might not go through async. We want all of them to
1684	* be accounted against wbc once. Let's do it here
1685	* before the paths diverge. wbc accounting is used
1686	* only for foreign writeback detection and doesn't
1687	* need full accuracy. Just account the whole thing
1688	* against the first page.
1689	*/
1690	wbc_account_cgroup_owner(wbc, folio: locked_folio,
1691	bytes: cur_end - start);
1692	async_chunk[i].locked_folio = locked_folio;
1693	locked_folio = NULL;
1694	} else {
1695	async_chunk[i].locked_folio = NULL;
1696	}
1697
1698	if (blkcg_css != blkcg_root_css) {
1699	css_get(css: blkcg_css);
1700	async_chunk[i].blkcg_css = blkcg_css;
1701	async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT;
1702	} else {
1703	async_chunk[i].blkcg_css = NULL;
1704	}
1705
1706	btrfs_init_work(work: &async_chunk[i].work, func: compress_file_range,
1707	ordered_func: submit_compressed_extents);
1708
1709	nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);
1710	atomic_add(i: nr_pages, v: &fs_info->async_delalloc_pages);
1711
1712	btrfs_queue_work(wq: fs_info->delalloc_workers, work: &async_chunk[i].work);
1713
1714	start = cur_end + 1;
1715	}
1716	return true;
1717	}
1718
1719	/*
1720	* Run the delalloc range from start to end, and write back any dirty pages
1721	* covered by the range.
1722	*/
1723	static noinline int run_delalloc_cow(struct btrfs_inode *inode,
1724	struct folio *locked_folio, u64 start,
1725	u64 end, struct writeback_control *wbc,
1726	bool pages_dirty)
1727	{
1728	u64 done_offset = end;
1729	int ret;
1730
1731	while (start <= end) {
1732	ret = cow_file_range(inode, locked_folio, start, end,
1733	done_offset: &done_offset, COW_FILE_RANGE_KEEP_LOCKED);
1734	if (ret)
1735	return ret;
1736	extent_write_locked_range(inode: &inode->vfs_inode, locked_folio,
1737	start, end: done_offset, wbc, pages_dirty);
1738	start = done_offset + 1;
1739	}
1740
1741	return 1;
1742	}
1743
1744	static int fallback_to_cow(struct btrfs_inode *inode,
1745	struct folio *locked_folio, const u64 start,
1746	const u64 end)
1747	{
1748	const bool is_space_ino = btrfs_is_free_space_inode(inode);
1749	const bool is_reloc_ino = btrfs_is_data_reloc_root(root: inode->root);
1750	const u64 range_bytes = end + 1 - start;
1751	struct extent_io_tree *io_tree = &inode->io_tree;
1752	struct extent_state *cached_state = NULL;
1753	u64 range_start = start;
1754	u64 count;
1755	int ret;
1756
1757	/*
1758	* If EXTENT_NORESERVE is set it means that when the buffered write was
1759	* made we had not enough available data space and therefore we did not
1760	* reserve data space for it, since we though we could do NOCOW for the
1761	* respective file range (either there is prealloc extent or the inode
1762	* has the NOCOW bit set).
1763	*
1764	* However when we need to fallback to COW mode (because for example the
1765	* block group for the corresponding extent was turned to RO mode by a
1766	* scrub or relocation) we need to do the following:
1767	*
1768	* 1) We increment the bytes_may_use counter of the data space info.
1769	* If COW succeeds, it allocates a new data extent and after doing
1770	* that it decrements the space info's bytes_may_use counter and
1771	* increments its bytes_reserved counter by the same amount (we do
1772	* this at btrfs_add_reserved_bytes()). So we need to increment the
1773	* bytes_may_use counter to compensate (when space is reserved at
1774	* buffered write time, the bytes_may_use counter is incremented);
1775	*
1776	* 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1777	* that if the COW path fails for any reason, it decrements (through
1778	* extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1779	* data space info, which we incremented in the step above.
1780	*
1781	* If we need to fallback to cow and the inode corresponds to a free
1782	* space cache inode or an inode of the data relocation tree, we must
1783	* also increment bytes_may_use of the data space_info for the same
1784	* reason. Space caches and relocated data extents always get a prealloc
1785	* extent for them, however scrub or balance may have set the block
1786	* group that contains that extent to RO mode and therefore force COW
1787	* when starting writeback.
1788	*/
1789	btrfs_lock_extent(tree: io_tree, start, end, cached: &cached_state);
1790	count = btrfs_count_range_bits(tree: io_tree, start: &range_start, search_end: end, max_bytes: range_bytes,
1791	bits: EXTENT_NORESERVE, contig: 0, NULL);
1792	if (count > 0 || is_space_ino || is_reloc_ino) {
1793	u64 bytes = count;
1794	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1795	struct btrfs_space_info *sinfo = fs_info->data_sinfo;
1796
1797	if (is_space_ino || is_reloc_ino)
1798	bytes = range_bytes;
1799
1800	spin_lock(lock: &sinfo->lock);
1801	btrfs_space_info_update_bytes_may_use(sinfo, bytes);
1802	spin_unlock(lock: &sinfo->lock);
1803
1804	if (count > 0)
1805	btrfs_clear_extent_bit(tree: io_tree, start, end, bits: EXTENT_NORESERVE,
1806	cached: &cached_state);
1807	}
1808	btrfs_unlock_extent(tree: io_tree, start, end, cached: &cached_state);
1809
1810	/*
1811	* Don't try to create inline extents, as a mix of inline extent that
1812	* is written out and unlocked directly and a normal NOCOW extent
1813	* doesn't work.
1814	*
1815	* And here we do not unlock the folio after a successful run.
1816	* The folios will be unlocked after everything is finished, or by error handling.
1817	*
1818	* This is to ensure error handling won't need to clear dirty/ordered flags without
1819	* a locked folio, which can race with writeback.
1820	*/
1821	ret = cow_file_range(inode, locked_folio, start, end, NULL,
1822	COW_FILE_RANGE_NO_INLINE | COW_FILE_RANGE_KEEP_LOCKED);
1823	ASSERT(ret != 1);
1824	return ret;
1825	}
1826
1827	struct can_nocow_file_extent_args {
1828	/* Input fields. */
1829
1830	/* Start file offset of the range we want to NOCOW. */
1831	u64 start;
1832	/* End file offset (inclusive) of the range we want to NOCOW. */
1833	u64 end;
1834	bool writeback_path;
1835	/*
1836	* Free the path passed to can_nocow_file_extent() once it's not needed
1837	* anymore.
1838	*/
1839	bool free_path;
1840
1841	/*
1842	* Output fields. Only set when can_nocow_file_extent() returns 1.
1843	* The expected file extent for the NOCOW write.
1844	*/
1845	struct btrfs_file_extent file_extent;
1846	};
1847
1848	/*
1849	* Check if we can NOCOW the file extent that the path points to.
1850	* This function may return with the path released, so the caller should check
1851	* if path->nodes[0] is NULL or not if it needs to use the path afterwards.
1852	*
1853	* Returns: < 0 on error
1854	* 0 if we can not NOCOW
1855	* 1 if we can NOCOW
1856	*/
1857	static int can_nocow_file_extent(struct btrfs_path *path,
1858	struct btrfs_key *key,
1859	struct btrfs_inode *inode,
1860	struct can_nocow_file_extent_args *args)
1861	{
1862	const bool is_freespace_inode = btrfs_is_free_space_inode(inode);
1863	struct extent_buffer *leaf = path->nodes[0];
1864	struct btrfs_root *root = inode->root;
1865	struct btrfs_file_extent_item *fi;
1866	struct btrfs_root *csum_root;
1867	u64 io_start;
1868	u64 extent_end;
1869	u8 extent_type;
1870	int can_nocow = 0;
1871	int ret = 0;
1872	bool nowait = path->nowait;
1873
1874	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
1875	extent_type = btrfs_file_extent_type(eb: leaf, s: fi);
1876
1877	if (extent_type == BTRFS_FILE_EXTENT_INLINE)
1878	goto out;
1879
1880	if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
1881	extent_type == BTRFS_FILE_EXTENT_REG)
1882	goto out;
1883
1884	/*
1885	* If the extent was created before the generation where the last snapshot
1886	* for its subvolume was created, then this implies the extent is shared,
1887	* hence we must COW.
1888	*/
1889	if (btrfs_file_extent_generation(eb: leaf, s: fi) <=
1890	btrfs_root_last_snapshot(s: &root->root_item))
1891	goto out;
1892
1893	/* An explicit hole, must COW. */
1894	if (btrfs_file_extent_disk_bytenr(eb: leaf, s: fi) == 0)
1895	goto out;
1896
1897	/* Compressed/encrypted/encoded extents must be COWed. */
1898	if (btrfs_file_extent_compression(eb: leaf, s: fi) ||
1899	btrfs_file_extent_encryption(eb: leaf, s: fi) ||
1900	btrfs_file_extent_other_encoding(eb: leaf, s: fi))
1901	goto out;
1902
1903	extent_end = btrfs_file_extent_end(path);
1904
1905	args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: fi);
1906	args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(eb: leaf, s: fi);
1907	args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(eb: leaf, s: fi);
1908	args->file_extent.offset = btrfs_file_extent_offset(eb: leaf, s: fi);
1909	args->file_extent.compression = btrfs_file_extent_compression(eb: leaf, s: fi);
1910
1911	/*
1912	* The following checks can be expensive, as they need to take other
1913	* locks and do btree or rbtree searches, so release the path to avoid
1914	* blocking other tasks for too long.
1915	*/
1916	btrfs_release_path(p: path);
1917
1918	ret = btrfs_cross_ref_exist(inode, offset: key->offset - args->file_extent.offset,
1919	bytenr: args->file_extent.disk_bytenr, path);
1920	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1921	if (ret != 0)
1922	goto out;
1923
1924	if (args->free_path) {
1925	/*
1926	* We don't need the path anymore, plus through the
1927	* btrfs_lookup_csums_list() call below we will end up allocating
1928	* another path. So free the path to avoid unnecessary extra
1929	* memory usage.
1930	*/
1931	btrfs_free_path(p: path);
1932	path = NULL;
1933	}
1934
1935	/* If there are pending snapshots for this root, we must COW. */
1936	if (args->writeback_path && !is_freespace_inode &&
1937	atomic_read(v: &root->snapshot_force_cow))
1938	goto out;
1939
1940	args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start;
1941	args->file_extent.offset += args->start - key->offset;
1942	io_start = args->file_extent.disk_bytenr + args->file_extent.offset;
1943
1944	/*
1945	* Force COW if csums exist in the range. This ensures that csums for a
1946	* given extent are either valid or do not exist.
1947	*/
1948
1949	csum_root = btrfs_csum_root(fs_info: root->fs_info, bytenr: io_start);
1950	ret = btrfs_lookup_csums_list(root: csum_root, start: io_start,
1951	end: io_start + args->file_extent.num_bytes - 1,
1952	NULL, nowait);
1953	WARN_ON_ONCE(ret > 0 && is_freespace_inode);
1954	if (ret != 0)
1955	goto out;
1956
1957	can_nocow = 1;
1958	out:
1959	if (args->free_path && path)
1960	btrfs_free_path(p: path);
1961
1962	return ret < 0 ? ret : can_nocow;
1963	}
1964
1965	static int nocow_one_range(struct btrfs_inode *inode, struct folio *locked_folio,
1966	struct extent_state **cached,
1967	struct can_nocow_file_extent_args *nocow_args,
1968	u64 file_pos, bool is_prealloc)
1969	{
1970	struct btrfs_ordered_extent *ordered;
1971	const u64 len = nocow_args->file_extent.num_bytes;
1972	const u64 end = file_pos + len - 1;
1973	int ret = 0;
1974
1975	btrfs_lock_extent(tree: &inode->io_tree, start: file_pos, end, cached);
1976
1977	if (is_prealloc) {
1978	struct extent_map *em;
1979
1980	em = btrfs_create_io_em(inode, start: file_pos, file_extent: &nocow_args->file_extent,
1981	type: BTRFS_ORDERED_PREALLOC);
1982	if (IS_ERR(ptr: em)) {
1983	ret = PTR_ERR(ptr: em);
1984	goto error;
1985	}
1986	btrfs_free_extent_map(em);
1987	}
1988
1989	ordered = btrfs_alloc_ordered_extent(inode, file_offset: file_pos, file_extent: &nocow_args->file_extent,
1990	flags: is_prealloc
1991	? (1U << BTRFS_ORDERED_PREALLOC)
1992	: (1U << BTRFS_ORDERED_NOCOW));
1993	if (IS_ERR(ptr: ordered)) {
1994	if (is_prealloc)
1995	btrfs_drop_extent_map_range(inode, start: file_pos, end, skip_pinned: false);
1996	ret = PTR_ERR(ptr: ordered);
1997	goto error;
1998	}
1999
2000	if (btrfs_is_data_reloc_root(root: inode->root))
2001	/*
2002	* Errors are handled later, as we must prevent
2003	* extent_clear_unlock_delalloc() in error handler from freeing
2004	* metadata of the created ordered extent.
2005	*/
2006	ret = btrfs_reloc_clone_csums(ordered);
2007	btrfs_put_ordered_extent(entry: ordered);
2008
2009	if (ret < 0)
2010	goto error;
2011	extent_clear_unlock_delalloc(inode, start: file_pos, end, locked_folio, cached,
2012	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC |
2013	EXTENT_CLEAR_DATA_RESV,
2014	page_ops: PAGE_SET_ORDERED);
2015	return ret;
2016
2017	error:
2018	btrfs_cleanup_ordered_extents(inode, offset: file_pos, bytes: len);
2019	extent_clear_unlock_delalloc(inode, start: file_pos, end, locked_folio, cached,
2020	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC |
2021	EXTENT_CLEAR_DATA_RESV,
2022	page_ops: PAGE_UNLOCK | PAGE_START_WRITEBACK |
2023	PAGE_END_WRITEBACK);
2024	btrfs_err(inode->root->fs_info,
2025	"%s failed, root=%lld inode=%llu start=%llu len=%llu: %d",
2026	__func__, btrfs_root_id(inode->root), btrfs_ino(inode),
2027	file_pos, len, ret);
2028	return ret;
2029	}
2030
2031	/*
2032	* When nocow writeback calls back. This checks for snapshots or COW copies
2033	* of the extents that exist in the file, and COWs the file as required.
2034	*
2035	* If no cow copies or snapshots exist, we write directly to the existing
2036	* blocks on disk
2037	*/
2038	static noinline int run_delalloc_nocow(struct btrfs_inode *inode,
2039	struct folio *locked_folio,
2040	const u64 start, const u64 end)
2041	{
2042	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2043	struct btrfs_root *root = inode->root;
2044	struct btrfs_path *path = NULL;
2045	u64 cow_start = (u64)-1;
2046	/*
2047	* If not 0, represents the inclusive end of the last fallback_to_cow()
2048	* range. Only for error handling.
2049	*
2050	* The same for nocow_end, it's to avoid double cleaning up the range
2051	* already cleaned by nocow_one_range().
2052	*/
2053	u64 cow_end = 0;
2054	u64 nocow_end = 0;
2055	u64 cur_offset = start;
2056	int ret;
2057	bool check_prev = true;
2058	u64 ino = btrfs_ino(inode);
2059	struct can_nocow_file_extent_args nocow_args = { 0 };
2060	/* The range that has ordered extent(s). */
2061	u64 oe_cleanup_start;
2062	u64 oe_cleanup_len = 0;
2063	/* The range that is untouched. */
2064	u64 untouched_start;
2065	u64 untouched_len = 0;
2066
2067	/*
2068	* Normally on a zoned device we're only doing COW writes, but in case
2069	* of relocation on a zoned filesystem serializes I/O so that we're only
2070	* writing sequentially and can end up here as well.
2071	*/
2072	ASSERT(!btrfs_is_zoned(fs_info) || btrfs_is_data_reloc_root(root));
2073
2074	if (unlikely(btrfs_is_shutdown(fs_info))) {
2075	ret = -EIO;
2076	goto error;
2077	}
2078	path = btrfs_alloc_path();
2079	if (!path) {
2080	ret = -ENOMEM;
2081	goto error;
2082	}
2083
2084	nocow_args.end = end;
2085	nocow_args.writeback_path = true;
2086
2087	while (cur_offset <= end) {
2088	struct btrfs_block_group *nocow_bg = NULL;
2089	struct btrfs_key found_key;
2090	struct btrfs_file_extent_item *fi;
2091	struct extent_buffer *leaf;
2092	struct extent_state *cached_state = NULL;
2093	u64 extent_end;
2094	int extent_type;
2095
2096	ret = btrfs_lookup_file_extent(NULL, root, path, objectid: ino,
2097	bytenr: cur_offset, mod: 0);
2098	if (ret < 0)
2099	goto error;
2100
2101	/*
2102	* If there is no extent for our range when doing the initial
2103	* search, then go back to the previous slot as it will be the
2104	* one containing the search offset
2105	*/
2106	if (ret > 0 && path->slots[0] > 0 && check_prev) {
2107	leaf = path->nodes[0];
2108	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key,
2109	nr: path->slots[0] - 1);
2110	if (found_key.objectid == ino &&
2111	found_key.type == BTRFS_EXTENT_DATA_KEY)
2112	path->slots[0]--;
2113	}
2114	check_prev = false;
2115	next_slot:
2116	/* Go to next leaf if we have exhausted the current one */
2117	leaf = path->nodes[0];
2118	if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
2119	ret = btrfs_next_leaf(root, path);
2120	if (ret < 0)
2121	goto error;
2122	if (ret > 0)
2123	break;
2124	leaf = path->nodes[0];
2125	}
2126
2127	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
2128
2129	/* Didn't find anything for our INO */
2130	if (found_key.objectid > ino)
2131	break;
2132	/*
2133	* Keep searching until we find an EXTENT_ITEM or there are no
2134	* more extents for this inode
2135	*/
2136	if (WARN_ON_ONCE(found_key.objectid < ino) ||
2137	found_key.type < BTRFS_EXTENT_DATA_KEY) {
2138	path->slots[0]++;
2139	goto next_slot;
2140	}
2141
2142	/* Found key is not EXTENT_DATA_KEY or starts after req range */
2143	if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
2144	found_key.offset > end)
2145	break;
2146
2147	/*
2148	* If the found extent starts after requested offset, then
2149	* adjust cur_offset to be right before this extent begins.
2150	*/
2151	if (found_key.offset > cur_offset) {
2152	if (cow_start == (u64)-1)
2153	cow_start = cur_offset;
2154	cur_offset = found_key.offset;
2155	goto next_slot;
2156	}
2157
2158	/*
2159	* Found extent which begins before our range and potentially
2160	* intersect it
2161	*/
2162	fi = btrfs_item_ptr(leaf, path->slots[0],
2163	struct btrfs_file_extent_item);
2164	extent_type = btrfs_file_extent_type(eb: leaf, s: fi);
2165	/* If this is triggered then we have a memory corruption. */
2166	ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES);
2167	if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) {
2168	ret = -EUCLEAN;
2169	goto error;
2170	}
2171	extent_end = btrfs_file_extent_end(path);
2172
2173	/*
2174	* If the extent we got ends before our current offset, skip to
2175	* the next extent.
2176	*/
2177	if (extent_end <= cur_offset) {
2178	path->slots[0]++;
2179	goto next_slot;
2180	}
2181
2182	nocow_args.start = cur_offset;
2183	ret = can_nocow_file_extent(path, key: &found_key, inode, args: &nocow_args);
2184	if (ret < 0)
2185	goto error;
2186	if (ret == 0)
2187	goto must_cow;
2188
2189	ret = 0;
2190	nocow_bg = btrfs_inc_nocow_writers(fs_info,
2191	bytenr: nocow_args.file_extent.disk_bytenr +
2192	nocow_args.file_extent.offset);
2193	if (!nocow_bg) {
2194	must_cow:
2195	/*
2196	* If we can't perform NOCOW writeback for the range,
2197	* then record the beginning of the range that needs to
2198	* be COWed. It will be written out before the next
2199	* NOCOW range if we find one, or when exiting this
2200	* loop.
2201	*/
2202	if (cow_start == (u64)-1)
2203	cow_start = cur_offset;
2204	cur_offset = extent_end;
2205	if (cur_offset > end)
2206	break;
2207	if (!path->nodes[0])
2208	continue;
2209	path->slots[0]++;
2210	goto next_slot;
2211	}
2212
2213	/*
2214	* COW range from cow_start to found_key.offset - 1. As the key
2215	* will contain the beginning of the first extent that can be
2216	* NOCOW, following one which needs to be COW'ed
2217	*/
2218	if (cow_start != (u64)-1) {
2219	ret = fallback_to_cow(inode, locked_folio, start: cow_start,
2220	end: found_key.offset - 1);
2221	if (ret) {
2222	cow_end = found_key.offset - 1;
2223	btrfs_dec_nocow_writers(bg: nocow_bg);
2224	goto error;
2225	}
2226	cow_start = (u64)-1;
2227	}
2228
2229	ret = nocow_one_range(inode, locked_folio, cached: &cached_state,
2230	nocow_args: &nocow_args, file_pos: cur_offset,
2231	is_prealloc: extent_type == BTRFS_FILE_EXTENT_PREALLOC);
2232	btrfs_dec_nocow_writers(bg: nocow_bg);
2233	if (ret < 0) {
2234	nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1;
2235	goto error;
2236	}
2237	cur_offset = extent_end;
2238	}
2239	btrfs_release_path(p: path);
2240
2241	if (cur_offset <= end && cow_start == (u64)-1)
2242	cow_start = cur_offset;
2243
2244	if (cow_start != (u64)-1) {
2245	ret = fallback_to_cow(inode, locked_folio, start: cow_start, end);
2246	if (ret) {
2247	cow_end = end;
2248	goto error;
2249	}
2250	cow_start = (u64)-1;
2251	}
2252
2253	/*
2254	* Everything is finished without an error, can unlock the folios now.
2255	*
2256	* No need to touch the io tree range nor set folio ordered flag, as
2257	* fallback_to_cow() and nocow_one_range() have already handled them.
2258	*/
2259	extent_clear_unlock_delalloc(inode, start, end, locked_folio, NULL, bits_to_clear: 0, page_ops: PAGE_UNLOCK);
2260
2261	btrfs_free_path(p: path);
2262	return 0;
2263
2264	error:
2265	if (cow_start == (u64)-1) {
2266	/*
2267	* case a)
2268	* start cur_offset end
2269	* | OE cleanup | Untouched |
2270	*
2271	* We finished a fallback_to_cow() or nocow_one_range() call,
2272	* but failed to check the next range.
2273	*
2274	* or
2275	* start cur_offset nocow_end end
2276	* | OE cleanup | Skip | Untouched |
2277	*
2278	* nocow_one_range() failed, the range [cur_offset, nocow_end] is
2279	* already cleaned up.
2280	*/
2281	oe_cleanup_start = start;
2282	oe_cleanup_len = cur_offset - start;
2283	if (nocow_end)
2284	untouched_start = nocow_end + 1;
2285	else
2286	untouched_start = cur_offset;
2287	untouched_len = end + 1 - untouched_start;
2288	} else if (cow_start != (u64)-1 && cow_end == 0) {
2289	/*
2290	* case b)
2291	* start cow_start cur_offset end
2292	* | OE cleanup | Untouched |
2293	*
2294	* We got a range that needs COW, but before we hit the next NOCOW range,
2295	* thus [cow_start, cur_offset) doesn't yet have any OE.
2296	*/
2297	oe_cleanup_start = start;
2298	oe_cleanup_len = cow_start - start;
2299	untouched_start = cow_start;
2300	untouched_len = end + 1 - untouched_start;
2301	} else {
2302	/*
2303	* case c)
2304	* start cow_start cow_end end
2305	* | OE cleanup | Skip | Untouched |
2306	*
2307	* fallback_to_cow() failed, and fallback_to_cow() will do the
2308	* cleanup for its range, we shouldn't touch the range
2309	* [cow_start, cow_end].
2310	*/
2311	ASSERT(cow_start != (u64)-1 && cow_end != 0);
2312	oe_cleanup_start = start;
2313	oe_cleanup_len = cow_start - start;
2314	untouched_start = cow_end + 1;
2315	untouched_len = end + 1 - untouched_start;
2316	}
2317
2318	if (oe_cleanup_len) {
2319	const u64 oe_cleanup_end = oe_cleanup_start + oe_cleanup_len - 1;
2320	btrfs_cleanup_ordered_extents(inode, offset: oe_cleanup_start, bytes: oe_cleanup_len);
2321	extent_clear_unlock_delalloc(inode, start: oe_cleanup_start, end: oe_cleanup_end,
2322	locked_folio, NULL,
2323	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC,
2324	page_ops: PAGE_UNLOCK | PAGE_START_WRITEBACK |
2325	PAGE_END_WRITEBACK);
2326	}
2327
2328	if (untouched_len) {
2329	struct extent_state *cached = NULL;
2330	const u64 untouched_end = untouched_start + untouched_len - 1;
2331
2332	/*
2333	* We need to lock the extent here because we're clearing DELALLOC and
2334	* we're not locked at this point.
2335	*/
2336	btrfs_lock_extent(tree: &inode->io_tree, start: untouched_start, end: untouched_end, cached: &cached);
2337	extent_clear_unlock_delalloc(inode, start: untouched_start, end: untouched_end,
2338	locked_folio, cached: &cached,
2339	bits_to_clear: EXTENT_LOCKED | EXTENT_DELALLOC |
2340	EXTENT_DEFRAG |
2341	EXTENT_DO_ACCOUNTING, page_ops: PAGE_UNLOCK |
2342	PAGE_START_WRITEBACK |
2343	PAGE_END_WRITEBACK);
2344	btrfs_qgroup_free_data(inode, NULL, start: untouched_start, len: untouched_len, NULL);
2345	}
2346	btrfs_free_path(p: path);
2347	btrfs_err(fs_info,
2348	"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %d",
2349	__func__, btrfs_root_id(inode->root), btrfs_ino(inode),
2350	start, end + 1 - start, cur_offset, oe_cleanup_start, oe_cleanup_len,
2351	untouched_start, untouched_len, ret);
2352	return ret;
2353	}
2354
2355	static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end)
2356	{
2357	if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) {
2358	if (inode->defrag_bytes &&
2359	btrfs_test_range_bit_exists(tree: &inode->io_tree, start, end, bit: EXTENT_DEFRAG))
2360	return false;
2361	return true;
2362	}
2363	return false;
2364	}
2365
2366	/*
2367	* Function to process delayed allocation (create CoW) for ranges which are
2368	* being touched for the first time.
2369	*/
2370	int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio,
2371	u64 start, u64 end, struct writeback_control *wbc)
2372	{
2373	const bool zoned = btrfs_is_zoned(fs_info: inode->root->fs_info);
2374	int ret;
2375
2376	/*
2377	* The range must cover part of the @locked_folio, or a return of 1
2378	* can confuse the caller.
2379	*/
2380	ASSERT(!(end <= folio_pos(locked_folio) ||
2381	start >= folio_next_pos(locked_folio)));
2382
2383	if (should_nocow(inode, start, end)) {
2384	ret = run_delalloc_nocow(inode, locked_folio, start, end);
2385	return ret;
2386	}
2387
2388	if (btrfs_inode_can_compress(inode) &&
2389	inode_need_compress(inode, start, end) &&
2390	run_delalloc_compressed(inode, locked_folio, start, end, wbc))
2391	return 1;
2392
2393	if (zoned)
2394	ret = run_delalloc_cow(inode, locked_folio, start, end, wbc,
2395	pages_dirty: true);
2396	else
2397	ret = cow_file_range(inode, locked_folio, start, end, NULL, flags: 0);
2398	return ret;
2399	}
2400
2401	void btrfs_split_delalloc_extent(struct btrfs_inode *inode,
2402	struct extent_state *orig, u64 split)
2403	{
2404	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2405	u64 size;
2406
2407	lockdep_assert_held(&inode->io_tree.lock);
2408
2409	/* not delalloc, ignore it */
2410	if (!(orig->state & EXTENT_DELALLOC))
2411	return;
2412
2413	size = orig->end - orig->start + 1;
2414	if (size > fs_info->max_extent_size) {
2415	u32 num_extents;
2416	u64 new_size;
2417
2418	/*
2419	* See the explanation in btrfs_merge_delalloc_extent, the same
2420	* applies here, just in reverse.
2421	*/
2422	new_size = orig->end - split + 1;
2423	num_extents = count_max_extents(fs_info, size: new_size);
2424	new_size = split - orig->start;
2425	num_extents += count_max_extents(fs_info, size: new_size);
2426	if (count_max_extents(fs_info, size) >= num_extents)
2427	return;
2428	}
2429
2430	spin_lock(lock: &inode->lock);
2431	btrfs_mod_outstanding_extents(inode, mod: 1);
2432	spin_unlock(lock: &inode->lock);
2433	}
2434
2435	/*
2436	* Handle merged delayed allocation extents so we can keep track of new extents
2437	* that are just merged onto old extents, such as when we are doing sequential
2438	* writes, so we can properly account for the metadata space we'll need.
2439	*/
2440	void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new,
2441	struct extent_state *other)
2442	{
2443	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2444	u64 new_size, old_size;
2445	u32 num_extents;
2446
2447	lockdep_assert_held(&inode->io_tree.lock);
2448
2449	/* not delalloc, ignore it */
2450	if (!(other->state & EXTENT_DELALLOC))
2451	return;
2452
2453	if (new->start > other->start)
2454	new_size = new->end - other->start + 1;
2455	else
2456	new_size = other->end - new->start + 1;
2457
2458	/* we're not bigger than the max, unreserve the space and go */
2459	if (new_size <= fs_info->max_extent_size) {
2460	spin_lock(lock: &inode->lock);
2461	btrfs_mod_outstanding_extents(inode, mod: -1);
2462	spin_unlock(lock: &inode->lock);
2463	return;
2464	}
2465
2466	/*
2467	* We have to add up either side to figure out how many extents were
2468	* accounted for before we merged into one big extent. If the number of
2469	* extents we accounted for is <= the amount we need for the new range
2470	* then we can return, otherwise drop. Think of it like this
2471	*
2472	* [ 4k][MAX_SIZE]
2473	*
2474	* So we've grown the extent by a MAX_SIZE extent, this would mean we
2475	* need 2 outstanding extents, on one side we have 1 and the other side
2476	* we have 1 so they are == and we can return. But in this case
2477	*
2478	* [MAX_SIZE+4k][MAX_SIZE+4k]
2479	*
2480	* Each range on their own accounts for 2 extents, but merged together
2481	* they are only 3 extents worth of accounting, so we need to drop in
2482	* this case.
2483	*/
2484	old_size = other->end - other->start + 1;
2485	num_extents = count_max_extents(fs_info, size: old_size);
2486	old_size = new->end - new->start + 1;
2487	num_extents += count_max_extents(fs_info, size: old_size);
2488	if (count_max_extents(fs_info, size: new_size) >= num_extents)
2489	return;
2490
2491	spin_lock(lock: &inode->lock);
2492	btrfs_mod_outstanding_extents(inode, mod: -1);
2493	spin_unlock(lock: &inode->lock);
2494	}
2495
2496	static void btrfs_add_delalloc_inode(struct btrfs_inode *inode)
2497	{
2498	struct btrfs_root *root = inode->root;
2499	struct btrfs_fs_info *fs_info = root->fs_info;
2500
2501	spin_lock(lock: &root->delalloc_lock);
2502	ASSERT(list_empty(&inode->delalloc_inodes));
2503	list_add_tail(new: &inode->delalloc_inodes, head: &root->delalloc_inodes);
2504	root->nr_delalloc_inodes++;
2505	if (root->nr_delalloc_inodes == 1) {
2506	spin_lock(lock: &fs_info->delalloc_root_lock);
2507	ASSERT(list_empty(&root->delalloc_root));
2508	list_add_tail(new: &root->delalloc_root, head: &fs_info->delalloc_roots);
2509	spin_unlock(lock: &fs_info->delalloc_root_lock);
2510	}
2511	spin_unlock(lock: &root->delalloc_lock);
2512	}
2513
2514	void btrfs_del_delalloc_inode(struct btrfs_inode *inode)
2515	{
2516	struct btrfs_root *root = inode->root;
2517	struct btrfs_fs_info *fs_info = root->fs_info;
2518
2519	lockdep_assert_held(&root->delalloc_lock);
2520
2521	/*
2522	* We may be called after the inode was already deleted from the list,
2523	* namely in the transaction abort path btrfs_destroy_delalloc_inodes(),
2524	* and then later through btrfs_clear_delalloc_extent() while the inode
2525	* still has ->delalloc_bytes > 0.
2526	*/
2527	if (!list_empty(head: &inode->delalloc_inodes)) {
2528	list_del_init(entry: &inode->delalloc_inodes);
2529	root->nr_delalloc_inodes--;
2530	if (!root->nr_delalloc_inodes) {
2531	ASSERT(list_empty(&root->delalloc_inodes));
2532	spin_lock(lock: &fs_info->delalloc_root_lock);
2533	ASSERT(!list_empty(&root->delalloc_root));
2534	list_del_init(entry: &root->delalloc_root);
2535	spin_unlock(lock: &fs_info->delalloc_root_lock);
2536	}
2537	}
2538	}
2539
2540	/*
2541	* Properly track delayed allocation bytes in the inode and to maintain the
2542	* list of inodes that have pending delalloc work to be done.
2543	*/
2544	void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state,
2545	u32 bits)
2546	{
2547	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2548
2549	lockdep_assert_held(&inode->io_tree.lock);
2550
2551	if ((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC))
2552	WARN_ON(1);
2553	/*
2554	* set_bit and clear bit hooks normally require _irqsave/restore
2555	* but in this case, we are only testing for the DELALLOC
2556	* bit, which is only set or cleared with irqs on
2557	*/
2558	if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2559	u64 len = state->end + 1 - state->start;
2560	u64 prev_delalloc_bytes;
2561	u32 num_extents = count_max_extents(fs_info, size: len);
2562
2563	spin_lock(lock: &inode->lock);
2564	btrfs_mod_outstanding_extents(inode, mod: num_extents);
2565	spin_unlock(lock: &inode->lock);
2566
2567	/* For sanity tests */
2568	if (btrfs_is_testing(fs_info))
2569	return;
2570
2571	percpu_counter_add_batch(fbc: &fs_info->delalloc_bytes, amount: len,
2572	batch: fs_info->delalloc_batch);
2573	spin_lock(lock: &inode->lock);
2574	prev_delalloc_bytes = inode->delalloc_bytes;
2575	inode->delalloc_bytes += len;
2576	if (bits & EXTENT_DEFRAG)
2577	inode->defrag_bytes += len;
2578	spin_unlock(lock: &inode->lock);
2579
2580	/*
2581	* We don't need to be under the protection of the inode's lock,
2582	* because we are called while holding the inode's io_tree lock
2583	* and are therefore protected against concurrent calls of this
2584	* function and btrfs_clear_delalloc_extent().
2585	*/
2586	if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0)
2587	btrfs_add_delalloc_inode(inode);
2588	}
2589
2590	if (!(state->state & EXTENT_DELALLOC_NEW) &&
2591	(bits & EXTENT_DELALLOC_NEW)) {
2592	spin_lock(lock: &inode->lock);
2593	inode->new_delalloc_bytes += state->end + 1 - state->start;
2594	spin_unlock(lock: &inode->lock);
2595	}
2596	}
2597
2598	/*
2599	* Once a range is no longer delalloc this function ensures that proper
2600	* accounting happens.
2601	*/
2602	void btrfs_clear_delalloc_extent(struct btrfs_inode *inode,
2603	struct extent_state *state, u32 bits)
2604	{
2605	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2606	u64 len = state->end + 1 - state->start;
2607	u32 num_extents = count_max_extents(fs_info, size: len);
2608
2609	lockdep_assert_held(&inode->io_tree.lock);
2610
2611	if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) {
2612	spin_lock(lock: &inode->lock);
2613	inode->defrag_bytes -= len;
2614	spin_unlock(lock: &inode->lock);
2615	}
2616
2617	/*
2618	* set_bit and clear bit hooks normally require _irqsave/restore
2619	* but in this case, we are only testing for the DELALLOC
2620	* bit, which is only set or cleared with irqs on
2621	*/
2622	if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) {
2623	struct btrfs_root *root = inode->root;
2624	u64 new_delalloc_bytes;
2625
2626	spin_lock(lock: &inode->lock);
2627	btrfs_mod_outstanding_extents(inode, mod: -num_extents);
2628	spin_unlock(lock: &inode->lock);
2629
2630	/*
2631	* We don't reserve metadata space for space cache inodes so we
2632	* don't need to call delalloc_release_metadata if there is an
2633	* error.
2634	*/
2635	if (bits & EXTENT_CLEAR_META_RESV &&
2636	root != fs_info->tree_root)
2637	btrfs_delalloc_release_metadata(inode, num_bytes: len, qgroup_free: true);
2638
2639	/* For sanity tests. */
2640	if (btrfs_is_testing(fs_info))
2641	return;
2642
2643	if (!btrfs_is_data_reloc_root(root) &&
2644	!btrfs_is_free_space_inode(inode) &&
2645	!(state->state & EXTENT_NORESERVE) &&
2646	(bits & EXTENT_CLEAR_DATA_RESV))
2647	btrfs_free_reserved_data_space_noquota(inode, len);
2648
2649	percpu_counter_add_batch(fbc: &fs_info->delalloc_bytes, amount: -len,
2650	batch: fs_info->delalloc_batch);
2651	spin_lock(lock: &inode->lock);
2652	inode->delalloc_bytes -= len;
2653	new_delalloc_bytes = inode->delalloc_bytes;
2654	spin_unlock(lock: &inode->lock);
2655
2656	/*
2657	* We don't need to be under the protection of the inode's lock,
2658	* because we are called while holding the inode's io_tree lock
2659	* and are therefore protected against concurrent calls of this
2660	* function and btrfs_set_delalloc_extent().
2661	*/
2662	if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) {
2663	spin_lock(lock: &root->delalloc_lock);
2664	btrfs_del_delalloc_inode(inode);
2665	spin_unlock(lock: &root->delalloc_lock);
2666	}
2667	}
2668
2669	if ((state->state & EXTENT_DELALLOC_NEW) &&
2670	(bits & EXTENT_DELALLOC_NEW)) {
2671	spin_lock(lock: &inode->lock);
2672	ASSERT(inode->new_delalloc_bytes >= len);
2673	inode->new_delalloc_bytes -= len;
2674	if (bits & EXTENT_ADD_INODE_BYTES)
2675	inode_add_bytes(inode: &inode->vfs_inode, bytes: len);
2676	spin_unlock(lock: &inode->lock);
2677	}
2678	}
2679
2680	/*
2681	* given a list of ordered sums record them in the inode. This happens
2682	* at IO completion time based on sums calculated at bio submission time.
2683	*/
2684	static int add_pending_csums(struct btrfs_trans_handle *trans,
2685	struct list_head *list)
2686	{
2687	struct btrfs_ordered_sum *sum;
2688	struct btrfs_root *csum_root = NULL;
2689	int ret;
2690
2691	list_for_each_entry(sum, list, list) {
2692	trans->adding_csums = true;
2693	if (!csum_root)
2694	csum_root = btrfs_csum_root(fs_info: trans->fs_info,
2695	bytenr: sum->logical);
2696	ret = btrfs_csum_file_blocks(trans, root: csum_root, sums: sum);
2697	trans->adding_csums = false;
2698	if (ret)
2699	return ret;
2700	}
2701	return 0;
2702	}
2703
2704	static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
2705	const u64 start,
2706	const u64 len,
2707	struct extent_state **cached_state)
2708	{
2709	u64 search_start = start;
2710	const u64 end = start + len - 1;
2711
2712	while (search_start < end) {
2713	const u64 search_len = end - search_start + 1;
2714	struct extent_map *em;
2715	u64 em_len;
2716	int ret = 0;
2717
2718	em = btrfs_get_extent(inode, NULL, start: search_start, len: search_len);
2719	if (IS_ERR(ptr: em))
2720	return PTR_ERR(ptr: em);
2721
2722	if (em->disk_bytenr != EXTENT_MAP_HOLE)
2723	goto next;
2724
2725	em_len = em->len;
2726	if (em->start < search_start)
2727	em_len -= search_start - em->start;
2728	if (em_len > search_len)
2729	em_len = search_len;
2730
2731	ret = btrfs_set_extent_bit(tree: &inode->io_tree, start: search_start,
2732	end: search_start + em_len - 1,
2733	bits: EXTENT_DELALLOC_NEW, cached_state);
2734	next:
2735	search_start = btrfs_extent_map_end(em);
2736	btrfs_free_extent_map(em);
2737	if (ret)
2738	return ret;
2739	}
2740	return 0;
2741	}
2742
2743	int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
2744	unsigned int extra_bits,
2745	struct extent_state **cached_state)
2746	{
2747	WARN_ON(PAGE_ALIGNED(end));
2748
2749	if (start >= i_size_read(inode: &inode->vfs_inode) &&
2750	!(inode->flags & BTRFS_INODE_PREALLOC)) {
2751	/*
2752	* There can't be any extents following eof in this case so just
2753	* set the delalloc new bit for the range directly.
2754	*/
2755	extra_bits |= EXTENT_DELALLOC_NEW;
2756	} else {
2757	int ret;
2758
2759	ret = btrfs_find_new_delalloc_bytes(inode, start,
2760	len: end + 1 - start,
2761	cached_state);
2762	if (ret)
2763	return ret;
2764	}
2765
2766	return btrfs_set_extent_bit(tree: &inode->io_tree, start, end,
2767	bits: EXTENT_DELALLOC | extra_bits, cached_state);
2768	}
2769
2770	/* see btrfs_writepage_start_hook for details on why this is required */
2771	struct btrfs_writepage_fixup {
2772	struct folio *folio;
2773	struct btrfs_inode *inode;
2774	struct btrfs_work work;
2775	};
2776
2777	static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2778	{
2779	struct btrfs_writepage_fixup *fixup =
2780	container_of(work, struct btrfs_writepage_fixup, work);
2781	struct btrfs_ordered_extent *ordered;
2782	struct extent_state *cached_state = NULL;
2783	struct extent_changeset *data_reserved = NULL;
2784	struct folio *folio = fixup->folio;
2785	struct btrfs_inode *inode = fixup->inode;
2786	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2787	u64 page_start = folio_pos(folio);
2788	u64 page_end = folio_next_pos(folio) - 1;
2789	int ret = 0;
2790	bool free_delalloc_space = true;
2791
2792	/*
2793	* This is similar to page_mkwrite, we need to reserve the space before
2794	* we take the folio lock.
2795	*/
2796	ret = btrfs_delalloc_reserve_space(inode, reserved: &data_reserved, start: page_start,
2797	len: folio_size(folio));
2798	again:
2799	folio_lock(folio);
2800
2801	/*
2802	* Before we queued this fixup, we took a reference on the folio.
2803	* folio->mapping may go NULL, but it shouldn't be moved to a different
2804	* address space.
2805	*/
2806	if (!folio->mapping || !folio_test_dirty(folio) ||
2807	!folio_test_checked(folio)) {
2808	/*
2809	* Unfortunately this is a little tricky, either
2810	*
2811	* 1) We got here and our folio had already been dealt with and
2812	* we reserved our space, thus ret == 0, so we need to just
2813	* drop our space reservation and bail. This can happen the
2814	* first time we come into the fixup worker, or could happen
2815	* while waiting for the ordered extent.
2816	* 2) Our folio was already dealt with, but we happened to get an
2817	* ENOSPC above from the btrfs_delalloc_reserve_space. In
2818	* this case we obviously don't have anything to release, but
2819	* because the folio was already dealt with we don't want to
2820	* mark the folio with an error, so make sure we're resetting
2821	* ret to 0. This is why we have this check _before_ the ret
2822	* check, because we do not want to have a surprise ENOSPC
2823	* when the folio was already properly dealt with.
2824	*/
2825	if (!ret) {
2826	btrfs_delalloc_release_extents(inode, num_bytes: folio_size(folio));
2827	btrfs_delalloc_release_space(inode, reserved: data_reserved,
2828	start: page_start, len: folio_size(folio),
2829	qgroup_free: true);
2830	}
2831	ret = 0;
2832	goto out_page;
2833	}
2834
2835	/*
2836	* We can't mess with the folio state unless it is locked, so now that
2837	* it is locked bail if we failed to make our space reservation.
2838	*/
2839	if (ret)
2840	goto out_page;
2841
2842	btrfs_lock_extent(tree: &inode->io_tree, start: page_start, end: page_end, cached: &cached_state);
2843
2844	/* already ordered? We're done */
2845	if (folio_test_ordered(folio))
2846	goto out_reserved;
2847
2848	ordered = btrfs_lookup_ordered_range(inode, file_offset: page_start, PAGE_SIZE);
2849	if (ordered) {
2850	btrfs_unlock_extent(tree: &inode->io_tree, start: page_start, end: page_end,
2851	cached: &cached_state);
2852	folio_unlock(folio);
2853	btrfs_start_ordered_extent(entry: ordered);
2854	btrfs_put_ordered_extent(entry: ordered);
2855	goto again;
2856	}
2857
2858	ret = btrfs_set_extent_delalloc(inode, start: page_start, end: page_end, extra_bits: 0,
2859	cached_state: &cached_state);
2860	if (ret)
2861	goto out_reserved;
2862
2863	/*
2864	* Everything went as planned, we're now the owner of a dirty page with
2865	* delayed allocation bits set and space reserved for our COW
2866	* destination.
2867	*
2868	* The page was dirty when we started, nothing should have cleaned it.
2869	*/
2870	BUG_ON(!folio_test_dirty(folio));
2871	free_delalloc_space = false;
2872	out_reserved:
2873	btrfs_delalloc_release_extents(inode, PAGE_SIZE);
2874	if (free_delalloc_space)
2875	btrfs_delalloc_release_space(inode, reserved: data_reserved, start: page_start,
2876	PAGE_SIZE, qgroup_free: true);
2877	btrfs_unlock_extent(tree: &inode->io_tree, start: page_start, end: page_end, cached: &cached_state);
2878	out_page:
2879	if (ret) {
2880	/*
2881	* We hit ENOSPC or other errors. Update the mapping and page
2882	* to reflect the errors and clean the page.
2883	*/
2884	mapping_set_error(mapping: folio->mapping, error: ret);
2885	btrfs_mark_ordered_io_finished(inode, folio, file_offset: page_start,
2886	num_bytes: folio_size(folio), uptodate: !ret);
2887	folio_clear_dirty_for_io(folio);
2888	}
2889	btrfs_folio_clear_checked(fs_info, folio, start: page_start, PAGE_SIZE);
2890	folio_unlock(folio);
2891	folio_put(folio);
2892	kfree(objp: fixup);
2893	extent_changeset_free(changeset: data_reserved);
2894	/*
2895	* As a precaution, do a delayed iput in case it would be the last iput
2896	* that could need flushing space. Recursing back to fixup worker would
2897	* deadlock.
2898	*/
2899	btrfs_add_delayed_iput(inode);
2900	}
2901
2902	/*
2903	* There are a few paths in the higher layers of the kernel that directly
2904	* set the folio dirty bit without asking the filesystem if it is a
2905	* good idea. This causes problems because we want to make sure COW
2906	* properly happens and the data=ordered rules are followed.
2907	*
2908	* In our case any range that doesn't have the ORDERED bit set
2909	* hasn't been properly setup for IO. We kick off an async process
2910	* to fix it up. The async helper will wait for ordered extents, set
2911	* the delalloc bit and make it safe to write the folio.
2912	*/
2913	int btrfs_writepage_cow_fixup(struct folio *folio)
2914	{
2915	struct inode *inode = folio->mapping->host;
2916	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2917	struct btrfs_writepage_fixup *fixup;
2918
2919	/* This folio has ordered extent covering it already */
2920	if (folio_test_ordered(folio))
2921	return 0;
2922
2923	/*
2924	* For experimental build, we error out instead of EAGAIN.
2925	*
2926	* We should not hit such out-of-band dirty folios anymore.
2927	*/
2928	if (IS_ENABLED(CONFIG_BTRFS_EXPERIMENTAL)) {
2929	DEBUG_WARN();
2930	btrfs_err_rl(fs_info,
2931	"root %lld ino %llu folio %llu is marked dirty without notifying the fs",
2932	btrfs_root_id(BTRFS_I(inode)->root),
2933	btrfs_ino(BTRFS_I(inode)),
2934	folio_pos(folio));
2935	return -EUCLEAN;
2936	}
2937
2938	/*
2939	* folio_checked is set below when we create a fixup worker for this
2940	* folio, don't try to create another one if we're already
2941	* folio_test_checked.
2942	*
2943	* The extent_io writepage code will redirty the foio if we send back
2944	* EAGAIN.
2945	*/
2946	if (folio_test_checked(folio))
2947	return -EAGAIN;
2948
2949	fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2950	if (!fixup)
2951	return -EAGAIN;
2952
2953	/*
2954	* We are already holding a reference to this inode from
2955	* write_cache_pages. We need to hold it because the space reservation
2956	* takes place outside of the folio lock, and we can't trust
2957	* folio->mapping outside of the folio lock.
2958	*/
2959	ihold(inode);
2960	btrfs_folio_set_checked(fs_info, folio, start: folio_pos(folio), len: folio_size(folio));
2961	folio_get(folio);
2962	btrfs_init_work(work: &fixup->work, func: btrfs_writepage_fixup_worker, NULL);
2963	fixup->folio = folio;
2964	fixup->inode = BTRFS_I(inode);
2965	btrfs_queue_work(wq: fs_info->fixup_workers, work: &fixup->work);
2966
2967	return -EAGAIN;
2968	}
2969
2970	static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2971	struct btrfs_inode *inode, u64 file_pos,
2972	struct btrfs_file_extent_item *stack_fi,
2973	const bool update_inode_bytes,
2974	u64 qgroup_reserved)
2975	{
2976	struct btrfs_root *root = inode->root;
2977	const u64 sectorsize = root->fs_info->sectorsize;
2978	BTRFS_PATH_AUTO_FREE(path);
2979	struct extent_buffer *leaf;
2980	struct btrfs_key ins;
2981	u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(s: stack_fi);
2982	u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(s: stack_fi);
2983	u64 offset = btrfs_stack_file_extent_offset(s: stack_fi);
2984	u64 num_bytes = btrfs_stack_file_extent_num_bytes(s: stack_fi);
2985	u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(s: stack_fi);
2986	struct btrfs_drop_extents_args drop_args = { 0 };
2987	int ret;
2988
2989	path = btrfs_alloc_path();
2990	if (!path)
2991	return -ENOMEM;
2992
2993	/*
2994	* we may be replacing one extent in the tree with another.
2995	* The new extent is pinned in the extent map, and we don't want
2996	* to drop it from the cache until it is completely in the btree.
2997	*
2998	* So, tell btrfs_drop_extents to leave this extent in the cache.
2999	* the caller is expected to unpin it and allow it to be merged
3000	* with the others.
3001	*/
3002	drop_args.path = path;
3003	drop_args.start = file_pos;
3004	drop_args.end = file_pos + num_bytes;
3005	drop_args.replace_extent = true;
3006	drop_args.extent_item_size = sizeof(*stack_fi);
3007	ret = btrfs_drop_extents(trans, root, inode, args: &drop_args);
3008	if (ret)
3009	goto out;
3010
3011	if (!drop_args.extent_inserted) {
3012	ins.objectid = btrfs_ino(inode);
3013	ins.type = BTRFS_EXTENT_DATA_KEY;
3014	ins.offset = file_pos;
3015
3016	ret = btrfs_insert_empty_item(trans, root, path, key: &ins,
3017	data_size: sizeof(*stack_fi));
3018	if (ret)
3019	goto out;
3020	}
3021	leaf = path->nodes[0];
3022	btrfs_set_stack_file_extent_generation(s: stack_fi, val: trans->transid);
3023	write_extent_buffer(eb: leaf, src: stack_fi,
3024	btrfs_item_ptr_offset(leaf, path->slots[0]),
3025	len: sizeof(struct btrfs_file_extent_item));
3026
3027	btrfs_release_path(p: path);
3028
3029	/*
3030	* If we dropped an inline extent here, we know the range where it is
3031	* was not marked with the EXTENT_DELALLOC_NEW bit, so we update the
3032	* number of bytes only for that range containing the inline extent.
3033	* The remaining of the range will be processed when clearing the
3034	* EXTENT_DELALLOC_BIT bit through the ordered extent completion.
3035	*/
3036	if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) {
3037	u64 inline_size = round_down(drop_args.bytes_found, sectorsize);
3038
3039	inline_size = drop_args.bytes_found - inline_size;
3040	btrfs_update_inode_bytes(inode, add_bytes: sectorsize, del_bytes: inline_size);
3041	drop_args.bytes_found -= inline_size;
3042	num_bytes -= sectorsize;
3043	}
3044
3045	if (update_inode_bytes)
3046	btrfs_update_inode_bytes(inode, add_bytes: num_bytes, del_bytes: drop_args.bytes_found);
3047
3048	ins.objectid = disk_bytenr;
3049	ins.type = BTRFS_EXTENT_ITEM_KEY;
3050	ins.offset = disk_num_bytes;
3051
3052	ret = btrfs_inode_set_file_extent_range(inode, start: file_pos, len: ram_bytes);
3053	if (ret)
3054	goto out;
3055
3056	ret = btrfs_alloc_reserved_file_extent(trans, root, owner: btrfs_ino(inode),
3057	offset: file_pos - offset,
3058	ram_bytes: qgroup_reserved, ins: &ins);
3059	out:
3060	return ret;
3061	}
3062
3063	static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
3064	u64 start, u64 len)
3065	{
3066	struct btrfs_block_group *cache;
3067
3068	cache = btrfs_lookup_block_group(info: fs_info, bytenr: start);
3069	ASSERT(cache);
3070
3071	spin_lock(lock: &cache->lock);
3072	cache->delalloc_bytes -= len;
3073	spin_unlock(lock: &cache->lock);
3074
3075	btrfs_put_block_group(cache);
3076	}
3077
3078	static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,
3079	struct btrfs_ordered_extent *oe)
3080	{
3081	struct btrfs_file_extent_item stack_fi;
3082	bool update_inode_bytes;
3083	u64 num_bytes = oe->num_bytes;
3084	u64 ram_bytes = oe->ram_bytes;
3085
3086	memset(&stack_fi, 0, sizeof(stack_fi));
3087	btrfs_set_stack_file_extent_type(s: &stack_fi, val: BTRFS_FILE_EXTENT_REG);
3088	btrfs_set_stack_file_extent_disk_bytenr(s: &stack_fi, val: oe->disk_bytenr);
3089	btrfs_set_stack_file_extent_disk_num_bytes(s: &stack_fi,
3090	val: oe->disk_num_bytes);
3091	btrfs_set_stack_file_extent_offset(s: &stack_fi, val: oe->offset);
3092	if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))
3093	num_bytes = oe->truncated_len;
3094	btrfs_set_stack_file_extent_num_bytes(s: &stack_fi, val: num_bytes);
3095	btrfs_set_stack_file_extent_ram_bytes(s: &stack_fi, val: ram_bytes);
3096	btrfs_set_stack_file_extent_compression(s: &stack_fi, val: oe->compress_type);
3097	/* Encryption and other encoding is reserved and all 0 */
3098
3099	/*
3100	* For delalloc, when completing an ordered extent we update the inode's
3101	* bytes when clearing the range in the inode's io tree, so pass false
3102	* as the argument 'update_inode_bytes' to insert_reserved_file_extent(),
3103	* except if the ordered extent was truncated.
3104	*/
3105	update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) ||
3106	test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) ||
3107	test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags);
3108
3109	return insert_reserved_file_extent(trans, inode: oe->inode,
3110	file_pos: oe->file_offset, stack_fi: &stack_fi,
3111	update_inode_bytes, qgroup_reserved: oe->qgroup_rsv);
3112	}
3113
3114	/*
3115	* As ordered data IO finishes, this gets called so we can finish
3116	* an ordered extent if the range of bytes in the file it covers are
3117	* fully written.
3118	*/
3119	int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent)
3120	{
3121	struct btrfs_inode *inode = ordered_extent->inode;
3122	struct btrfs_root *root = inode->root;
3123	struct btrfs_fs_info *fs_info = root->fs_info;
3124	struct btrfs_trans_handle *trans = NULL;
3125	struct extent_io_tree *io_tree = &inode->io_tree;
3126	struct extent_state *cached_state = NULL;
3127	u64 start, end;
3128	int compress_type = 0;
3129	int ret = 0;
3130	u64 logical_len = ordered_extent->num_bytes;
3131	bool freespace_inode;
3132	bool truncated = false;
3133	bool clear_reserved_extent = true;
3134	unsigned int clear_bits = EXTENT_DEFRAG;
3135
3136	start = ordered_extent->file_offset;
3137	end = start + ordered_extent->num_bytes - 1;
3138
3139	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3140	!test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
3141	!test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) &&
3142	!test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags))
3143	clear_bits |= EXTENT_DELALLOC_NEW;
3144
3145	freespace_inode = btrfs_is_free_space_inode(inode);
3146	if (!freespace_inode)
3147	btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent);
3148
3149	if (unlikely(test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags))) {
3150	ret = -EIO;
3151	goto out;
3152	}
3153
3154	ret = btrfs_zone_finish_endio(fs_info, logical: ordered_extent->disk_bytenr,
3155	length: ordered_extent->disk_num_bytes);
3156	if (ret)
3157	goto out;
3158
3159	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
3160	truncated = true;
3161	logical_len = ordered_extent->truncated_len;
3162	/* Truncated the entire extent, don't bother adding */
3163	if (!logical_len)
3164	goto out;
3165	}
3166
3167	/*
3168	* If it's a COW write we need to lock the extent range as we will be
3169	* inserting/replacing file extent items and unpinning an extent map.
3170	* This must be taken before joining a transaction, as it's a higher
3171	* level lock (like the inode's VFS lock), otherwise we can run into an
3172	* ABBA deadlock with other tasks (transactions work like a lock,
3173	* depending on their current state).
3174	*/
3175	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3176	clear_bits |= EXTENT_LOCKED | EXTENT_FINISHING_ORDERED;
3177	btrfs_lock_extent_bits(tree: io_tree, start, end,
3178	bits: EXTENT_LOCKED | EXTENT_FINISHING_ORDERED,
3179	cached: &cached_state);
3180	}
3181
3182	if (freespace_inode)
3183	trans = btrfs_join_transaction_spacecache(root);
3184	else
3185	trans = btrfs_join_transaction(root);
3186	if (IS_ERR(ptr: trans)) {
3187	ret = PTR_ERR(ptr: trans);
3188	trans = NULL;
3189	goto out;
3190	}
3191
3192	trans->block_rsv = &inode->block_rsv;
3193
3194	ret = btrfs_insert_raid_extent(trans, ordered_extent);
3195	if (unlikely(ret)) {
3196	btrfs_abort_transaction(trans, ret);
3197	goto out;
3198	}
3199
3200	if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
3201	/* Logic error */
3202	ASSERT(list_empty(&ordered_extent->list));
3203	if (unlikely(!list_empty(&ordered_extent->list))) {
3204	ret = -EINVAL;
3205	btrfs_abort_transaction(trans, ret);
3206	goto out;
3207	}
3208
3209	btrfs_inode_safe_disk_i_size_write(inode, new_i_size: 0);
3210	ret = btrfs_update_inode_fallback(trans, inode);
3211	if (unlikely(ret)) {
3212	/* -ENOMEM or corruption */
3213	btrfs_abort_transaction(trans, ret);
3214	}
3215	goto out;
3216	}
3217
3218	if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3219	compress_type = ordered_extent->compress_type;
3220	if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3221	BUG_ON(compress_type);
3222	ret = btrfs_mark_extent_written(trans, inode,
3223	start: ordered_extent->file_offset,
3224	end: ordered_extent->file_offset +
3225	logical_len);
3226	btrfs_zoned_release_data_reloc_bg(fs_info, logical: ordered_extent->disk_bytenr,
3227	length: ordered_extent->disk_num_bytes);
3228	} else {
3229	BUG_ON(root == fs_info->tree_root);
3230	ret = insert_ordered_extent_file_extent(trans, oe: ordered_extent);
3231	if (!ret) {
3232	clear_reserved_extent = false;
3233	btrfs_release_delalloc_bytes(fs_info,
3234	start: ordered_extent->disk_bytenr,
3235	len: ordered_extent->disk_num_bytes);
3236	}
3237	}
3238	if (unlikely(ret < 0)) {
3239	btrfs_abort_transaction(trans, ret);
3240	goto out;
3241	}
3242
3243	ret = btrfs_unpin_extent_cache(inode, start: ordered_extent->file_offset,
3244	len: ordered_extent->num_bytes, gen: trans->transid);
3245	if (unlikely(ret < 0)) {
3246	btrfs_abort_transaction(trans, ret);
3247	goto out;
3248	}
3249
3250	ret = add_pending_csums(trans, list: &ordered_extent->list);
3251	if (unlikely(ret)) {
3252	btrfs_abort_transaction(trans, ret);
3253	goto out;
3254	}
3255
3256	/*
3257	* If this is a new delalloc range, clear its new delalloc flag to
3258	* update the inode's number of bytes. This needs to be done first
3259	* before updating the inode item.
3260	*/
3261	if ((clear_bits & EXTENT_DELALLOC_NEW) &&
3262	!test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags))
3263	btrfs_clear_extent_bit(tree: &inode->io_tree, start, end,
3264	bits: EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES,
3265	cached: &cached_state);
3266
3267	btrfs_inode_safe_disk_i_size_write(inode, new_i_size: 0);
3268	ret = btrfs_update_inode_fallback(trans, inode);
3269	if (unlikely(ret)) { /* -ENOMEM or corruption */
3270	btrfs_abort_transaction(trans, ret);
3271	goto out;
3272	}
3273	out:
3274	btrfs_clear_extent_bit(tree: &inode->io_tree, start, end, bits: clear_bits,
3275	cached: &cached_state);
3276
3277	if (trans)
3278	btrfs_end_transaction(trans);
3279
3280	if (ret || truncated) {
3281	/*
3282	* If we failed to finish this ordered extent for any reason we
3283	* need to make sure BTRFS_ORDERED_IOERR is set on the ordered
3284	* extent, and mark the inode with the error if it wasn't
3285	* already set. Any error during writeback would have already
3286	* set the mapping error, so we need to set it if we're the ones
3287	* marking this ordered extent as failed.
3288	*/
3289	if (ret)
3290	btrfs_mark_ordered_extent_error(ordered: ordered_extent);
3291
3292	/*
3293	* Drop extent maps for the part of the extent we didn't write.
3294	*
3295	* We have an exception here for the free_space_inode, this is
3296	* because when we do btrfs_get_extent() on the free space inode
3297	* we will search the commit root. If this is a new block group
3298	* we won't find anything, and we will trip over the assert in
3299	* writepage where we do ASSERT(em->block_start !=
3300	* EXTENT_MAP_HOLE).
3301	*
3302	* Theoretically we could also skip this for any NOCOW extent as
3303	* we don't mess with the extent map tree in the NOCOW case, but
3304	* for now simply skip this if we are the free space inode.
3305	*/
3306	if (!btrfs_is_free_space_inode(inode)) {
3307	u64 unwritten_start = start;
3308
3309	if (truncated)
3310	unwritten_start += logical_len;
3311
3312	btrfs_drop_extent_map_range(inode, start: unwritten_start,
3313	end, skip_pinned: false);
3314	}
3315
3316	/*
3317	* If the ordered extent had an IOERR or something else went
3318	* wrong we need to return the space for this ordered extent
3319	* back to the allocator. We only free the extent in the
3320	* truncated case if we didn't write out the extent at all.
3321	*
3322	* If we made it past insert_reserved_file_extent before we
3323	* errored out then we don't need to do this as the accounting
3324	* has already been done.
3325	*/
3326	if ((ret || !logical_len) &&
3327	clear_reserved_extent &&
3328	!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3329	!test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3330	/*
3331	* Discard the range before returning it back to the
3332	* free space pool
3333	*/
3334	if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC))
3335	btrfs_discard_extent(fs_info,
3336	bytenr: ordered_extent->disk_bytenr,
3337	num_bytes: ordered_extent->disk_num_bytes,
3338	NULL);
3339	btrfs_free_reserved_extent(fs_info,
3340	start: ordered_extent->disk_bytenr,
3341	len: ordered_extent->disk_num_bytes, is_delalloc: true);
3342	/*
3343	* Actually free the qgroup rsv which was released when
3344	* the ordered extent was created.
3345	*/
3346	btrfs_qgroup_free_refroot(fs_info, ref_root: btrfs_root_id(root: inode->root),
3347	num_bytes: ordered_extent->qgroup_rsv,
3348	type: BTRFS_QGROUP_RSV_DATA);
3349	}
3350	}
3351
3352	/*
3353	* This needs to be done to make sure anybody waiting knows we are done
3354	* updating everything for this ordered extent.
3355	*/
3356	btrfs_remove_ordered_extent(btrfs_inode: inode, entry: ordered_extent);
3357
3358	/* once for us */
3359	btrfs_put_ordered_extent(entry: ordered_extent);
3360	/* once for the tree */
3361	btrfs_put_ordered_extent(entry: ordered_extent);
3362
3363	return ret;
3364	}
3365
3366	int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)
3367	{
3368	if (btrfs_is_zoned(fs_info: ordered->inode->root->fs_info) &&
3369	!test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) &&
3370	list_empty(head: &ordered->bioc_list))
3371	btrfs_finish_ordered_zoned(ordered);
3372	return btrfs_finish_one_ordered(ordered_extent: ordered);
3373	}
3374
3375	/*
3376	* Calculate the checksum of an fs block at physical memory address @paddr,
3377	* and save the result to @dest.
3378	*
3379	* The folio containing @paddr must be large enough to contain a full fs block.
3380	*/
3381	void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info,
3382	const phys_addr_t paddr, u8 *dest)
3383	{
3384	struct folio *folio = page_folio(phys_to_page(paddr));
3385	const u32 blocksize = fs_info->sectorsize;
3386	const u32 step = min(blocksize, PAGE_SIZE);
3387	const u32 nr_steps = blocksize / step;
3388	phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE];
3389
3390	/* The full block must be inside the folio. */
3391	ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio));
3392
3393	for (int i = 0; i < nr_steps; i++) {
3394	u32 pindex = offset_in_folio(folio, paddr + i * step) >> PAGE_SHIFT;
3395
3396	/*
3397	* For bs <= ps cases, we will only run the loop once, so the offset
3398	* inside the page will only added to paddrs[0].
3399	*
3400	* For bs > ps cases, the block must be page aligned, thus offset
3401	* inside the page will always be 0.
3402	*/
3403	paddrs[i] = page_to_phys(folio_page(folio, pindex)) + offset_in_page(paddr);
3404	}
3405	return btrfs_calculate_block_csum_pages(fs_info, paddrs, dest);
3406	}
3407
3408	/*
3409	* Calculate the checksum of a fs block backed by multiple noncontiguous pages
3410	* at @paddrs[] and save the result to @dest.
3411	*
3412	* The folio containing @paddr must be large enough to contain a full fs block.
3413	*/
3414	void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info,
3415	const phys_addr_t paddrs[], u8 *dest)
3416	{
3417	const u32 blocksize = fs_info->sectorsize;
3418	const u32 step = min(blocksize, PAGE_SIZE);
3419	const u32 nr_steps = blocksize / step;
3420	SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
3421
3422	shash->tfm = fs_info->csum_shash;
3423	crypto_shash_init(desc: shash);
3424	for (int i = 0; i < nr_steps; i++) {
3425	const phys_addr_t paddr = paddrs[i];
3426	void *kaddr;
3427
3428	ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE);
3429	kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr);
3430	crypto_shash_update(desc: shash, data: kaddr, len: step);
3431	kunmap_local(kaddr);
3432	}
3433	crypto_shash_final(desc: shash, out: dest);
3434	}
3435
3436	/*
3437	* Verify the checksum for a single sector without any extra action that depend
3438	* on the type of I/O.
3439	*
3440	* @kaddr must be a properly kmapped address.
3441	*/
3442	int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum,
3443	const u8 * const csum_expected)
3444	{
3445	btrfs_calculate_block_csum_folio(fs_info, paddr, dest: csum);
3446	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))
3447	return -EIO;
3448	return 0;
3449	}
3450
3451	/*
3452	* Verify the checksum of a single data sector, which can be scattered at
3453	* different noncontiguous pages.
3454	*
3455	* @bbio: btrfs_io_bio which contains the csum
3456	* @dev: device the sector is on
3457	* @bio_offset: offset to the beginning of the bio (in bytes)
3458	* @paddrs: physical addresses which back the fs block
3459	*
3460	* Check if the checksum on a data block is valid. When a checksum mismatch is
3461	* detected, report the error and fill the corrupted range with zero.
3462	*
3463	* Return %true if the sector is ok or had no checksum to start with, else %false.
3464	*/
3465	bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev,
3466	u32 bio_offset, const phys_addr_t paddrs[])
3467	{
3468	struct btrfs_inode *inode = bbio->inode;
3469	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3470	const u32 blocksize = fs_info->sectorsize;
3471	const u32 step = min(blocksize, PAGE_SIZE);
3472	const u32 nr_steps = blocksize / step;
3473	u64 file_offset = bbio->file_offset + bio_offset;
3474	u64 end = file_offset + blocksize - 1;
3475	u8 *csum_expected;
3476	u8 csum[BTRFS_CSUM_SIZE];
3477
3478	if (!bbio->csum)
3479	return true;
3480
3481	if (btrfs_is_data_reloc_root(root: inode->root) &&
3482	btrfs_test_range_bit(tree: &inode->io_tree, start: file_offset, end, bit: EXTENT_NODATASUM,
3483	NULL)) {
3484	/* Skip the range without csum for data reloc inode */
3485	btrfs_clear_extent_bit(tree: &inode->io_tree, start: file_offset, end,
3486	bits: EXTENT_NODATASUM, NULL);
3487	return true;
3488	}
3489
3490	csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) *
3491	fs_info->csum_size;
3492	btrfs_calculate_block_csum_pages(fs_info, paddrs, dest: csum);
3493	if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0))
3494	goto zeroit;
3495	return true;
3496
3497	zeroit:
3498	btrfs_print_data_csum_error(inode, logical_start: file_offset, csum, csum_expected,
3499	mirror_num: bbio->mirror_num);
3500	if (dev)
3501	btrfs_dev_stat_inc_and_print(dev, index: BTRFS_DEV_STAT_CORRUPTION_ERRS);
3502	for (int i = 0; i < nr_steps; i++)
3503	memzero_page(phys_to_page(paddrs[i]), offset_in_page(paddrs[i]), len: step);
3504	return false;
3505	}
3506
3507	/*
3508	* Perform a delayed iput on @inode.
3509	*
3510	* @inode: The inode we want to perform iput on
3511	*
3512	* This function uses the generic vfs_inode::i_count to track whether we should
3513	* just decrement it (in case it's > 1) or if this is the last iput then link
3514	* the inode to the delayed iput machinery. Delayed iputs are processed at
3515	* transaction commit time/superblock commit/cleaner kthread.
3516	*/
3517	void btrfs_add_delayed_iput(struct btrfs_inode *inode)
3518	{
3519	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3520	unsigned long flags;
3521
3522	if (atomic_add_unless(v: &inode->vfs_inode.i_count, a: -1, u: 1))
3523	return;
3524
3525	WARN_ON_ONCE(test_bit(BTRFS_FS_STATE_NO_DELAYED_IPUT, &fs_info->fs_state));
3526	atomic_inc(v: &fs_info->nr_delayed_iputs);
3527	/*
3528	* Need to be irq safe here because we can be called from either an irq
3529	* context (see bio.c and btrfs_put_ordered_extent()) or a non-irq
3530	* context.
3531	*/
3532	spin_lock_irqsave(&fs_info->delayed_iput_lock, flags);
3533	ASSERT(list_empty(&inode->delayed_iput));
3534	list_add_tail(new: &inode->delayed_iput, head: &fs_info->delayed_iputs);
3535	spin_unlock_irqrestore(lock: &fs_info->delayed_iput_lock, flags);
3536	if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))
3537	wake_up_process(tsk: fs_info->cleaner_kthread);
3538	}
3539
3540	static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,
3541	struct btrfs_inode *inode)
3542	{
3543	list_del_init(entry: &inode->delayed_iput);
3544	spin_unlock_irq(lock: &fs_info->delayed_iput_lock);
3545	iput(&inode->vfs_inode);
3546	if (atomic_dec_and_test(v: &fs_info->nr_delayed_iputs))
3547	wake_up(&fs_info->delayed_iputs_wait);
3548	spin_lock_irq(lock: &fs_info->delayed_iput_lock);
3549	}
3550
3551	static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,
3552	struct btrfs_inode *inode)
3553	{
3554	if (!list_empty(head: &inode->delayed_iput)) {
3555	spin_lock_irq(lock: &fs_info->delayed_iput_lock);
3556	if (!list_empty(head: &inode->delayed_iput))
3557	run_delayed_iput_locked(fs_info, inode);
3558	spin_unlock_irq(lock: &fs_info->delayed_iput_lock);
3559	}
3560	}
3561
3562	void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3563	{
3564	/*
3565	* btrfs_put_ordered_extent() can run in irq context (see bio.c), which
3566	* calls btrfs_add_delayed_iput() and that needs to lock
3567	* fs_info->delayed_iput_lock. So we need to disable irqs here to
3568	* prevent a deadlock.
3569	*/
3570	spin_lock_irq(lock: &fs_info->delayed_iput_lock);
3571	while (!list_empty(head: &fs_info->delayed_iputs)) {
3572	struct btrfs_inode *inode;
3573
3574	inode = list_first_entry(&fs_info->delayed_iputs,
3575	struct btrfs_inode, delayed_iput);
3576	run_delayed_iput_locked(fs_info, inode);
3577	if (need_resched()) {
3578	spin_unlock_irq(lock: &fs_info->delayed_iput_lock);
3579	cond_resched();
3580	spin_lock_irq(lock: &fs_info->delayed_iput_lock);
3581	}
3582	}
3583	spin_unlock_irq(lock: &fs_info->delayed_iput_lock);
3584	}
3585
3586	/*
3587	* Wait for flushing all delayed iputs
3588	*
3589	* @fs_info: the filesystem
3590	*
3591	* This will wait on any delayed iputs that are currently running with KILLABLE
3592	* set. Once they are all done running we will return, unless we are killed in
3593	* which case we return EINTR. This helps in user operations like fallocate etc
3594	* that might get blocked on the iputs.
3595	*
3596	* Return EINTR if we were killed, 0 if nothing's pending
3597	*/
3598	int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)
3599	{
3600	int ret = wait_event_killable(fs_info->delayed_iputs_wait,
3601	atomic_read(&fs_info->nr_delayed_iputs) == 0);
3602	if (ret)
3603	return -EINTR;
3604	return 0;
3605	}
3606
3607	/*
3608	* This creates an orphan entry for the given inode in case something goes wrong
3609	* in the middle of an unlink.
3610	*/
3611	int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3612	struct btrfs_inode *inode)
3613	{
3614	int ret;
3615
3616	ret = btrfs_insert_orphan_item(trans, root: inode->root, offset: btrfs_ino(inode));
3617	if (unlikely(ret && ret != -EEXIST)) {
3618	btrfs_abort_transaction(trans, ret);
3619	return ret;
3620	}
3621
3622	return 0;
3623	}
3624
3625	/*
3626	* We have done the delete so we can go ahead and remove the orphan item for
3627	* this particular inode.
3628	*/
3629	static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3630	struct btrfs_inode *inode)
3631	{
3632	return btrfs_del_orphan_item(trans, root: inode->root, offset: btrfs_ino(inode));
3633	}
3634
3635	/*
3636	* this cleans up any orphans that may be left on the list from the last use
3637	* of this root.
3638	*/
3639	int btrfs_orphan_cleanup(struct btrfs_root *root)
3640	{
3641	struct btrfs_fs_info *fs_info = root->fs_info;
3642	BTRFS_PATH_AUTO_FREE(path);
3643	struct extent_buffer *leaf;
3644	struct btrfs_key key, found_key;
3645	struct btrfs_trans_handle *trans;
3646	u64 last_objectid = 0;
3647	int ret = 0, nr_unlink = 0;
3648
3649	if (test_and_set_bit(nr: BTRFS_ROOT_ORPHAN_CLEANUP, addr: &root->state))
3650	return 0;
3651
3652	path = btrfs_alloc_path();
3653	if (!path) {
3654	ret = -ENOMEM;
3655	goto out;
3656	}
3657	path->reada = READA_BACK;
3658
3659	key.objectid = BTRFS_ORPHAN_OBJECTID;
3660	key.type = BTRFS_ORPHAN_ITEM_KEY;
3661	key.offset = (u64)-1;
3662
3663	while (1) {
3664	struct btrfs_inode *inode;
3665
3666	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
3667	if (ret < 0)
3668	goto out;
3669
3670	/*
3671	* if ret == 0 means we found what we were searching for, which
3672	* is weird, but possible, so only screw with path if we didn't
3673	* find the key and see if we have stuff that matches
3674	*/
3675	if (ret > 0) {
3676	ret = 0;
3677	if (path->slots[0] == 0)
3678	break;
3679	path->slots[0]--;
3680	}
3681
3682	/* pull out the item */
3683	leaf = path->nodes[0];
3684	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
3685
3686	/* make sure the item matches what we want */
3687	if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3688	break;
3689	if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3690	break;
3691
3692	/* release the path since we're done with it */
3693	btrfs_release_path(p: path);
3694
3695	/*
3696	* this is where we are basically btrfs_lookup, without the
3697	* crossing root thing. we store the inode number in the
3698	* offset of the orphan item.
3699	*/
3700
3701	if (found_key.offset == last_objectid) {
3702	/*
3703	* We found the same inode as before. This means we were
3704	* not able to remove its items via eviction triggered
3705	* by an iput(). A transaction abort may have happened,
3706	* due to -ENOSPC for example, so try to grab the error
3707	* that lead to a transaction abort, if any.
3708	*/
3709	btrfs_err(fs_info,
3710	"Error removing orphan entry, stopping orphan cleanup");
3711	ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL;
3712	goto out;
3713	}
3714
3715	last_objectid = found_key.offset;
3716
3717	found_key.objectid = found_key.offset;
3718	found_key.type = BTRFS_INODE_ITEM_KEY;
3719	found_key.offset = 0;
3720	inode = btrfs_iget(ino: last_objectid, root);
3721	if (IS_ERR(ptr: inode)) {
3722	ret = PTR_ERR(ptr: inode);
3723	inode = NULL;
3724	if (ret != -ENOENT)
3725	goto out;
3726	}
3727
3728	if (!inode && root == fs_info->tree_root) {
3729	struct btrfs_root *dead_root;
3730	int is_dead_root = 0;
3731
3732	/*
3733	* This is an orphan in the tree root. Currently these
3734	* could come from 2 sources:
3735	* a) a root (snapshot/subvolume) deletion in progress
3736	* b) a free space cache inode
3737	* We need to distinguish those two, as the orphan item
3738	* for a root must not get deleted before the deletion
3739	* of the snapshot/subvolume's tree completes.
3740	*
3741	* btrfs_find_orphan_roots() ran before us, which has
3742	* found all deleted roots and loaded them into
3743	* fs_info->fs_roots_radix. So here we can find if an
3744	* orphan item corresponds to a deleted root by looking
3745	* up the root from that radix tree.
3746	*/
3747
3748	spin_lock(lock: &fs_info->fs_roots_radix_lock);
3749	dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,
3750	(unsigned long)found_key.objectid);
3751	if (dead_root && btrfs_root_refs(s: &dead_root->root_item) == 0)
3752	is_dead_root = 1;
3753	spin_unlock(lock: &fs_info->fs_roots_radix_lock);
3754
3755	if (is_dead_root) {
3756	/* prevent this orphan from being found again */
3757	key.offset = found_key.objectid - 1;
3758	continue;
3759	}
3760
3761	}
3762
3763	/*
3764	* If we have an inode with links, there are a couple of
3765	* possibilities:
3766	*
3767	* 1. We were halfway through creating fsverity metadata for the
3768	* file. In that case, the orphan item represents incomplete
3769	* fsverity metadata which must be cleaned up with
3770	* btrfs_drop_verity_items and deleting the orphan item.
3771
3772	* 2. Old kernels (before v3.12) used to create an
3773	* orphan item for truncate indicating that there were possibly
3774	* extent items past i_size that needed to be deleted. In v3.12,
3775	* truncate was changed to update i_size in sync with the extent
3776	* items, but the (useless) orphan item was still created. Since
3777	* v4.18, we don't create the orphan item for truncate at all.
3778	*
3779	* So, this item could mean that we need to do a truncate, but
3780	* only if this filesystem was last used on a pre-v3.12 kernel
3781	* and was not cleanly unmounted. The odds of that are quite
3782	* slim, and it's a pain to do the truncate now, so just delete
3783	* the orphan item.
3784	*
3785	* It's also possible that this orphan item was supposed to be
3786	* deleted but wasn't. The inode number may have been reused,
3787	* but either way, we can delete the orphan item.
3788	*/
3789	if (!inode || inode->vfs_inode.i_nlink) {
3790	if (inode) {
3791	ret = btrfs_drop_verity_items(inode);
3792	iput(&inode->vfs_inode);
3793	inode = NULL;
3794	if (ret)
3795	goto out;
3796	}
3797	trans = btrfs_start_transaction(root, num_items: 1);
3798	if (IS_ERR(ptr: trans)) {
3799	ret = PTR_ERR(ptr: trans);
3800	goto out;
3801	}
3802	btrfs_debug(fs_info, "auto deleting %Lu",
3803	found_key.objectid);
3804	ret = btrfs_del_orphan_item(trans, root,
3805	offset: found_key.objectid);
3806	btrfs_end_transaction(trans);
3807	if (ret)
3808	goto out;
3809	continue;
3810	}
3811
3812	nr_unlink++;
3813
3814	/* this will do delete_inode and everything for us */
3815	iput(&inode->vfs_inode);
3816	}
3817	/* release the path since we're done with it */
3818	btrfs_release_path(p: path);
3819
3820	if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3821	trans = btrfs_join_transaction(root);
3822	if (!IS_ERR(ptr: trans))
3823	btrfs_end_transaction(trans);
3824	}
3825
3826	if (nr_unlink)
3827	btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3828
3829	out:
3830	if (ret)
3831	btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3832	return ret;
3833	}
3834
3835	/*
3836	* Look ahead in the leaf for xattrs. If we don't find any then we know there
3837	* can't be any ACLs.
3838	*
3839	* @leaf: the eb leaf where to search
3840	* @slot: the slot the inode is in
3841	* @objectid: the objectid of the inode
3842	*
3843	* Return true if there is xattr/ACL, false otherwise.
3844	*/
3845	static noinline bool acls_after_inode_item(struct extent_buffer *leaf,
3846	int slot, u64 objectid,
3847	int *first_xattr_slot)
3848	{
3849	u32 nritems = btrfs_header_nritems(eb: leaf);
3850	struct btrfs_key found_key;
3851	static u64 xattr_access = 0;
3852	static u64 xattr_default = 0;
3853	int scanned = 0;
3854
3855	if (!xattr_access) {
3856	xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3857	strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3858	xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3859	strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3860	}
3861
3862	slot++;
3863	*first_xattr_slot = -1;
3864	while (slot < nritems) {
3865	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: slot);
3866
3867	/* We found a different objectid, there must be no ACLs. */
3868	if (found_key.objectid != objectid)
3869	return false;
3870
3871	/* We found an xattr, assume we've got an ACL. */
3872	if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3873	if (*first_xattr_slot == -1)
3874	*first_xattr_slot = slot;
3875	if (found_key.offset == xattr_access ||
3876	found_key.offset == xattr_default)
3877	return true;
3878	}
3879
3880	/*
3881	* We found a key greater than an xattr key, there can't be any
3882	* ACLs later on.
3883	*/
3884	if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3885	return false;
3886
3887	slot++;
3888	scanned++;
3889
3890	/*
3891	* The item order goes like:
3892	* - inode
3893	* - inode backrefs
3894	* - xattrs
3895	* - extents,
3896	*
3897	* so if there are lots of hard links to an inode there can be
3898	* a lot of backrefs. Don't waste time searching too hard,
3899	* this is just an optimization.
3900	*/
3901	if (scanned >= 8)
3902	break;
3903	}
3904	/*
3905	* We hit the end of the leaf before we found an xattr or something
3906	* larger than an xattr. We have to assume the inode has ACLs.
3907	*/
3908	if (*first_xattr_slot == -1)
3909	*first_xattr_slot = slot;
3910	return true;
3911	}
3912
3913	static int btrfs_init_file_extent_tree(struct btrfs_inode *inode)
3914	{
3915	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3916
3917	if (WARN_ON_ONCE(inode->file_extent_tree))
3918	return 0;
3919	if (btrfs_fs_incompat(fs_info, NO_HOLES))
3920	return 0;
3921	if (!S_ISREG(inode->vfs_inode.i_mode))
3922	return 0;
3923	if (btrfs_is_free_space_inode(inode))
3924	return 0;
3925
3926	inode->file_extent_tree = kmalloc(sizeof(struct extent_io_tree), GFP_KERNEL);
3927	if (!inode->file_extent_tree)
3928	return -ENOMEM;
3929
3930	btrfs_extent_io_tree_init(fs_info, tree: inode->file_extent_tree,
3931	owner: IO_TREE_INODE_FILE_EXTENT);
3932	/* Lockdep class is set only for the file extent tree. */
3933	lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class);
3934
3935	return 0;
3936	}
3937
3938	static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc)
3939	{
3940	struct btrfs_root *root = inode->root;
3941	struct btrfs_inode *existing;
3942	const u64 ino = btrfs_ino(inode);
3943	int ret;
3944
3945	if (inode_unhashed(inode: &inode->vfs_inode))
3946	return 0;
3947
3948	if (prealloc) {
3949	ret = xa_reserve(xa: &root->inodes, index: ino, GFP_NOFS);
3950	if (ret)
3951	return ret;
3952	}
3953
3954	existing = xa_store(&root->inodes, index: ino, entry: inode, GFP_ATOMIC);
3955
3956	if (xa_is_err(entry: existing)) {
3957	ret = xa_err(entry: existing);
3958	ASSERT(ret != -EINVAL);
3959	ASSERT(ret != -ENOMEM);
3960	return ret;
3961	} else if (existing) {
3962	WARN_ON(!(inode_state_read_once(&existing->vfs_inode) & (I_WILL_FREE | I_FREEING)));
3963	}
3964
3965	return 0;
3966	}
3967
3968	/*
3969	* Read a locked inode from the btree into the in-memory inode and add it to
3970	* its root list/tree.
3971	*
3972	* On failure clean up the inode.
3973	*/
3974	static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path *path)
3975	{
3976	struct btrfs_root *root = inode->root;
3977	struct btrfs_fs_info *fs_info = root->fs_info;
3978	struct extent_buffer *leaf;
3979	struct btrfs_inode_item *inode_item;
3980	struct inode *vfs_inode = &inode->vfs_inode;
3981	struct btrfs_key location;
3982	unsigned long ptr;
3983	int maybe_acls;
3984	u32 rdev;
3985	int ret;
3986	bool filled = false;
3987	int first_xattr_slot;
3988
3989	ret = btrfs_fill_inode(inode, rdev: &rdev);
3990	if (!ret)
3991	filled = true;
3992
3993	ASSERT(path);
3994
3995	btrfs_get_inode_key(inode, key: &location);
3996
3997	ret = btrfs_lookup_inode(NULL, root, path, location: &location, mod: 0);
3998	if (ret) {
3999	/*
4000	* ret > 0 can come from btrfs_search_slot called by
4001	* btrfs_lookup_inode(), this means the inode was not found.
4002	*/
4003	if (ret > 0)
4004	ret = -ENOENT;
4005	goto out;
4006	}
4007
4008	leaf = path->nodes[0];
4009
4010	if (filled)
4011	goto cache_index;
4012
4013	inode_item = btrfs_item_ptr(leaf, path->slots[0],
4014	struct btrfs_inode_item);
4015	vfs_inode->i_mode = btrfs_inode_mode(eb: leaf, s: inode_item);
4016	set_nlink(inode: vfs_inode, nlink: btrfs_inode_nlink(eb: leaf, s: inode_item));
4017	i_uid_write(inode: vfs_inode, uid: btrfs_inode_uid(eb: leaf, s: inode_item));
4018	i_gid_write(inode: vfs_inode, gid: btrfs_inode_gid(eb: leaf, s: inode_item));
4019	btrfs_i_size_write(inode, size: btrfs_inode_size(eb: leaf, s: inode_item));
4020
4021	inode_set_atime(inode: vfs_inode, sec: btrfs_timespec_sec(eb: leaf, s: &inode_item->atime),
4022	nsec: btrfs_timespec_nsec(eb: leaf, s: &inode_item->atime));
4023
4024	inode_set_mtime(inode: vfs_inode, sec: btrfs_timespec_sec(eb: leaf, s: &inode_item->mtime),
4025	nsec: btrfs_timespec_nsec(eb: leaf, s: &inode_item->mtime));
4026
4027	inode_set_ctime(inode: vfs_inode, sec: btrfs_timespec_sec(eb: leaf, s: &inode_item->ctime),
4028	nsec: btrfs_timespec_nsec(eb: leaf, s: &inode_item->ctime));
4029
4030	inode->i_otime_sec = btrfs_timespec_sec(eb: leaf, s: &inode_item->otime);
4031	inode->i_otime_nsec = btrfs_timespec_nsec(eb: leaf, s: &inode_item->otime);
4032
4033	inode_set_bytes(inode: vfs_inode, bytes: btrfs_inode_nbytes(eb: leaf, s: inode_item));
4034	inode->generation = btrfs_inode_generation(eb: leaf, s: inode_item);
4035	inode->last_trans = btrfs_inode_transid(eb: leaf, s: inode_item);
4036
4037	inode_set_iversion_queried(inode: vfs_inode, val: btrfs_inode_sequence(eb: leaf, s: inode_item));
4038	vfs_inode->i_generation = inode->generation;
4039	vfs_inode->i_rdev = 0;
4040	rdev = btrfs_inode_rdev(eb: leaf, s: inode_item);
4041
4042	if (S_ISDIR(vfs_inode->i_mode))
4043	inode->index_cnt = (u64)-1;
4044
4045	btrfs_inode_split_flags(inode_item_flags: btrfs_inode_flags(eb: leaf, s: inode_item),
4046	flags: &inode->flags, ro_flags: &inode->ro_flags);
4047	btrfs_update_inode_mapping_flags(inode);
4048	btrfs_set_inode_mapping_order(inode);
4049
4050	cache_index:
4051	/*
4052	* If we were modified in the current generation and evicted from memory
4053	* and then re-read we need to do a full sync since we don't have any
4054	* idea about which extents were modified before we were evicted from
4055	* cache.
4056	*
4057	* This is required for both inode re-read from disk and delayed inode
4058	* in the delayed_nodes xarray.
4059	*/
4060	if (inode->last_trans == btrfs_get_fs_generation(fs_info))
4061	set_bit(nr: BTRFS_INODE_NEEDS_FULL_SYNC, addr: &inode->runtime_flags);
4062
4063	/*
4064	* We don't persist the id of the transaction where an unlink operation
4065	* against the inode was last made. So here we assume the inode might
4066	* have been evicted, and therefore the exact value of last_unlink_trans
4067	* lost, and set it to last_trans to avoid metadata inconsistencies
4068	* between the inode and its parent if the inode is fsync'ed and the log
4069	* replayed. For example, in the scenario:
4070	*
4071	* touch mydir/foo
4072	* ln mydir/foo mydir/bar
4073	* sync
4074	* unlink mydir/bar
4075	* echo 2 > /proc/sys/vm/drop_caches # evicts inode
4076	* xfs_io -c fsync mydir/foo
4077	* <power failure>
4078	* mount fs, triggers fsync log replay
4079	*
4080	* We must make sure that when we fsync our inode foo we also log its
4081	* parent inode, otherwise after log replay the parent still has the
4082	* dentry with the "bar" name but our inode foo has a link count of 1
4083	* and doesn't have an inode ref with the name "bar" anymore.
4084	*
4085	* Setting last_unlink_trans to last_trans is a pessimistic approach,
4086	* but it guarantees correctness at the expense of occasional full
4087	* transaction commits on fsync if our inode is a directory, or if our
4088	* inode is not a directory, logging its parent unnecessarily.
4089	*/
4090	inode->last_unlink_trans = inode->last_trans;
4091
4092	/*
4093	* Same logic as for last_unlink_trans. We don't persist the generation
4094	* of the last transaction where this inode was used for a reflink
4095	* operation, so after eviction and reloading the inode we must be
4096	* pessimistic and assume the last transaction that modified the inode.
4097	*/
4098	inode->last_reflink_trans = inode->last_trans;
4099
4100	path->slots[0]++;
4101	if (vfs_inode->i_nlink != 1 ||
4102	path->slots[0] >= btrfs_header_nritems(eb: leaf))
4103	goto cache_acl;
4104
4105	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &location, nr: path->slots[0]);
4106	if (location.objectid != btrfs_ino(inode))
4107	goto cache_acl;
4108
4109	ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
4110	if (location.type == BTRFS_INODE_REF_KEY) {
4111	struct btrfs_inode_ref *ref;
4112
4113	ref = (struct btrfs_inode_ref *)ptr;
4114	inode->dir_index = btrfs_inode_ref_index(eb: leaf, s: ref);
4115	} else if (location.type == BTRFS_INODE_EXTREF_KEY) {
4116	struct btrfs_inode_extref *extref;
4117
4118	extref = (struct btrfs_inode_extref *)ptr;
4119	inode->dir_index = btrfs_inode_extref_index(eb: leaf, s: extref);
4120	}
4121	cache_acl:
4122	/*
4123	* try to precache a NULL acl entry for files that don't have
4124	* any xattrs or acls
4125	*/
4126	maybe_acls = acls_after_inode_item(leaf, slot: path->slots[0],
4127	objectid: btrfs_ino(inode), first_xattr_slot: &first_xattr_slot);
4128	if (first_xattr_slot != -1) {
4129	path->slots[0] = first_xattr_slot;
4130	ret = btrfs_load_inode_props(inode, path);
4131	if (ret)
4132	btrfs_err(fs_info,
4133	"error loading props for ino %llu (root %llu): %d",
4134	btrfs_ino(inode), btrfs_root_id(root), ret);
4135	}
4136
4137	/*
4138	* We don't need the path anymore, so release it to avoid holding a read
4139	* lock on a leaf while calling btrfs_init_file_extent_tree(), which can
4140	* allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking
4141	* dependency.
4142	*/
4143	btrfs_release_path(p: path);
4144
4145	ret = btrfs_init_file_extent_tree(inode);
4146	if (ret)
4147	goto out;
4148	btrfs_inode_set_file_extent_range(inode, start: 0,
4149	round_up(i_size_read(vfs_inode), fs_info->sectorsize));
4150
4151	if (!maybe_acls)
4152	cache_no_acl(inode: vfs_inode);
4153
4154	switch (vfs_inode->i_mode & S_IFMT) {
4155	case S_IFREG:
4156	vfs_inode->i_mapping->a_ops = &btrfs_aops;
4157	vfs_inode->i_fop = &btrfs_file_operations;
4158	vfs_inode->i_op = &btrfs_file_inode_operations;
4159	break;
4160	case S_IFDIR:
4161	vfs_inode->i_fop = &btrfs_dir_file_operations;
4162	vfs_inode->i_op = &btrfs_dir_inode_operations;
4163	break;
4164	case S_IFLNK:
4165	vfs_inode->i_op = &btrfs_symlink_inode_operations;
4166	inode_nohighmem(inode: vfs_inode);
4167	vfs_inode->i_mapping->a_ops = &btrfs_aops;
4168	break;
4169	default:
4170	vfs_inode->i_op = &btrfs_special_inode_operations;
4171	init_special_inode(vfs_inode, vfs_inode->i_mode, rdev);
4172	break;
4173	}
4174
4175	btrfs_sync_inode_flags_to_i_flags(inode);
4176
4177	ret = btrfs_add_inode_to_root(inode, prealloc: true);
4178	if (ret)
4179	goto out;
4180
4181	return 0;
4182	out:
4183	/*
4184	* We may have a read locked leaf and iget_failed() triggers inode
4185	* eviction which needs to release the delayed inode and that needs
4186	* to lock the delayed inode's mutex. This can cause a ABBA deadlock
4187	* with a task running delayed items, as that require first locking
4188	* the delayed inode's mutex and then modifying its subvolume btree.
4189	* So release the path before iget_failed().
4190	*/
4191	btrfs_release_path(p: path);
4192	iget_failed(vfs_inode);
4193	return ret;
4194	}
4195
4196	/*
4197	* given a leaf and an inode, copy the inode fields into the leaf
4198	*/
4199	static void fill_inode_item(struct btrfs_trans_handle *trans,
4200	struct extent_buffer *leaf,
4201	struct btrfs_inode_item *item,
4202	struct inode *inode)
4203	{
4204	u64 flags;
4205
4206	btrfs_set_inode_uid(eb: leaf, s: item, val: i_uid_read(inode));
4207	btrfs_set_inode_gid(eb: leaf, s: item, val: i_gid_read(inode));
4208	btrfs_set_inode_size(eb: leaf, s: item, BTRFS_I(inode)->disk_i_size);
4209	btrfs_set_inode_mode(eb: leaf, s: item, val: inode->i_mode);
4210	btrfs_set_inode_nlink(eb: leaf, s: item, val: inode->i_nlink);
4211
4212	btrfs_set_timespec_sec(eb: leaf, s: &item->atime, val: inode_get_atime_sec(inode));
4213	btrfs_set_timespec_nsec(eb: leaf, s: &item->atime, val: inode_get_atime_nsec(inode));
4214
4215	btrfs_set_timespec_sec(eb: leaf, s: &item->mtime, val: inode_get_mtime_sec(inode));
4216	btrfs_set_timespec_nsec(eb: leaf, s: &item->mtime, val: inode_get_mtime_nsec(inode));
4217
4218	btrfs_set_timespec_sec(eb: leaf, s: &item->ctime, val: inode_get_ctime_sec(inode));
4219	btrfs_set_timespec_nsec(eb: leaf, s: &item->ctime, val: inode_get_ctime_nsec(inode));
4220
4221	btrfs_set_timespec_sec(eb: leaf, s: &item->otime, BTRFS_I(inode)->i_otime_sec);
4222	btrfs_set_timespec_nsec(eb: leaf, s: &item->otime, BTRFS_I(inode)->i_otime_nsec);
4223
4224	btrfs_set_inode_nbytes(eb: leaf, s: item, val: inode_get_bytes(inode));
4225	btrfs_set_inode_generation(eb: leaf, s: item, BTRFS_I(inode)->generation);
4226	btrfs_set_inode_sequence(eb: leaf, s: item, val: inode_peek_iversion(inode));
4227	btrfs_set_inode_transid(eb: leaf, s: item, val: trans->transid);
4228	btrfs_set_inode_rdev(eb: leaf, s: item, val: inode->i_rdev);
4229	flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags,
4230	BTRFS_I(inode)->ro_flags);
4231	btrfs_set_inode_flags(eb: leaf, s: item, val: flags);
4232	btrfs_set_inode_block_group(eb: leaf, s: item, val: 0);
4233	}
4234
4235	/*
4236	* copy everything in the in-memory inode into the btree.
4237	*/
4238	static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
4239	struct btrfs_inode *inode)
4240	{
4241	struct btrfs_inode_item *inode_item;
4242	BTRFS_PATH_AUTO_FREE(path);
4243	struct extent_buffer *leaf;
4244	struct btrfs_key key;
4245	int ret;
4246
4247	path = btrfs_alloc_path();
4248	if (!path)
4249	return -ENOMEM;
4250
4251	btrfs_get_inode_key(inode, key: &key);
4252	ret = btrfs_lookup_inode(trans, root: inode->root, path, location: &key, mod: 1);
4253	if (ret) {
4254	if (ret > 0)
4255	ret = -ENOENT;
4256	return ret;
4257	}
4258
4259	leaf = path->nodes[0];
4260	inode_item = btrfs_item_ptr(leaf, path->slots[0],
4261	struct btrfs_inode_item);
4262
4263	fill_inode_item(trans, leaf, item: inode_item, inode: &inode->vfs_inode);
4264	btrfs_set_inode_last_trans(trans, inode);
4265	return 0;
4266	}
4267
4268	/*
4269	* copy everything in the in-memory inode into the btree.
4270	*/
4271	int btrfs_update_inode(struct btrfs_trans_handle *trans,
4272	struct btrfs_inode *inode)
4273	{
4274	struct btrfs_root *root = inode->root;
4275	struct btrfs_fs_info *fs_info = root->fs_info;
4276	int ret;
4277
4278	/*
4279	* If the inode is a free space inode, we can deadlock during commit
4280	* if we put it into the delayed code.
4281	*
4282	* The data relocation inode should also be directly updated
4283	* without delay
4284	*/
4285	if (!btrfs_is_free_space_inode(inode)
4286	&& !btrfs_is_data_reloc_root(root)
4287	&& !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4288	btrfs_update_root_times(trans, root);
4289
4290	ret = btrfs_delayed_update_inode(trans, inode);
4291	if (!ret)
4292	btrfs_set_inode_last_trans(trans, inode);
4293	return ret;
4294	}
4295
4296	return btrfs_update_inode_item(trans, inode);
4297	}
4298
4299	int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4300	struct btrfs_inode *inode)
4301	{
4302	int ret;
4303
4304	ret = btrfs_update_inode(trans, inode);
4305	if (ret == -ENOSPC)
4306	return btrfs_update_inode_item(trans, inode);
4307	return ret;
4308	}
4309
4310	static void update_time_after_link_or_unlink(struct btrfs_inode *dir)
4311	{
4312	struct timespec64 now;
4313
4314	/*
4315	* If we are replaying a log tree, we do not want to update the mtime
4316	* and ctime of the parent directory with the current time, since the
4317	* log replay procedure is responsible for setting them to their correct
4318	* values (the ones it had when the fsync was done).
4319	*/
4320	if (test_bit(BTRFS_FS_LOG_RECOVERING, &dir->root->fs_info->flags))
4321	return;
4322
4323	now = inode_set_ctime_current(inode: &dir->vfs_inode);
4324	inode_set_mtime_to_ts(inode: &dir->vfs_inode, ts: now);
4325	}
4326
4327	/*
4328	* unlink helper that gets used here in inode.c and in the tree logging
4329	* recovery code. It remove a link in a directory with a given name, and
4330	* also drops the back refs in the inode to the directory
4331	*/
4332	static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4333	struct btrfs_inode *dir,
4334	struct btrfs_inode *inode,
4335	const struct fscrypt_str *name,
4336	struct btrfs_rename_ctx *rename_ctx)
4337	{
4338	struct btrfs_root *root = dir->root;
4339	struct btrfs_fs_info *fs_info = root->fs_info;
4340	struct btrfs_path *path;
4341	int ret = 0;
4342	struct btrfs_dir_item *di;
4343	u64 index;
4344	u64 ino = btrfs_ino(inode);
4345	u64 dir_ino = btrfs_ino(inode: dir);
4346
4347	path = btrfs_alloc_path();
4348	if (!path)
4349	return -ENOMEM;
4350
4351	di = btrfs_lookup_dir_item(trans, root, path, dir: dir_ino, name, mod: -1);
4352	if (IS_ERR_OR_NULL(ptr: di)) {
4353	btrfs_free_path(p: path);
4354	return di ? PTR_ERR(ptr: di) : -ENOENT;
4355	}
4356	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4357	/*
4358	* Down the call chains below we'll also need to allocate a path, so no
4359	* need to hold on to this one for longer than necessary.
4360	*/
4361	btrfs_free_path(p: path);
4362	if (ret)
4363	return ret;
4364
4365	/*
4366	* If we don't have dir index, we have to get it by looking up
4367	* the inode ref, since we get the inode ref, remove it directly,
4368	* it is unnecessary to do delayed deletion.
4369	*
4370	* But if we have dir index, needn't search inode ref to get it.
4371	* Since the inode ref is close to the inode item, it is better
4372	* that we delay to delete it, and just do this deletion when
4373	* we update the inode item.
4374	*/
4375	if (inode->dir_index) {
4376	ret = btrfs_delayed_delete_inode_ref(inode);
4377	if (!ret) {
4378	index = inode->dir_index;
4379	goto skip_backref;
4380	}
4381	}
4382
4383	ret = btrfs_del_inode_ref(trans, root, name, inode_objectid: ino, ref_objectid: dir_ino, index: &index);
4384	if (unlikely(ret)) {
4385	btrfs_crit(fs_info,
4386	"failed to delete reference to %.*s, root %llu inode %llu parent %llu",
4387	name->len, name->name, btrfs_root_id(root), ino, dir_ino);
4388	btrfs_abort_transaction(trans, ret);
4389	return ret;
4390	}
4391	skip_backref:
4392	if (rename_ctx)
4393	rename_ctx->index = index;
4394
4395	ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4396	if (unlikely(ret)) {
4397	btrfs_abort_transaction(trans, ret);
4398	return ret;
4399	}
4400
4401	/*
4402	* If we are in a rename context, we don't need to update anything in the
4403	* log. That will be done later during the rename by btrfs_log_new_name().
4404	* Besides that, doing it here would only cause extra unnecessary btree
4405	* operations on the log tree, increasing latency for applications.
4406	*/
4407	if (!rename_ctx) {
4408	btrfs_del_inode_ref_in_log(trans, name, inode, dir);
4409	btrfs_del_dir_entries_in_log(trans, name, dir, index);
4410	}
4411
4412	/*
4413	* If we have a pending delayed iput we could end up with the final iput
4414	* being run in btrfs-cleaner context. If we have enough of these built
4415	* up we can end up burning a lot of time in btrfs-cleaner without any
4416	* way to throttle the unlinks. Since we're currently holding a ref on
4417	* the inode we can run the delayed iput here without any issues as the
4418	* final iput won't be done until after we drop the ref we're currently
4419	* holding.
4420	*/
4421	btrfs_run_delayed_iput(fs_info, inode);
4422
4423	btrfs_i_size_write(inode: dir, size: dir->vfs_inode.i_size - name->len * 2);
4424	inode_inc_iversion(inode: &inode->vfs_inode);
4425	inode_set_ctime_current(inode: &inode->vfs_inode);
4426	inode_inc_iversion(inode: &dir->vfs_inode);
4427	update_time_after_link_or_unlink(dir);
4428
4429	return btrfs_update_inode(trans, inode: dir);
4430	}
4431
4432	int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4433	struct btrfs_inode *dir, struct btrfs_inode *inode,
4434	const struct fscrypt_str *name)
4435	{
4436	int ret;
4437
4438	ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL);
4439	if (!ret) {
4440	drop_nlink(inode: &inode->vfs_inode);
4441	ret = btrfs_update_inode(trans, inode);
4442	}
4443	return ret;
4444	}
4445
4446	/*
4447	* helper to start transaction for unlink and rmdir.
4448	*
4449	* unlink and rmdir are special in btrfs, they do not always free space, so
4450	* if we cannot make our reservations the normal way try and see if there is
4451	* plenty of slack room in the global reserve to migrate, otherwise we cannot
4452	* allow the unlink to occur.
4453	*/
4454	static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir)
4455	{
4456	struct btrfs_root *root = dir->root;
4457
4458	return btrfs_start_transaction_fallback_global_rsv(root,
4459	BTRFS_UNLINK_METADATA_UNITS);
4460	}
4461
4462	static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4463	{
4464	struct btrfs_trans_handle *trans;
4465	struct inode *inode = d_inode(dentry);
4466	int ret;
4467	struct fscrypt_name fname;
4468
4469	ret = fscrypt_setup_filename(inode: dir, iname: &dentry->d_name, lookup: 1, fname: &fname);
4470	if (ret)
4471	return ret;
4472
4473	/* This needs to handle no-key deletions later on */
4474
4475	trans = __unlink_start_trans(BTRFS_I(dir));
4476	if (IS_ERR(ptr: trans)) {
4477	ret = PTR_ERR(ptr: trans);
4478	goto fscrypt_free;
4479	}
4480
4481	btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4482	for_rename: false);
4483
4484	ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4485	name: &fname.disk_name);
4486	if (ret)
4487	goto end_trans;
4488
4489	if (inode->i_nlink == 0) {
4490	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4491	if (ret)
4492	goto end_trans;
4493	}
4494
4495	end_trans:
4496	btrfs_end_transaction(trans);
4497	btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info);
4498	fscrypt_free:
4499	fscrypt_free_filename(fname: &fname);
4500	return ret;
4501	}
4502
4503	static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4504	struct btrfs_inode *dir, struct dentry *dentry)
4505	{
4506	struct btrfs_root *root = dir->root;
4507	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4508	BTRFS_PATH_AUTO_FREE(path);
4509	struct extent_buffer *leaf;
4510	struct btrfs_dir_item *di;
4511	struct btrfs_key key;
4512	u64 index;
4513	int ret;
4514	u64 objectid;
4515	u64 dir_ino = btrfs_ino(inode: dir);
4516	struct fscrypt_name fname;
4517
4518	ret = fscrypt_setup_filename(inode: &dir->vfs_inode, iname: &dentry->d_name, lookup: 1, fname: &fname);
4519	if (ret)
4520	return ret;
4521
4522	/* This needs to handle no-key deletions later on */
4523
4524	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4525	objectid = btrfs_root_id(root: inode->root);
4526	} else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4527	objectid = inode->ref_root_id;
4528	} else {
4529	WARN_ON(1);
4530	fscrypt_free_filename(fname: &fname);
4531	return -EINVAL;
4532	}
4533
4534	path = btrfs_alloc_path();
4535	if (!path) {
4536	ret = -ENOMEM;
4537	goto out;
4538	}
4539
4540	di = btrfs_lookup_dir_item(trans, root, path, dir: dir_ino,
4541	name: &fname.disk_name, mod: -1);
4542	if (IS_ERR_OR_NULL(ptr: di)) {
4543	ret = di ? PTR_ERR(ptr: di) : -ENOENT;
4544	goto out;
4545	}
4546
4547	leaf = path->nodes[0];
4548	btrfs_dir_item_key_to_cpu(eb: leaf, item: di, cpu_key: &key);
4549	WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4550	ret = btrfs_delete_one_dir_name(trans, root, path, di);
4551	if (unlikely(ret)) {
4552	btrfs_abort_transaction(trans, ret);
4553	goto out;
4554	}
4555	btrfs_release_path(p: path);
4556
4557	/*
4558	* This is a placeholder inode for a subvolume we didn't have a
4559	* reference to at the time of the snapshot creation. In the meantime
4560	* we could have renamed the real subvol link into our snapshot, so
4561	* depending on btrfs_del_root_ref to return -ENOENT here is incorrect.
4562	* Instead simply lookup the dir_index_item for this entry so we can
4563	* remove it. Otherwise we know we have a ref to the root and we can
4564	* call btrfs_del_root_ref, and it _shouldn't_ fail.
4565	*/
4566	if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
4567	di = btrfs_search_dir_index_item(root, path, dirid: dir_ino, name: &fname.disk_name);
4568	if (IS_ERR(ptr: di)) {
4569	ret = PTR_ERR(ptr: di);
4570	btrfs_abort_transaction(trans, ret);
4571	goto out;
4572	}
4573
4574	leaf = path->nodes[0];
4575	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
4576	index = key.offset;
4577	btrfs_release_path(p: path);
4578	} else {
4579	ret = btrfs_del_root_ref(trans, root_id: objectid,
4580	ref_id: btrfs_root_id(root), dirid: dir_ino,
4581	sequence: &index, name: &fname.disk_name);
4582	if (unlikely(ret)) {
4583	btrfs_abort_transaction(trans, ret);
4584	goto out;
4585	}
4586	}
4587
4588	ret = btrfs_delete_delayed_dir_index(trans, dir, index);
4589	if (unlikely(ret)) {
4590	btrfs_abort_transaction(trans, ret);
4591	goto out;
4592	}
4593
4594	btrfs_i_size_write(inode: dir, size: dir->vfs_inode.i_size - fname.disk_name.len * 2);
4595	inode_inc_iversion(inode: &dir->vfs_inode);
4596	inode_set_mtime_to_ts(inode: &dir->vfs_inode, ts: inode_set_ctime_current(inode: &dir->vfs_inode));
4597	ret = btrfs_update_inode_fallback(trans, inode: dir);
4598	if (ret)
4599	btrfs_abort_transaction(trans, ret);
4600	out:
4601	fscrypt_free_filename(fname: &fname);
4602	return ret;
4603	}
4604
4605	/*
4606	* Helper to check if the subvolume references other subvolumes or if it's
4607	* default.
4608	*/
4609	static noinline int may_destroy_subvol(struct btrfs_root *root)
4610	{
4611	struct btrfs_fs_info *fs_info = root->fs_info;
4612	BTRFS_PATH_AUTO_FREE(path);
4613	struct btrfs_dir_item *di;
4614	struct btrfs_key key;
4615	struct fscrypt_str name = FSTR_INIT("default", 7);
4616	u64 dir_id;
4617	int ret;
4618
4619	path = btrfs_alloc_path();
4620	if (!path)
4621	return -ENOMEM;
4622
4623	/* Make sure this root isn't set as the default subvol */
4624	dir_id = btrfs_super_root_dir(s: fs_info->super_copy);
4625	di = btrfs_lookup_dir_item(NULL, root: fs_info->tree_root, path,
4626	dir: dir_id, name: &name, mod: 0);
4627	if (di && !IS_ERR(ptr: di)) {
4628	btrfs_dir_item_key_to_cpu(eb: path->nodes[0], item: di, cpu_key: &key);
4629	if (key.objectid == btrfs_root_id(root)) {
4630	ret = -EPERM;
4631	btrfs_err(fs_info,
4632	"deleting default subvolume %llu is not allowed",
4633	key.objectid);
4634	return ret;
4635	}
4636	btrfs_release_path(p: path);
4637	}
4638
4639	key.objectid = btrfs_root_id(root);
4640	key.type = BTRFS_ROOT_REF_KEY;
4641	key.offset = (u64)-1;
4642
4643	ret = btrfs_search_slot(NULL, root: fs_info->tree_root, key: &key, p: path, ins_len: 0, cow: 0);
4644	if (ret < 0)
4645	return ret;
4646	if (unlikely(ret == 0)) {
4647	/*
4648	* Key with offset -1 found, there would have to exist a root
4649	* with such id, but this is out of valid range.
4650	*/
4651	return -EUCLEAN;
4652	}
4653
4654	ret = 0;
4655	if (path->slots[0] > 0) {
4656	path->slots[0]--;
4657	btrfs_item_key_to_cpu(eb: path->nodes[0], cpu_key: &key, nr: path->slots[0]);
4658	if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY)
4659	ret = -ENOTEMPTY;
4660	}
4661
4662	return ret;
4663	}
4664
4665	/* Delete all dentries for inodes belonging to the root */
4666	static void btrfs_prune_dentries(struct btrfs_root *root)
4667	{
4668	struct btrfs_fs_info *fs_info = root->fs_info;
4669	struct btrfs_inode *inode;
4670	u64 min_ino = 0;
4671
4672	if (!BTRFS_FS_ERROR(fs_info))
4673	WARN_ON(btrfs_root_refs(&root->root_item) != 0);
4674
4675	inode = btrfs_find_first_inode(root, min_ino);
4676	while (inode) {
4677	if (icount_read(inode: &inode->vfs_inode) > 1)
4678	d_prune_aliases(&inode->vfs_inode);
4679
4680	min_ino = btrfs_ino(inode) + 1;
4681	/*
4682	* btrfs_drop_inode() will have it removed from the inode
4683	* cache when its usage count hits zero.
4684	*/
4685	iput(&inode->vfs_inode);
4686	cond_resched();
4687	inode = btrfs_find_first_inode(root, min_ino);
4688	}
4689	}
4690
4691	int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry)
4692	{
4693	struct btrfs_root *root = dir->root;
4694	struct btrfs_fs_info *fs_info = root->fs_info;
4695	struct inode *inode = d_inode(dentry);
4696	struct btrfs_root *dest = BTRFS_I(inode)->root;
4697	struct btrfs_trans_handle *trans;
4698	struct btrfs_block_rsv block_rsv;
4699	u64 root_flags;
4700	u64 qgroup_reserved = 0;
4701	int ret;
4702
4703	down_write(sem: &fs_info->subvol_sem);
4704
4705	/*
4706	* Don't allow to delete a subvolume with send in progress. This is
4707	* inside the inode lock so the error handling that has to drop the bit
4708	* again is not run concurrently.
4709	*/
4710	spin_lock(lock: &dest->root_item_lock);
4711	if (dest->send_in_progress) {
4712	spin_unlock(lock: &dest->root_item_lock);
4713	btrfs_warn(fs_info,
4714	"attempt to delete subvolume %llu during send",
4715	btrfs_root_id(dest));
4716	ret = -EPERM;
4717	goto out_up_write;
4718	}
4719	if (atomic_read(v: &dest->nr_swapfiles)) {
4720	spin_unlock(lock: &dest->root_item_lock);
4721	btrfs_warn(fs_info,
4722	"attempt to delete subvolume %llu with active swapfile",
4723	btrfs_root_id(root));
4724	ret = -EPERM;
4725	goto out_up_write;
4726	}
4727	root_flags = btrfs_root_flags(s: &dest->root_item);
4728	btrfs_set_root_flags(s: &dest->root_item,
4729	val: root_flags | BTRFS_ROOT_SUBVOL_DEAD);
4730	spin_unlock(lock: &dest->root_item_lock);
4731
4732	ret = may_destroy_subvol(root: dest);
4733	if (ret)
4734	goto out_undead;
4735
4736	btrfs_init_block_rsv(rsv: &block_rsv, type: BTRFS_BLOCK_RSV_TEMP);
4737	/*
4738	* One for dir inode,
4739	* two for dir entries,
4740	* two for root ref/backref.
4741	*/
4742	ret = btrfs_subvolume_reserve_metadata(root, rsv: &block_rsv, nitems: 5, use_global_rsv: true);
4743	if (ret)
4744	goto out_undead;
4745	qgroup_reserved = block_rsv.qgroup_rsv_reserved;
4746
4747	trans = btrfs_start_transaction(root, num_items: 0);
4748	if (IS_ERR(ptr: trans)) {
4749	ret = PTR_ERR(ptr: trans);
4750	goto out_release;
4751	}
4752	btrfs_qgroup_convert_reserved_meta(root, num_bytes: qgroup_reserved);
4753	qgroup_reserved = 0;
4754	trans->block_rsv = &block_rsv;
4755	trans->bytes_reserved = block_rsv.size;
4756
4757	btrfs_record_snapshot_destroy(trans, dir);
4758
4759	ret = btrfs_unlink_subvol(trans, dir, dentry);
4760	if (unlikely(ret)) {
4761	btrfs_abort_transaction(trans, ret);
4762	goto out_end_trans;
4763	}
4764
4765	ret = btrfs_record_root_in_trans(trans, root: dest);
4766	if (unlikely(ret)) {
4767	btrfs_abort_transaction(trans, ret);
4768	goto out_end_trans;
4769	}
4770
4771	memset(&dest->root_item.drop_progress, 0,
4772	sizeof(dest->root_item.drop_progress));
4773	btrfs_set_root_drop_level(s: &dest->root_item, val: 0);
4774	btrfs_set_root_refs(s: &dest->root_item, val: 0);
4775
4776	if (!test_and_set_bit(nr: BTRFS_ROOT_ORPHAN_ITEM_INSERTED, addr: &dest->state)) {
4777	ret = btrfs_insert_orphan_item(trans,
4778	root: fs_info->tree_root,
4779	offset: btrfs_root_id(root: dest));
4780	if (unlikely(ret)) {
4781	btrfs_abort_transaction(trans, ret);
4782	goto out_end_trans;
4783	}
4784	}
4785
4786	ret = btrfs_uuid_tree_remove(trans, uuid: dest->root_item.uuid,
4787	BTRFS_UUID_KEY_SUBVOL, subid: btrfs_root_id(root: dest));
4788	if (unlikely(ret && ret != -ENOENT)) {
4789	btrfs_abort_transaction(trans, ret);
4790	goto out_end_trans;
4791	}
4792	if (!btrfs_is_empty_uuid(uuid: dest->root_item.received_uuid)) {
4793	ret = btrfs_uuid_tree_remove(trans,
4794	uuid: dest->root_item.received_uuid,
4795	BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4796	subid: btrfs_root_id(root: dest));
4797	if (unlikely(ret && ret != -ENOENT)) {
4798	btrfs_abort_transaction(trans, ret);
4799	goto out_end_trans;
4800	}
4801	}
4802
4803	free_anon_bdev(dest->anon_dev);
4804	dest->anon_dev = 0;
4805	out_end_trans:
4806	trans->block_rsv = NULL;
4807	trans->bytes_reserved = 0;
4808	ret = btrfs_end_transaction(trans);
4809	inode->i_flags |= S_DEAD;
4810	out_release:
4811	btrfs_block_rsv_release(fs_info, block_rsv: &block_rsv, num_bytes: (u64)-1, NULL);
4812	if (qgroup_reserved)
4813	btrfs_qgroup_free_meta_prealloc(root, num_bytes: qgroup_reserved);
4814	out_undead:
4815	if (ret) {
4816	spin_lock(lock: &dest->root_item_lock);
4817	root_flags = btrfs_root_flags(s: &dest->root_item);
4818	btrfs_set_root_flags(s: &dest->root_item,
4819	val: root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);
4820	spin_unlock(lock: &dest->root_item_lock);
4821	}
4822	out_up_write:
4823	up_write(sem: &fs_info->subvol_sem);
4824	if (!ret) {
4825	d_invalidate(dentry);
4826	btrfs_prune_dentries(root: dest);
4827	ASSERT(dest->send_in_progress == 0);
4828	}
4829
4830	return ret;
4831	}
4832
4833	static int btrfs_rmdir(struct inode *vfs_dir, struct dentry *dentry)
4834	{
4835	struct btrfs_inode *dir = BTRFS_I(vfs_dir);
4836	struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
4837	struct btrfs_fs_info *fs_info = inode->root->fs_info;
4838	int ret = 0;
4839	struct btrfs_trans_handle *trans;
4840	struct fscrypt_name fname;
4841
4842	if (inode->vfs_inode.i_size > BTRFS_EMPTY_DIR_SIZE)
4843	return -ENOTEMPTY;
4844	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
4845	if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) {
4846	btrfs_err(fs_info,
4847	"extent tree v2 doesn't support snapshot deletion yet");
4848	return -EOPNOTSUPP;
4849	}
4850	return btrfs_delete_subvolume(dir, dentry);
4851	}
4852
4853	ret = fscrypt_setup_filename(inode: vfs_dir, iname: &dentry->d_name, lookup: 1, fname: &fname);
4854	if (ret)
4855	return ret;
4856
4857	/* This needs to handle no-key deletions later on */
4858
4859	trans = __unlink_start_trans(dir);
4860	if (IS_ERR(ptr: trans)) {
4861	ret = PTR_ERR(ptr: trans);
4862	goto out_notrans;
4863	}
4864
4865	/*
4866	* Propagate the last_unlink_trans value of the deleted dir to its
4867	* parent directory. This is to prevent an unrecoverable log tree in the
4868	* case we do something like this:
4869	* 1) create dir foo
4870	* 2) create snapshot under dir foo
4871	* 3) delete the snapshot
4872	* 4) rmdir foo
4873	* 5) mkdir foo
4874	* 6) fsync foo or some file inside foo
4875	*
4876	* This is because we can't unlink other roots when replaying the dir
4877	* deletes for directory foo.
4878	*/
4879	if (inode->last_unlink_trans >= trans->transid)
4880	btrfs_record_snapshot_destroy(trans, dir);
4881
4882	if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4883	ret = btrfs_unlink_subvol(trans, dir, dentry);
4884	goto out;
4885	}
4886
4887	ret = btrfs_orphan_add(trans, inode);
4888	if (ret)
4889	goto out;
4890
4891	/* now the directory is empty */
4892	ret = btrfs_unlink_inode(trans, dir, inode, name: &fname.disk_name);
4893	if (!ret)
4894	btrfs_i_size_write(inode, size: 0);
4895	out:
4896	btrfs_end_transaction(trans);
4897	out_notrans:
4898	btrfs_btree_balance_dirty(fs_info);
4899	fscrypt_free_filename(fname: &fname);
4900
4901	return ret;
4902	}
4903
4904	static bool is_inside_block(u64 bytenr, u64 blockstart, u32 blocksize)
4905	{
4906	ASSERT(IS_ALIGNED(blockstart, blocksize), "blockstart=%llu blocksize=%u",
4907	blockstart, blocksize);
4908
4909	if (blockstart <= bytenr && bytenr <= blockstart + blocksize - 1)
4910	return true;
4911	return false;
4912	}
4913
4914	static int truncate_block_zero_beyond_eof(struct btrfs_inode *inode, u64 start)
4915	{
4916	const pgoff_t index = (start >> PAGE_SHIFT);
4917	struct address_space *mapping = inode->vfs_inode.i_mapping;
4918	struct folio *folio;
4919	u64 zero_start;
4920	u64 zero_end;
4921	int ret = 0;
4922
4923	again:
4924	folio = filemap_lock_folio(mapping, index);
4925	/* No folio present. */
4926	if (IS_ERR(ptr: folio))
4927	return 0;
4928
4929	if (!folio_test_uptodate(folio)) {
4930	ret = btrfs_read_folio(NULL, folio);
4931	folio_lock(folio);
4932	if (folio->mapping != mapping) {
4933	folio_unlock(folio);
4934	folio_put(folio);
4935	goto again;
4936	}
4937	if (unlikely(!folio_test_uptodate(folio))) {
4938	ret = -EIO;
4939	goto out_unlock;
4940	}
4941	}
4942	folio_wait_writeback(folio);
4943
4944	/*
4945	* We do not need to lock extents nor wait for OE, as it's already
4946	* beyond EOF.
4947	*/
4948
4949	zero_start = max_t(u64, folio_pos(folio), start);
4950	zero_end = folio_next_pos(folio);
4951	folio_zero_range(folio, start: zero_start - folio_pos(folio),
4952	length: zero_end - zero_start);
4953
4954	out_unlock:
4955	folio_unlock(folio);
4956	folio_put(folio);
4957	return ret;
4958	}
4959
4960	/*
4961	* Handle the truncation of a fs block.
4962	*
4963	* @inode - inode that we're zeroing
4964	* @offset - the file offset of the block to truncate
4965	* The value must be inside [@start, @end], and the function will do
4966	* extra checks if the block that covers @offset needs to be zeroed.
4967	* @start - the start file offset of the range we want to zero
4968	* @end - the end (inclusive) file offset of the range we want to zero.
4969	*
4970	* If the range is not block aligned, read out the folio that covers @offset,
4971	* and if needed zero blocks that are inside the folio and covered by [@start, @end).
4972	* If @start or @end + 1 lands inside a block, that block will be marked dirty
4973	* for writeback.
4974	*
4975	* This is utilized by hole punch, zero range, file expansion.
4976	*/
4977	int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end)
4978	{
4979	struct btrfs_fs_info *fs_info = inode->root->fs_info;
4980	struct address_space *mapping = inode->vfs_inode.i_mapping;
4981	struct extent_io_tree *io_tree = &inode->io_tree;
4982	struct btrfs_ordered_extent *ordered;
4983	struct extent_state *cached_state = NULL;
4984	struct extent_changeset *data_reserved = NULL;
4985	bool only_release_metadata = false;
4986	u32 blocksize = fs_info->sectorsize;
4987	pgoff_t index = (offset >> PAGE_SHIFT);
4988	struct folio *folio;
4989	gfp_t mask = btrfs_alloc_write_mask(mapping);
4990	int ret = 0;
4991	const bool in_head_block = is_inside_block(bytenr: offset, round_down(start, blocksize),
4992	blocksize);
4993	const bool in_tail_block = is_inside_block(bytenr: offset, round_down(end, blocksize),
4994	blocksize);
4995	bool need_truncate_head = false;
4996	bool need_truncate_tail = false;
4997	u64 zero_start;
4998	u64 zero_end;
4999	u64 block_start;
5000	u64 block_end;
5001
5002	/* @offset should be inside the range. */
5003	ASSERT(start <= offset && offset <= end, "offset=%llu start=%llu end=%llu",
5004	offset, start, end);
5005
5006	/* The range is aligned at both ends. */
5007	if (IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize)) {
5008	/*
5009	* For block size < page size case, we may have polluted blocks
5010	* beyond EOF. So we also need to zero them out.
5011	*/
5012	if (end == (u64)-1 && blocksize < PAGE_SIZE)
5013	ret = truncate_block_zero_beyond_eof(inode, start);
5014	goto out;
5015	}
5016
5017	/*
5018	* @offset may not be inside the head nor tail block. In that case we
5019	* don't need to do anything.
5020	*/
5021	if (!in_head_block && !in_tail_block)
5022	goto out;
5023
5024	/*
5025	* Skip the truncation if the range in the target block is already aligned.
5026	* The seemingly complex check will also handle the same block case.
5027	*/
5028	if (in_head_block && !IS_ALIGNED(start, blocksize))
5029	need_truncate_head = true;
5030	if (in_tail_block && !IS_ALIGNED(end + 1, blocksize))
5031	need_truncate_tail = true;
5032	if (!need_truncate_head && !need_truncate_tail)
5033	goto out;
5034
5035	block_start = round_down(offset, blocksize);
5036	block_end = block_start + blocksize - 1;
5037
5038	ret = btrfs_check_data_free_space(inode, reserved: &data_reserved, start: block_start,
5039	len: blocksize, noflush: false);
5040	if (ret < 0) {
5041	size_t write_bytes = blocksize;
5042
5043	if (btrfs_check_nocow_lock(inode, pos: block_start, write_bytes: &write_bytes, nowait: false) > 0) {
5044	/* For nocow case, no need to reserve data space. */
5045	ASSERT(write_bytes == blocksize, "write_bytes=%zu blocksize=%u",
5046	write_bytes, blocksize);
5047	only_release_metadata = true;
5048	} else {
5049	goto out;
5050	}
5051	}
5052	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes: blocksize, disk_num_bytes: blocksize, noflush: false);
5053	if (ret < 0) {
5054	if (!only_release_metadata)
5055	btrfs_free_reserved_data_space(inode, reserved: data_reserved,
5056	start: block_start, len: blocksize);
5057	goto out;
5058	}
5059	again:
5060	folio = __filemap_get_folio(mapping, index,
5061	FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp: mask);
5062	if (IS_ERR(ptr: folio)) {
5063	if (only_release_metadata)
5064	btrfs_delalloc_release_metadata(inode, num_bytes: blocksize, qgroup_free: true);
5065	else
5066	btrfs_delalloc_release_space(inode, reserved: data_reserved,
5067	start: block_start, len: blocksize, qgroup_free: true);
5068	btrfs_delalloc_release_extents(inode, num_bytes: blocksize);
5069	ret = PTR_ERR(ptr: folio);
5070	goto out;
5071	}
5072
5073	if (!folio_test_uptodate(folio)) {
5074	ret = btrfs_read_folio(NULL, folio);
5075	folio_lock(folio);
5076	if (folio->mapping != mapping) {
5077	folio_unlock(folio);
5078	folio_put(folio);
5079	goto again;
5080	}
5081	if (unlikely(!folio_test_uptodate(folio))) {
5082	ret = -EIO;
5083	goto out_unlock;
5084	}
5085	}
5086
5087	/*
5088	* We unlock the page after the io is completed and then re-lock it
5089	* above. release_folio() could have come in between that and cleared
5090	* folio private, but left the page in the mapping. Set the page mapped
5091	* here to make sure it's properly set for the subpage stuff.
5092	*/
5093	ret = set_folio_extent_mapped(folio);
5094	if (ret < 0)
5095	goto out_unlock;
5096
5097	folio_wait_writeback(folio);
5098
5099	btrfs_lock_extent(tree: io_tree, start: block_start, end: block_end, cached: &cached_state);
5100
5101	ordered = btrfs_lookup_ordered_extent(inode, file_offset: block_start);
5102	if (ordered) {
5103	btrfs_unlock_extent(tree: io_tree, start: block_start, end: block_end, cached: &cached_state);
5104	folio_unlock(folio);
5105	folio_put(folio);
5106	btrfs_start_ordered_extent(entry: ordered);
5107	btrfs_put_ordered_extent(entry: ordered);
5108	goto again;
5109	}
5110
5111	btrfs_clear_extent_bit(tree: &inode->io_tree, start: block_start, end: block_end,
5112	bits: EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
5113	cached: &cached_state);
5114
5115	ret = btrfs_set_extent_delalloc(inode, start: block_start, end: block_end, extra_bits: 0,
5116	cached_state: &cached_state);
5117	if (ret) {
5118	btrfs_unlock_extent(tree: io_tree, start: block_start, end: block_end, cached: &cached_state);
5119	goto out_unlock;
5120	}
5121
5122	if (end == (u64)-1) {
5123	/*
5124	* We're truncating beyond EOF, the remaining blocks normally are
5125	* already holes thus no need to zero again, but it's possible for
5126	* fs block size < page size cases to have memory mapped writes
5127	* to pollute ranges beyond EOF.
5128	*
5129	* In that case although such polluted blocks beyond EOF will
5130	* not reach disk, it still affects our page caches.
5131	*/
5132	zero_start = max_t(u64, folio_pos(folio), start);
5133	zero_end = min_t(u64, folio_next_pos(folio) - 1, end);
5134	} else {
5135	zero_start = max_t(u64, block_start, start);
5136	zero_end = min_t(u64, block_end, end);
5137	}
5138	folio_zero_range(folio, start: zero_start - folio_pos(folio),
5139	length: zero_end - zero_start + 1);
5140
5141	btrfs_folio_clear_checked(fs_info, folio, start: block_start,
5142	len: block_end + 1 - block_start);
5143	btrfs_folio_set_dirty(fs_info, folio, start: block_start,
5144	len: block_end + 1 - block_start);
5145
5146	if (only_release_metadata)
5147	btrfs_set_extent_bit(tree: &inode->io_tree, start: block_start, end: block_end,
5148	bits: EXTENT_NORESERVE, cached_state: &cached_state);
5149
5150	btrfs_unlock_extent(tree: io_tree, start: block_start, end: block_end, cached: &cached_state);
5151
5152	out_unlock:
5153	if (ret) {
5154	if (only_release_metadata)
5155	btrfs_delalloc_release_metadata(inode, num_bytes: blocksize, qgroup_free: true);
5156	else
5157	btrfs_delalloc_release_space(inode, reserved: data_reserved,
5158	start: block_start, len: blocksize, qgroup_free: true);
5159	}
5160	btrfs_delalloc_release_extents(inode, num_bytes: blocksize);
5161	folio_unlock(folio);
5162	folio_put(folio);
5163	out:
5164	if (only_release_metadata)
5165	btrfs_check_nocow_unlock(inode);
5166	extent_changeset_free(changeset: data_reserved);
5167	return ret;
5168	}
5169
5170	static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len)
5171	{
5172	struct btrfs_root *root = inode->root;
5173	struct btrfs_fs_info *fs_info = root->fs_info;
5174	struct btrfs_trans_handle *trans;
5175	struct btrfs_drop_extents_args drop_args = { 0 };
5176	int ret;
5177
5178	/*
5179	* If NO_HOLES is enabled, we don't need to do anything.
5180	* Later, up in the call chain, either btrfs_set_inode_last_sub_trans()
5181	* or btrfs_update_inode() will be called, which guarantee that the next
5182	* fsync will know this inode was changed and needs to be logged.
5183	*/
5184	if (btrfs_fs_incompat(fs_info, NO_HOLES))
5185	return 0;
5186
5187	/*
5188	* 1 - for the one we're dropping
5189	* 1 - for the one we're adding
5190	* 1 - for updating the inode.
5191	*/
5192	trans = btrfs_start_transaction(root, num_items: 3);
5193	if (IS_ERR(ptr: trans))
5194	return PTR_ERR(ptr: trans);
5195
5196	drop_args.start = offset;
5197	drop_args.end = offset + len;
5198	drop_args.drop_cache = true;
5199
5200	ret = btrfs_drop_extents(trans, root, inode, args: &drop_args);
5201	if (unlikely(ret)) {
5202	btrfs_abort_transaction(trans, ret);
5203	btrfs_end_transaction(trans);
5204	return ret;
5205	}
5206
5207	ret = btrfs_insert_hole_extent(trans, root, objectid: btrfs_ino(inode), pos: offset, num_bytes: len);
5208	if (ret) {
5209	btrfs_abort_transaction(trans, ret);
5210	} else {
5211	btrfs_update_inode_bytes(inode, add_bytes: 0, del_bytes: drop_args.bytes_found);
5212	btrfs_update_inode(trans, inode);
5213	}
5214	btrfs_end_transaction(trans);
5215	return ret;
5216	}
5217
5218	/*
5219	* This function puts in dummy file extents for the area we're creating a hole
5220	* for. So if we are truncating this file to a larger size we need to insert
5221	* these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
5222	* the range between oldsize and size
5223	*/
5224	int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size)
5225	{
5226	struct btrfs_root *root = inode->root;
5227	struct btrfs_fs_info *fs_info = root->fs_info;
5228	struct extent_io_tree *io_tree = &inode->io_tree;
5229	struct extent_map *em = NULL;
5230	struct extent_state *cached_state = NULL;
5231	u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
5232	u64 block_end = ALIGN(size, fs_info->sectorsize);
5233	u64 last_byte;
5234	u64 cur_offset;
5235	u64 hole_size;
5236	int ret = 0;
5237
5238	/*
5239	* If our size started in the middle of a block we need to zero out the
5240	* rest of the block before we expand the i_size, otherwise we could
5241	* expose stale data.
5242	*/
5243	ret = btrfs_truncate_block(inode, offset: oldsize, start: oldsize, end: -1);
5244	if (ret)
5245	return ret;
5246
5247	if (size <= hole_start)
5248	return 0;
5249
5250	btrfs_lock_and_flush_ordered_range(inode, start: hole_start, end: block_end - 1,
5251	cached_state: &cached_state);
5252	cur_offset = hole_start;
5253	while (1) {
5254	em = btrfs_get_extent(inode, NULL, start: cur_offset, len: block_end - cur_offset);
5255	if (IS_ERR(ptr: em)) {
5256	ret = PTR_ERR(ptr: em);
5257	em = NULL;
5258	break;
5259	}
5260	last_byte = min(btrfs_extent_map_end(em), block_end);
5261	last_byte = ALIGN(last_byte, fs_info->sectorsize);
5262	hole_size = last_byte - cur_offset;
5263
5264	if (!(em->flags & EXTENT_FLAG_PREALLOC)) {
5265	struct extent_map *hole_em;
5266
5267	ret = maybe_insert_hole(inode, offset: cur_offset, len: hole_size);
5268	if (ret)
5269	break;
5270
5271	ret = btrfs_inode_set_file_extent_range(inode,
5272	start: cur_offset, len: hole_size);
5273	if (ret)
5274	break;
5275
5276	hole_em = btrfs_alloc_extent_map();
5277	if (!hole_em) {
5278	btrfs_drop_extent_map_range(inode, start: cur_offset,
5279	end: cur_offset + hole_size - 1,
5280	skip_pinned: false);
5281	btrfs_set_inode_full_sync(inode);
5282	goto next;
5283	}
5284	hole_em->start = cur_offset;
5285	hole_em->len = hole_size;
5286
5287	hole_em->disk_bytenr = EXTENT_MAP_HOLE;
5288	hole_em->disk_num_bytes = 0;
5289	hole_em->ram_bytes = hole_size;
5290	hole_em->generation = btrfs_get_fs_generation(fs_info);
5291
5292	ret = btrfs_replace_extent_map_range(inode, new_em: hole_em, modified: true);
5293	btrfs_free_extent_map(em: hole_em);
5294	} else {
5295	ret = btrfs_inode_set_file_extent_range(inode,
5296	start: cur_offset, len: hole_size);
5297	if (ret)
5298	break;
5299	}
5300	next:
5301	btrfs_free_extent_map(em);
5302	em = NULL;
5303	cur_offset = last_byte;
5304	if (cur_offset >= block_end)
5305	break;
5306	}
5307	btrfs_free_extent_map(em);
5308	btrfs_unlock_extent(tree: io_tree, start: hole_start, end: block_end - 1, cached: &cached_state);
5309	return ret;
5310	}
5311
5312	static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5313	{
5314	struct btrfs_root *root = BTRFS_I(inode)->root;
5315	struct btrfs_trans_handle *trans;
5316	loff_t oldsize = i_size_read(inode);
5317	loff_t newsize = attr->ia_size;
5318	int mask = attr->ia_valid;
5319	int ret;
5320
5321	/*
5322	* The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5323	* special case where we need to update the times despite not having
5324	* these flags set. For all other operations the VFS set these flags
5325	* explicitly if it wants a timestamp update.
5326	*/
5327	if (newsize != oldsize) {
5328	inode_inc_iversion(inode);
5329	if (!(mask & (ATTR_CTIME | ATTR_MTIME))) {
5330	inode_set_mtime_to_ts(inode,
5331	ts: inode_set_ctime_current(inode));
5332	}
5333	}
5334
5335	if (newsize > oldsize) {
5336	/*
5337	* Don't do an expanding truncate while snapshotting is ongoing.
5338	* This is to ensure the snapshot captures a fully consistent
5339	* state of this file - if the snapshot captures this expanding
5340	* truncation, it must capture all writes that happened before
5341	* this truncation.
5342	*/
5343	btrfs_drew_write_lock(lock: &root->snapshot_lock);
5344	ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, size: newsize);
5345	if (ret) {
5346	btrfs_drew_write_unlock(lock: &root->snapshot_lock);
5347	return ret;
5348	}
5349
5350	trans = btrfs_start_transaction(root, num_items: 1);
5351	if (IS_ERR(ptr: trans)) {
5352	btrfs_drew_write_unlock(lock: &root->snapshot_lock);
5353	return PTR_ERR(ptr: trans);
5354	}
5355
5356	i_size_write(inode, i_size: newsize);
5357	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), new_i_size: 0);
5358	pagecache_isize_extended(inode, from: oldsize, to: newsize);
5359	ret = btrfs_update_inode(trans, BTRFS_I(inode));
5360	btrfs_drew_write_unlock(lock: &root->snapshot_lock);
5361	btrfs_end_transaction(trans);
5362	} else {
5363	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
5364
5365	if (btrfs_is_zoned(fs_info)) {
5366	ret = btrfs_wait_ordered_range(BTRFS_I(inode),
5367	ALIGN(newsize, fs_info->sectorsize),
5368	len: (u64)-1);
5369	if (ret)
5370	return ret;
5371	}
5372
5373	/*
5374	* We're truncating a file that used to have good data down to
5375	* zero. Make sure any new writes to the file get on disk
5376	* on close.
5377	*/
5378	if (newsize == 0)
5379	set_bit(nr: BTRFS_INODE_FLUSH_ON_CLOSE,
5380	addr: &BTRFS_I(inode)->runtime_flags);
5381
5382	truncate_setsize(inode, newsize);
5383
5384	inode_dio_wait(inode);
5385
5386	ret = btrfs_truncate(BTRFS_I(inode), skip_writeback: newsize == oldsize);
5387	if (ret && inode->i_nlink) {
5388	int ret2;
5389
5390	/*
5391	* Truncate failed, so fix up the in-memory size. We
5392	* adjusted disk_i_size down as we removed extents, so
5393	* wait for disk_i_size to be stable and then update the
5394	* in-memory size to match.
5395	*/
5396	ret2 = btrfs_wait_ordered_range(BTRFS_I(inode), start: 0, len: (u64)-1);
5397	if (ret2)
5398	return ret2;
5399	i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5400	}
5401	}
5402
5403	return ret;
5404	}
5405
5406	static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
5407	struct iattr *attr)
5408	{
5409	struct inode *inode = d_inode(dentry);
5410	struct btrfs_root *root = BTRFS_I(inode)->root;
5411	int ret;
5412
5413	if (btrfs_root_readonly(root))
5414	return -EROFS;
5415
5416	ret = setattr_prepare(idmap, dentry, attr);
5417	if (ret)
5418	return ret;
5419
5420	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5421	ret = btrfs_setsize(inode, attr);
5422	if (ret)
5423	return ret;
5424	}
5425
5426	if (attr->ia_valid) {
5427	setattr_copy(idmap, inode, attr);
5428	inode_inc_iversion(inode);
5429	ret = btrfs_dirty_inode(BTRFS_I(inode));
5430
5431	if (!ret && attr->ia_valid & ATTR_MODE)
5432	ret = posix_acl_chmod(idmap, dentry, inode->i_mode);
5433	}
5434
5435	return ret;
5436	}
5437
5438	/*
5439	* While truncating the inode pages during eviction, we get the VFS
5440	* calling btrfs_invalidate_folio() against each folio of the inode. This
5441	* is slow because the calls to btrfs_invalidate_folio() result in a
5442	* huge amount of calls to lock_extent() and clear_extent_bit(),
5443	* which keep merging and splitting extent_state structures over and over,
5444	* wasting lots of time.
5445	*
5446	* Therefore if the inode is being evicted, let btrfs_invalidate_folio()
5447	* skip all those expensive operations on a per folio basis and do only
5448	* the ordered io finishing, while we release here the extent_map and
5449	* extent_state structures, without the excessive merging and splitting.
5450	*/
5451	static void evict_inode_truncate_pages(struct inode *inode)
5452	{
5453	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5454	struct rb_node *node;
5455
5456	ASSERT(inode_state_read_once(inode) & I_FREEING);
5457	truncate_inode_pages_final(mapping: &inode->i_data);
5458
5459	btrfs_drop_extent_map_range(BTRFS_I(inode), start: 0, end: (u64)-1, skip_pinned: false);
5460
5461	/*
5462	* Keep looping until we have no more ranges in the io tree.
5463	* We can have ongoing bios started by readahead that have
5464	* their endio callback (extent_io.c:end_bio_extent_readpage)
5465	* still in progress (unlocked the pages in the bio but did not yet
5466	* unlocked the ranges in the io tree). Therefore this means some
5467	* ranges can still be locked and eviction started because before
5468	* submitting those bios, which are executed by a separate task (work
5469	* queue kthread), inode references (inode->i_count) were not taken
5470	* (which would be dropped in the end io callback of each bio).
5471	* Therefore here we effectively end up waiting for those bios and
5472	* anyone else holding locked ranges without having bumped the inode's
5473	* reference count - if we don't do it, when they access the inode's
5474	* io_tree to unlock a range it may be too late, leading to an
5475	* use-after-free issue.
5476	*/
5477	spin_lock(lock: &io_tree->lock);
5478	while (!RB_EMPTY_ROOT(&io_tree->state)) {
5479	struct extent_state *state;
5480	struct extent_state *cached_state = NULL;
5481	u64 start;
5482	u64 end;
5483	unsigned state_flags;
5484
5485	node = rb_first(root: &io_tree->state);
5486	state = rb_entry(node, struct extent_state, rb_node);
5487	start = state->start;
5488	end = state->end;
5489	state_flags = state->state;
5490	spin_unlock(lock: &io_tree->lock);
5491
5492	btrfs_lock_extent(tree: io_tree, start, end, cached: &cached_state);
5493
5494	/*
5495	* If still has DELALLOC flag, the extent didn't reach disk,
5496	* and its reserved space won't be freed by delayed_ref.
5497	* So we need to free its reserved space here.
5498	* (Refer to comment in btrfs_invalidate_folio, case 2)
5499	*
5500	* Note, end is the bytenr of last byte, so we need + 1 here.
5501	*/
5502	if (state_flags & EXTENT_DELALLOC)
5503	btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start,
5504	len: end - start + 1, NULL);
5505
5506	btrfs_clear_extent_bit(tree: io_tree, start, end,
5507	bits: EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING,
5508	cached: &cached_state);
5509
5510	cond_resched();
5511	spin_lock(lock: &io_tree->lock);
5512	}
5513	spin_unlock(lock: &io_tree->lock);
5514	}
5515
5516	static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
5517	struct btrfs_block_rsv *rsv)
5518	{
5519	struct btrfs_fs_info *fs_info = root->fs_info;
5520	struct btrfs_trans_handle *trans;
5521	u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, num_delayed_refs: 1);
5522	int ret;
5523
5524	/*
5525	* Eviction should be taking place at some place safe because of our
5526	* delayed iputs. However the normal flushing code will run delayed
5527	* iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
5528	*
5529	* We reserve the delayed_refs_extra here again because we can't use
5530	* btrfs_start_transaction(root, 0) for the same deadlocky reason as
5531	* above. We reserve our extra bit here because we generate a ton of
5532	* delayed refs activity by truncating.
5533	*
5534	* BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can,
5535	* if we fail to make this reservation we can re-try without the
5536	* delayed_refs_extra so we can make some forward progress.
5537	*/
5538	ret = btrfs_block_rsv_refill(fs_info, block_rsv: rsv, num_bytes: rsv->size + delayed_refs_extra,
5539	flush: BTRFS_RESERVE_FLUSH_EVICT);
5540	if (ret) {
5541	ret = btrfs_block_rsv_refill(fs_info, block_rsv: rsv, num_bytes: rsv->size,
5542	flush: BTRFS_RESERVE_FLUSH_EVICT);
5543	if (ret) {
5544	btrfs_warn(fs_info,
5545	"could not allocate space for delete; will truncate on mount");
5546	return ERR_PTR(error: -ENOSPC);
5547	}
5548	delayed_refs_extra = 0;
5549	}
5550
5551	trans = btrfs_join_transaction(root);
5552	if (IS_ERR(ptr: trans))
5553	return trans;
5554
5555	if (delayed_refs_extra) {
5556	trans->block_rsv = &fs_info->trans_block_rsv;
5557	trans->bytes_reserved = delayed_refs_extra;
5558	btrfs_block_rsv_migrate(src_rsv: rsv, dst_rsv: trans->block_rsv,
5559	num_bytes: delayed_refs_extra, update_size: true);
5560	}
5561	return trans;
5562	}
5563
5564	void btrfs_evict_inode(struct inode *inode)
5565	{
5566	struct btrfs_fs_info *fs_info;
5567	struct btrfs_trans_handle *trans;
5568	struct btrfs_root *root = BTRFS_I(inode)->root;
5569	struct btrfs_block_rsv rsv;
5570	int ret;
5571
5572	trace_btrfs_inode_evict(inode);
5573
5574	if (!root) {
5575	fsverity_cleanup_inode(inode);
5576	clear_inode(inode);
5577	return;
5578	}
5579
5580	fs_info = inode_to_fs_info(inode);
5581	evict_inode_truncate_pages(inode);
5582
5583	if (inode->i_nlink &&
5584	((btrfs_root_refs(s: &root->root_item) != 0 &&
5585	btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) ||
5586	btrfs_is_free_space_inode(BTRFS_I(inode))))
5587	goto out;
5588
5589	if (is_bad_inode(inode))
5590	goto out;
5591
5592	if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
5593	goto out;
5594
5595	if (inode->i_nlink > 0) {
5596	BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5597	btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID);
5598	goto out;
5599	}
5600
5601	/*
5602	* This makes sure the inode item in tree is uptodate and the space for
5603	* the inode update is released.
5604	*/
5605	ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5606	if (ret)
5607	goto out;
5608
5609	/*
5610	* This drops any pending insert or delete operations we have for this
5611	* inode. We could have a delayed dir index deletion queued up, but
5612	* we're removing the inode completely so that'll be taken care of in
5613	* the truncate.
5614	*/
5615	btrfs_kill_delayed_inode_items(BTRFS_I(inode));
5616
5617	btrfs_init_metadata_block_rsv(fs_info, rsv: &rsv, type: BTRFS_BLOCK_RSV_TEMP);
5618	rsv.size = btrfs_calc_metadata_size(fs_info, num_items: 1);
5619	rsv.failfast = true;
5620
5621	btrfs_i_size_write(BTRFS_I(inode), size: 0);
5622
5623	while (1) {
5624	struct btrfs_truncate_control control = {
5625	.inode = BTRFS_I(inode),
5626	.ino = btrfs_ino(BTRFS_I(inode)),
5627	.new_size = 0,
5628	.min_type = 0,
5629	};
5630
5631	trans = evict_refill_and_join(root, rsv: &rsv);
5632	if (IS_ERR(ptr: trans))
5633	goto out_release;
5634
5635	trans->block_rsv = &rsv;
5636
5637	ret = btrfs_truncate_inode_items(trans, root, control: &control);
5638	trans->block_rsv = &fs_info->trans_block_rsv;
5639	btrfs_end_transaction(trans);
5640	/*
5641	* We have not added new delayed items for our inode after we
5642	* have flushed its delayed items, so no need to throttle on
5643	* delayed items. However we have modified extent buffers.
5644	*/
5645	btrfs_btree_balance_dirty_nodelay(fs_info);
5646	if (ret && ret != -ENOSPC && ret != -EAGAIN)
5647	goto out_release;
5648	else if (!ret)
5649	break;
5650	}
5651
5652	/*
5653	* Errors here aren't a big deal, it just means we leave orphan items in
5654	* the tree. They will be cleaned up on the next mount. If the inode
5655	* number gets reused, cleanup deletes the orphan item without doing
5656	* anything, and unlink reuses the existing orphan item.
5657	*
5658	* If it turns out that we are dropping too many of these, we might want
5659	* to add a mechanism for retrying these after a commit.
5660	*/
5661	trans = evict_refill_and_join(root, rsv: &rsv);
5662	if (!IS_ERR(ptr: trans)) {
5663	trans->block_rsv = &rsv;
5664	btrfs_orphan_del(trans, BTRFS_I(inode));
5665	trans->block_rsv = &fs_info->trans_block_rsv;
5666	btrfs_end_transaction(trans);
5667	}
5668
5669	out_release:
5670	btrfs_block_rsv_release(fs_info, block_rsv: &rsv, num_bytes: (u64)-1, NULL);
5671	out:
5672	/*
5673	* If we didn't successfully delete, the orphan item will still be in
5674	* the tree and we'll retry on the next mount. Again, we might also want
5675	* to retry these periodically in the future.
5676	*/
5677	btrfs_remove_delayed_node(BTRFS_I(inode));
5678	fsverity_cleanup_inode(inode);
5679	clear_inode(inode);
5680	}
5681
5682	/*
5683	* Return the key found in the dir entry in the location pointer, fill @type
5684	* with BTRFS_FT_*, and return 0.
5685	*
5686	* If no dir entries were found, returns -ENOENT.
5687	* If found a corrupted location in dir entry, returns -EUCLEAN.
5688	*/
5689	static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry,
5690	struct btrfs_key *location, u8 *type)
5691	{
5692	struct btrfs_dir_item *di;
5693	BTRFS_PATH_AUTO_FREE(path);
5694	struct btrfs_root *root = dir->root;
5695	int ret = 0;
5696	struct fscrypt_name fname;
5697
5698	path = btrfs_alloc_path();
5699	if (!path)
5700	return -ENOMEM;
5701
5702	ret = fscrypt_setup_filename(inode: &dir->vfs_inode, iname: &dentry->d_name, lookup: 1, fname: &fname);
5703	if (ret < 0)
5704	return ret;
5705	/*
5706	* fscrypt_setup_filename() should never return a positive value, but
5707	* gcc on sparc/parisc thinks it can, so assert that doesn't happen.
5708	*/
5709	ASSERT(ret == 0);
5710
5711	/* This needs to handle no-key deletions later on */
5712
5713	di = btrfs_lookup_dir_item(NULL, root, path, dir: btrfs_ino(inode: dir),
5714	name: &fname.disk_name, mod: 0);
5715	if (IS_ERR_OR_NULL(ptr: di)) {
5716	ret = di ? PTR_ERR(ptr: di) : -ENOENT;
5717	goto out;
5718	}
5719
5720	btrfs_dir_item_key_to_cpu(eb: path->nodes[0], item: di, cpu_key: location);
5721	if (unlikely(location->type != BTRFS_INODE_ITEM_KEY &&
5722	location->type != BTRFS_ROOT_ITEM_KEY)) {
5723	ret = -EUCLEAN;
5724	btrfs_warn(root->fs_info,
5725	"%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location " BTRFS_KEY_FMT ")",
5726	__func__, fname.disk_name.name, btrfs_ino(dir),
5727	BTRFS_KEY_FMT_VALUE(location));
5728	}
5729	if (!ret)
5730	*type = btrfs_dir_ftype(eb: path->nodes[0], item: di);
5731	out:
5732	fscrypt_free_filename(fname: &fname);
5733	return ret;
5734	}
5735
5736	/*
5737	* when we hit a tree root in a directory, the btrfs part of the inode
5738	* needs to be changed to reflect the root directory of the tree root. This
5739	* is kind of like crossing a mount point.
5740	*/
5741	static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5742	struct btrfs_inode *dir,
5743	struct dentry *dentry,
5744	struct btrfs_key *location,
5745	struct btrfs_root **sub_root)
5746	{
5747	BTRFS_PATH_AUTO_FREE(path);
5748	struct btrfs_root *new_root;
5749	struct btrfs_root_ref *ref;
5750	struct extent_buffer *leaf;
5751	struct btrfs_key key;
5752	int ret;
5753	int err = 0;
5754	struct fscrypt_name fname;
5755
5756	ret = fscrypt_setup_filename(inode: &dir->vfs_inode, iname: &dentry->d_name, lookup: 0, fname: &fname);
5757	if (ret)
5758	return ret;
5759
5760	path = btrfs_alloc_path();
5761	if (!path) {
5762	err = -ENOMEM;
5763	goto out;
5764	}
5765
5766	err = -ENOENT;
5767	key.objectid = btrfs_root_id(root: dir->root);
5768	key.type = BTRFS_ROOT_REF_KEY;
5769	key.offset = location->objectid;
5770
5771	ret = btrfs_search_slot(NULL, root: fs_info->tree_root, key: &key, p: path, ins_len: 0, cow: 0);
5772	if (ret) {
5773	if (ret < 0)
5774	err = ret;
5775	goto out;
5776	}
5777
5778	leaf = path->nodes[0];
5779	ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5780	if (btrfs_root_ref_dirid(eb: leaf, s: ref) != btrfs_ino(inode: dir) ||
5781	btrfs_root_ref_name_len(eb: leaf, s: ref) != fname.disk_name.len)
5782	goto out;
5783
5784	ret = memcmp_extent_buffer(eb: leaf, ptrv: fname.disk_name.name,
5785	start: (unsigned long)(ref + 1), len: fname.disk_name.len);
5786	if (ret)
5787	goto out;
5788
5789	btrfs_release_path(p: path);
5790
5791	new_root = btrfs_get_fs_root(fs_info, objectid: location->objectid, check_ref: true);
5792	if (IS_ERR(ptr: new_root)) {
5793	err = PTR_ERR(ptr: new_root);
5794	goto out;
5795	}
5796
5797	*sub_root = new_root;
5798	location->objectid = btrfs_root_dirid(s: &new_root->root_item);
5799	location->type = BTRFS_INODE_ITEM_KEY;
5800	location->offset = 0;
5801	err = 0;
5802	out:
5803	fscrypt_free_filename(fname: &fname);
5804	return err;
5805	}
5806
5807
5808
5809	static void btrfs_del_inode_from_root(struct btrfs_inode *inode)
5810	{
5811	struct btrfs_root *root = inode->root;
5812	struct btrfs_inode *entry;
5813	bool empty = false;
5814
5815	xa_lock(&root->inodes);
5816	/*
5817	* This btrfs_inode is being freed and has already been unhashed at this
5818	* point. It's possible that another btrfs_inode has already been
5819	* allocated for the same inode and inserted itself into the root, so
5820	* don't delete it in that case.
5821	*
5822	* Note that this shouldn't need to allocate memory, so the gfp flags
5823	* don't really matter.
5824	*/
5825	entry = __xa_cmpxchg(&root->inodes, index: btrfs_ino(inode), old: inode, NULL,
5826	GFP_ATOMIC);
5827	if (entry == inode)
5828	empty = xa_empty(xa: &root->inodes);
5829	xa_unlock(&root->inodes);
5830
5831	if (empty && btrfs_root_refs(s: &root->root_item) == 0) {
5832	xa_lock(&root->inodes);
5833	empty = xa_empty(xa: &root->inodes);
5834	xa_unlock(&root->inodes);
5835	if (empty)
5836	btrfs_add_dead_root(root);
5837	}
5838	}
5839
5840
5841	static int btrfs_init_locked_inode(struct inode *inode, void *p)
5842	{
5843	struct btrfs_iget_args *args = p;
5844
5845	btrfs_set_inode_number(BTRFS_I(inode), ino: args->ino);
5846	BTRFS_I(inode)->root = btrfs_grab_root(root: args->root);
5847
5848	if (args->root && args->root == args->root->fs_info->tree_root &&
5849	args->ino != BTRFS_BTREE_INODE_OBJECTID)
5850	set_bit(nr: BTRFS_INODE_FREE_SPACE_INODE,
5851	addr: &BTRFS_I(inode)->runtime_flags);
5852	return 0;
5853	}
5854
5855	static int btrfs_find_actor(struct inode *inode, void *opaque)
5856	{
5857	struct btrfs_iget_args *args = opaque;
5858
5859	return args->ino == btrfs_ino(BTRFS_I(inode)) &&
5860	args->root == BTRFS_I(inode)->root;
5861	}
5862
5863	static struct btrfs_inode *btrfs_iget_locked(u64 ino, struct btrfs_root *root)
5864	{
5865	struct inode *inode;
5866	struct btrfs_iget_args args;
5867	unsigned long hashval = btrfs_inode_hash(objectid: ino, root);
5868
5869	args.ino = ino;
5870	args.root = root;
5871
5872	inode = iget5_locked_rcu(root->fs_info->sb, hashval, test: btrfs_find_actor,
5873	set: btrfs_init_locked_inode,
5874	(void *)&args);
5875	if (!inode)
5876	return NULL;
5877	return BTRFS_I(inode);
5878	}
5879
5880	/*
5881	* Get an inode object given its inode number and corresponding root. Path is
5882	* preallocated to prevent recursing back to iget through allocator.
5883	*/
5884	struct btrfs_inode *btrfs_iget_path(u64 ino, struct btrfs_root *root,
5885	struct btrfs_path *path)
5886	{
5887	struct btrfs_inode *inode;
5888	int ret;
5889
5890	inode = btrfs_iget_locked(ino, root);
5891	if (!inode)
5892	return ERR_PTR(error: -ENOMEM);
5893
5894	if (!(inode_state_read_once(inode: &inode->vfs_inode) & I_NEW))
5895	return inode;
5896
5897	ret = btrfs_read_locked_inode(inode, path);
5898	if (ret)
5899	return ERR_PTR(error: ret);
5900
5901	unlock_new_inode(&inode->vfs_inode);
5902	return inode;
5903	}
5904
5905	/*
5906	* Get an inode object given its inode number and corresponding root.
5907	*/
5908	struct btrfs_inode *btrfs_iget(u64 ino, struct btrfs_root *root)
5909	{
5910	struct btrfs_inode *inode;
5911	struct btrfs_path *path;
5912	int ret;
5913
5914	inode = btrfs_iget_locked(ino, root);
5915	if (!inode)
5916	return ERR_PTR(error: -ENOMEM);
5917
5918	if (!(inode_state_read_once(inode: &inode->vfs_inode) & I_NEW))
5919	return inode;
5920
5921	path = btrfs_alloc_path();
5922	if (!path) {
5923	iget_failed(&inode->vfs_inode);
5924	return ERR_PTR(error: -ENOMEM);
5925	}
5926
5927	ret = btrfs_read_locked_inode(inode, path);
5928	btrfs_free_path(p: path);
5929	if (ret)
5930	return ERR_PTR(error: ret);
5931
5932	if (S_ISDIR(inode->vfs_inode.i_mode))
5933	inode->vfs_inode.i_opflags |= IOP_FASTPERM_MAY_EXEC;
5934	unlock_new_inode(&inode->vfs_inode);
5935	return inode;
5936	}
5937
5938	static struct btrfs_inode *new_simple_dir(struct inode *dir,
5939	struct btrfs_key *key,
5940	struct btrfs_root *root)
5941	{
5942	struct timespec64 ts;
5943	struct inode *vfs_inode;
5944	struct btrfs_inode *inode;
5945
5946	vfs_inode = new_inode(sb: dir->i_sb);
5947	if (!vfs_inode)
5948	return ERR_PTR(error: -ENOMEM);
5949
5950	inode = BTRFS_I(vfs_inode);
5951	inode->root = btrfs_grab_root(root);
5952	inode->ref_root_id = key->objectid;
5953	set_bit(nr: BTRFS_INODE_ROOT_STUB, addr: &inode->runtime_flags);
5954	set_bit(nr: BTRFS_INODE_DUMMY, addr: &inode->runtime_flags);
5955
5956	btrfs_set_inode_number(inode, BTRFS_EMPTY_SUBVOL_DIR_OBJECTID);
5957	/*
5958	* We only need lookup, the rest is read-only and there's no inode
5959	* associated with the dentry
5960	*/
5961	vfs_inode->i_op = &simple_dir_inode_operations;
5962	vfs_inode->i_opflags &= ~IOP_XATTR;
5963	vfs_inode->i_fop = &simple_dir_operations;
5964	vfs_inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5965
5966	ts = inode_set_ctime_current(inode: vfs_inode);
5967	inode_set_mtime_to_ts(inode: vfs_inode, ts);
5968	inode_set_atime_to_ts(inode: vfs_inode, ts: inode_get_atime(inode: dir));
5969	inode->i_otime_sec = ts.tv_sec;
5970	inode->i_otime_nsec = ts.tv_nsec;
5971
5972	vfs_inode->i_uid = dir->i_uid;
5973	vfs_inode->i_gid = dir->i_gid;
5974
5975	return inode;
5976	}
5977
5978	static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN);
5979	static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE);
5980	static_assert(BTRFS_FT_DIR == FT_DIR);
5981	static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV);
5982	static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV);
5983	static_assert(BTRFS_FT_FIFO == FT_FIFO);
5984	static_assert(BTRFS_FT_SOCK == FT_SOCK);
5985	static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK);
5986
5987	static inline u8 btrfs_inode_type(const struct btrfs_inode *inode)
5988	{
5989	return fs_umode_to_ftype(mode: inode->vfs_inode.i_mode);
5990	}
5991
5992	struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5993	{
5994	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
5995	struct btrfs_inode *inode;
5996	struct btrfs_root *root = BTRFS_I(dir)->root;
5997	struct btrfs_root *sub_root = root;
5998	struct btrfs_key location = { 0 };
5999	u8 di_type = 0;
6000	int ret = 0;
6001
6002	if (dentry->d_name.len > BTRFS_NAME_LEN)
6003	return ERR_PTR(error: -ENAMETOOLONG);
6004
6005	ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, location: &location, type: &di_type);
6006	if (ret < 0)
6007	return ERR_PTR(error: ret);
6008
6009	if (location.type == BTRFS_INODE_ITEM_KEY) {
6010	inode = btrfs_iget(ino: location.objectid, root);
6011	if (IS_ERR(ptr: inode))
6012	return ERR_CAST(ptr: inode);
6013
6014	/* Do extra check against inode mode with di_type */
6015	if (unlikely(btrfs_inode_type(inode) != di_type)) {
6016	btrfs_crit(fs_info,
6017	"inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",
6018	inode->vfs_inode.i_mode, btrfs_inode_type(inode),
6019	di_type);
6020	iput(&inode->vfs_inode);
6021	return ERR_PTR(error: -EUCLEAN);
6022	}
6023	return &inode->vfs_inode;
6024	}
6025
6026	ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry,
6027	location: &location, sub_root: &sub_root);
6028	if (ret < 0) {
6029	if (ret != -ENOENT)
6030	inode = ERR_PTR(error: ret);
6031	else
6032	inode = new_simple_dir(dir, key: &location, root);
6033	} else {
6034	inode = btrfs_iget(ino: location.objectid, root: sub_root);
6035	btrfs_put_root(root: sub_root);
6036
6037	if (IS_ERR(ptr: inode))
6038	return ERR_CAST(ptr: inode);
6039
6040	down_read(sem: &fs_info->cleanup_work_sem);
6041	if (!sb_rdonly(sb: inode->vfs_inode.i_sb))
6042	ret = btrfs_orphan_cleanup(root: sub_root);
6043	up_read(sem: &fs_info->cleanup_work_sem);
6044	if (ret) {
6045	iput(&inode->vfs_inode);
6046	inode = ERR_PTR(error: ret);
6047	}
6048	}
6049
6050	if (IS_ERR(ptr: inode))
6051	return ERR_CAST(ptr: inode);
6052
6053	return &inode->vfs_inode;
6054	}
6055
6056	static int btrfs_dentry_delete(const struct dentry *dentry)
6057	{
6058	struct btrfs_root *root;
6059	struct inode *inode = d_inode(dentry);
6060
6061	if (!inode && !IS_ROOT(dentry))
6062	inode = d_inode(dentry: dentry->d_parent);
6063
6064	if (inode) {
6065	root = BTRFS_I(inode)->root;
6066	if (btrfs_root_refs(s: &root->root_item) == 0)
6067	return 1;
6068
6069	if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
6070	return 1;
6071	}
6072	return 0;
6073	}
6074
6075	static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
6076	unsigned int flags)
6077	{
6078	struct inode *inode = btrfs_lookup_dentry(dir, dentry);
6079
6080	if (inode == ERR_PTR(error: -ENOENT))
6081	inode = NULL;
6082	return d_splice_alias(inode, dentry);
6083	}
6084
6085	/*
6086	* Find the highest existing sequence number in a directory and then set the
6087	* in-memory index_cnt variable to the first free sequence number.
6088	*/
6089	static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6090	{
6091	struct btrfs_root *root = inode->root;
6092	struct btrfs_key key, found_key;
6093	BTRFS_PATH_AUTO_FREE(path);
6094	struct extent_buffer *leaf;
6095	int ret;
6096
6097	key.objectid = btrfs_ino(inode);
6098	key.type = BTRFS_DIR_INDEX_KEY;
6099	key.offset = (u64)-1;
6100
6101	path = btrfs_alloc_path();
6102	if (!path)
6103	return -ENOMEM;
6104
6105	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
6106	if (ret < 0)
6107	return ret;
6108	/* FIXME: we should be able to handle this */
6109	if (ret == 0)
6110	return ret;
6111
6112	if (path->slots[0] == 0) {
6113	inode->index_cnt = BTRFS_DIR_START_INDEX;
6114	return 0;
6115	}
6116
6117	path->slots[0]--;
6118
6119	leaf = path->nodes[0];
6120	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
6121
6122	if (found_key.objectid != btrfs_ino(inode) ||
6123	found_key.type != BTRFS_DIR_INDEX_KEY) {
6124	inode->index_cnt = BTRFS_DIR_START_INDEX;
6125	return 0;
6126	}
6127
6128	inode->index_cnt = found_key.offset + 1;
6129
6130	return 0;
6131	}
6132
6133	static int btrfs_get_dir_last_index(struct btrfs_inode *dir, u64 *index)
6134	{
6135	int ret = 0;
6136
6137	btrfs_inode_lock(inode: dir, ilock_flags: 0);
6138	if (dir->index_cnt == (u64)-1) {
6139	ret = btrfs_inode_delayed_dir_index_count(inode: dir);
6140	if (ret) {
6141	ret = btrfs_set_inode_index_count(inode: dir);
6142	if (ret)
6143	goto out;
6144	}
6145	}
6146
6147	/* index_cnt is the index number of next new entry, so decrement it. */
6148	*index = dir->index_cnt - 1;
6149	out:
6150	btrfs_inode_unlock(inode: dir, ilock_flags: 0);
6151
6152	return ret;
6153	}
6154
6155	/*
6156	* All this infrastructure exists because dir_emit can fault, and we are holding
6157	* the tree lock when doing readdir. For now just allocate a buffer and copy
6158	* our information into that, and then dir_emit from the buffer. This is
6159	* similar to what NFS does, only we don't keep the buffer around in pagecache
6160	* because I'm afraid I'll mess that up. Long term we need to make filldir do
6161	* copy_to_user_inatomic so we don't have to worry about page faulting under the
6162	* tree lock.
6163	*/
6164	static int btrfs_opendir(struct inode *inode, struct file *file)
6165	{
6166	struct btrfs_file_private *private;
6167	u64 last_index;
6168	int ret;
6169
6170	ret = btrfs_get_dir_last_index(BTRFS_I(inode), index: &last_index);
6171	if (ret)
6172	return ret;
6173
6174	private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
6175	if (!private)
6176	return -ENOMEM;
6177	private->last_index = last_index;
6178	private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
6179	if (!private->filldir_buf) {
6180	kfree(objp: private);
6181	return -ENOMEM;
6182	}
6183	file->private_data = private;
6184	return 0;
6185	}
6186
6187	static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence)
6188	{
6189	struct btrfs_file_private *private = file->private_data;
6190	int ret;
6191
6192	ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)),
6193	index: &private->last_index);
6194	if (ret)
6195	return ret;
6196
6197	return generic_file_llseek(file, offset, whence);
6198	}
6199
6200	struct dir_entry {
6201	u64 ino;
6202	u64 offset;
6203	unsigned type;
6204	int name_len;
6205	};
6206
6207	static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
6208	{
6209	while (entries--) {
6210	struct dir_entry *entry = addr;
6211	char *name = (char *)(entry + 1);
6212
6213	ctx->pos = get_unaligned(&entry->offset);
6214	if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),
6215	get_unaligned(&entry->ino),
6216	get_unaligned(&entry->type)))
6217	return 1;
6218	addr += sizeof(struct dir_entry) +
6219	get_unaligned(&entry->name_len);
6220	ctx->pos++;
6221	}
6222	return 0;
6223	}
6224
6225	static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
6226	{
6227	struct inode *inode = file_inode(f: file);
6228	struct btrfs_root *root = BTRFS_I(inode)->root;
6229	struct btrfs_file_private *private = file->private_data;
6230	struct btrfs_dir_item *di;
6231	struct btrfs_key key;
6232	struct btrfs_key found_key;
6233	BTRFS_PATH_AUTO_FREE(path);
6234	void *addr;
6235	LIST_HEAD(ins_list);
6236	LIST_HEAD(del_list);
6237	int ret;
6238	char *name_ptr;
6239	int name_len;
6240	int entries = 0;
6241	int total_len = 0;
6242	bool put = false;
6243	struct btrfs_key location;
6244
6245	if (!dir_emit_dots(file, ctx))
6246	return 0;
6247
6248	path = btrfs_alloc_path();
6249	if (!path)
6250	return -ENOMEM;
6251
6252	addr = private->filldir_buf;
6253	path->reada = READA_FORWARD;
6254
6255	put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), last_index: private->last_index,
6256	ins_list: &ins_list, del_list: &del_list);
6257
6258	again:
6259	key.type = BTRFS_DIR_INDEX_KEY;
6260	key.offset = ctx->pos;
6261	key.objectid = btrfs_ino(BTRFS_I(inode));
6262
6263	btrfs_for_each_slot(root, &key, &found_key, path, ret) {
6264	struct dir_entry *entry;
6265	struct extent_buffer *leaf = path->nodes[0];
6266	u8 ftype;
6267
6268	if (found_key.objectid != key.objectid)
6269	break;
6270	if (found_key.type != BTRFS_DIR_INDEX_KEY)
6271	break;
6272	if (found_key.offset < ctx->pos)
6273	continue;
6274	if (found_key.offset > private->last_index)
6275	break;
6276	if (btrfs_should_delete_dir_index(del_list: &del_list, index: found_key.offset))
6277	continue;
6278	di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
6279	name_len = btrfs_dir_name_len(eb: leaf, s: di);
6280	if ((total_len + sizeof(struct dir_entry) + name_len) >=
6281	PAGE_SIZE) {
6282	btrfs_release_path(p: path);
6283	ret = btrfs_filldir(addr: private->filldir_buf, entries, ctx);
6284	if (ret)
6285	goto nopos;
6286	addr = private->filldir_buf;
6287	entries = 0;
6288	total_len = 0;
6289	goto again;
6290	}
6291
6292	ftype = btrfs_dir_flags_to_ftype(flags: btrfs_dir_flags(eb: leaf, s: di));
6293	entry = addr;
6294	name_ptr = (char *)(entry + 1);
6295	read_extent_buffer(eb: leaf, dst: name_ptr,
6296	start: (unsigned long)(di + 1), len: name_len);
6297	put_unaligned(name_len, &entry->name_len);
6298	put_unaligned(fs_ftype_to_dtype(ftype), &entry->type);
6299	btrfs_dir_item_key_to_cpu(eb: leaf, item: di, cpu_key: &location);
6300	put_unaligned(location.objectid, &entry->ino);
6301	put_unaligned(found_key.offset, &entry->offset);
6302	entries++;
6303	addr += sizeof(struct dir_entry) + name_len;
6304	total_len += sizeof(struct dir_entry) + name_len;
6305	}
6306	/* Catch error encountered during iteration */
6307	if (ret < 0)
6308	goto err;
6309
6310	btrfs_release_path(p: path);
6311
6312	ret = btrfs_filldir(addr: private->filldir_buf, entries, ctx);
6313	if (ret)
6314	goto nopos;
6315
6316	if (btrfs_readdir_delayed_dir_index(ctx, ins_list: &ins_list))
6317	goto nopos;
6318
6319	/*
6320	* Stop new entries from being returned after we return the last
6321	* entry.
6322	*
6323	* New directory entries are assigned a strictly increasing
6324	* offset. This means that new entries created during readdir
6325	* are *guaranteed* to be seen in the future by that readdir.
6326	* This has broken buggy programs which operate on names as
6327	* they're returned by readdir. Until we reuse freed offsets
6328	* we have this hack to stop new entries from being returned
6329	* under the assumption that they'll never reach this huge
6330	* offset.
6331	*
6332	* This is being careful not to overflow 32bit loff_t unless the
6333	* last entry requires it because doing so has broken 32bit apps
6334	* in the past.
6335	*/
6336	if (ctx->pos >= INT_MAX)
6337	ctx->pos = LLONG_MAX;
6338	else
6339	ctx->pos = INT_MAX;
6340	nopos:
6341	ret = 0;
6342	err:
6343	if (put)
6344	btrfs_readdir_put_delayed_items(BTRFS_I(inode), ins_list: &ins_list, del_list: &del_list);
6345	return ret;
6346	}
6347
6348	/*
6349	* This is somewhat expensive, updating the tree every time the
6350	* inode changes. But, it is most likely to find the inode in cache.
6351	* FIXME, needs more benchmarking...there are no reasons other than performance
6352	* to keep or drop this code.
6353	*/
6354	static int btrfs_dirty_inode(struct btrfs_inode *inode)
6355	{
6356	struct btrfs_root *root = inode->root;
6357	struct btrfs_fs_info *fs_info = root->fs_info;
6358	struct btrfs_trans_handle *trans;
6359	int ret;
6360
6361	if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags))
6362	return 0;
6363
6364	trans = btrfs_join_transaction(root);
6365	if (IS_ERR(ptr: trans))
6366	return PTR_ERR(ptr: trans);
6367
6368	ret = btrfs_update_inode(trans, inode);
6369	if (ret == -ENOSPC || ret == -EDQUOT) {
6370	/* whoops, lets try again with the full transaction */
6371	btrfs_end_transaction(trans);
6372	trans = btrfs_start_transaction(root, num_items: 1);
6373	if (IS_ERR(ptr: trans))
6374	return PTR_ERR(ptr: trans);
6375
6376	ret = btrfs_update_inode(trans, inode);
6377	}
6378	btrfs_end_transaction(trans);
6379	if (inode->delayed_node)
6380	btrfs_balance_delayed_items(fs_info);
6381
6382	return ret;
6383	}
6384
6385	/*
6386	* We need our own ->update_time so that we can return error on ENOSPC for
6387	* updating the inode in the case of file write and mmap writes.
6388	*/
6389	static int btrfs_update_time(struct inode *inode, int flags)
6390	{
6391	struct btrfs_root *root = BTRFS_I(inode)->root;
6392	bool dirty;
6393
6394	if (btrfs_root_readonly(root))
6395	return -EROFS;
6396
6397	dirty = inode_update_timestamps(inode, flags);
6398	return dirty ? btrfs_dirty_inode(BTRFS_I(inode)) : 0;
6399	}
6400
6401	/*
6402	* helper to find a free sequence number in a given directory. This current
6403	* code is very simple, later versions will do smarter things in the btree
6404	*/
6405	int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6406	{
6407	int ret = 0;
6408
6409	if (dir->index_cnt == (u64)-1) {
6410	ret = btrfs_inode_delayed_dir_index_count(inode: dir);
6411	if (ret) {
6412	ret = btrfs_set_inode_index_count(inode: dir);
6413	if (ret)
6414	return ret;
6415	}
6416	}
6417
6418	*index = dir->index_cnt;
6419	dir->index_cnt++;
6420
6421	return ret;
6422	}
6423
6424	static int btrfs_insert_inode_locked(struct inode *inode)
6425	{
6426	struct btrfs_iget_args args;
6427
6428	args.ino = btrfs_ino(BTRFS_I(inode));
6429	args.root = BTRFS_I(inode)->root;
6430
6431	return insert_inode_locked4(inode,
6432	btrfs_inode_hash(objectid: inode->i_ino, BTRFS_I(inode)->root),
6433	test: btrfs_find_actor, &args);
6434	}
6435
6436	int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args,
6437	unsigned int *trans_num_items)
6438	{
6439	struct inode *dir = args->dir;
6440	struct inode *inode = args->inode;
6441	int ret;
6442
6443	if (!args->orphan) {
6444	ret = fscrypt_setup_filename(inode: dir, iname: &args->dentry->d_name, lookup: 0,
6445	fname: &args->fname);
6446	if (ret)
6447	return ret;
6448	}
6449
6450	ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl);
6451	if (ret) {
6452	fscrypt_free_filename(fname: &args->fname);
6453	return ret;
6454	}
6455
6456	/* 1 to add inode item */
6457	*trans_num_items = 1;
6458	/* 1 to add compression property */
6459	if (BTRFS_I(dir)->prop_compress)
6460	(*trans_num_items)++;
6461	/* 1 to add default ACL xattr */
6462	if (args->default_acl)
6463	(*trans_num_items)++;
6464	/* 1 to add access ACL xattr */
6465	if (args->acl)
6466	(*trans_num_items)++;
6467	#ifdef CONFIG_SECURITY
6468	/* 1 to add LSM xattr */
6469	if (dir->i_security)
6470	(*trans_num_items)++;
6471	#endif
6472	if (args->orphan) {
6473	/* 1 to add orphan item */
6474	(*trans_num_items)++;
6475	} else {
6476	/*
6477	* 1 to add dir item
6478	* 1 to add dir index
6479	* 1 to update parent inode item
6480	*
6481	* No need for 1 unit for the inode ref item because it is
6482	* inserted in a batch together with the inode item at
6483	* btrfs_create_new_inode().
6484	*/
6485	*trans_num_items += 3;
6486	}
6487	return 0;
6488	}
6489
6490	void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args)
6491	{
6492	posix_acl_release(acl: args->acl);
6493	posix_acl_release(acl: args->default_acl);
6494	fscrypt_free_filename(fname: &args->fname);
6495	}
6496
6497	/*
6498	* Inherit flags from the parent inode.
6499	*
6500	* Currently only the compression flags and the cow flags are inherited.
6501	*/
6502	static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir)
6503	{
6504	unsigned int flags;
6505
6506	flags = dir->flags;
6507
6508	if (flags & BTRFS_INODE_NOCOMPRESS) {
6509	inode->flags &= ~BTRFS_INODE_COMPRESS;
6510	inode->flags |= BTRFS_INODE_NOCOMPRESS;
6511	} else if (flags & BTRFS_INODE_COMPRESS) {
6512	inode->flags &= ~BTRFS_INODE_NOCOMPRESS;
6513	inode->flags |= BTRFS_INODE_COMPRESS;
6514	}
6515
6516	if (flags & BTRFS_INODE_NODATACOW) {
6517	inode->flags |= BTRFS_INODE_NODATACOW;
6518	if (S_ISREG(inode->vfs_inode.i_mode))
6519	inode->flags |= BTRFS_INODE_NODATASUM;
6520	}
6521
6522	btrfs_sync_inode_flags_to_i_flags(inode);
6523	}
6524
6525	int btrfs_create_new_inode(struct btrfs_trans_handle *trans,
6526	struct btrfs_new_inode_args *args)
6527	{
6528	struct timespec64 ts;
6529	struct inode *dir = args->dir;
6530	struct inode *inode = args->inode;
6531	const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name;
6532	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6533	struct btrfs_root *root;
6534	struct btrfs_inode_item *inode_item;
6535	struct btrfs_path *path;
6536	u64 objectid;
6537	struct btrfs_inode_ref *ref;
6538	struct btrfs_key key[2];
6539	u32 sizes[2];
6540	struct btrfs_item_batch batch;
6541	unsigned long ptr;
6542	int ret;
6543	bool xa_reserved = false;
6544
6545	path = btrfs_alloc_path();
6546	if (!path)
6547	return -ENOMEM;
6548
6549	if (!args->subvol)
6550	BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root);
6551	root = BTRFS_I(inode)->root;
6552
6553	ret = btrfs_init_file_extent_tree(BTRFS_I(inode));
6554	if (ret)
6555	goto out;
6556
6557	ret = btrfs_get_free_objectid(root, objectid: &objectid);
6558	if (ret)
6559	goto out;
6560	btrfs_set_inode_number(BTRFS_I(inode), ino: objectid);
6561
6562	ret = xa_reserve(xa: &root->inodes, index: objectid, GFP_NOFS);
6563	if (ret)
6564	goto out;
6565	xa_reserved = true;
6566
6567	if (args->orphan) {
6568	/*
6569	* O_TMPFILE, set link count to 0, so that after this point, we
6570	* fill in an inode item with the correct link count.
6571	*/
6572	set_nlink(inode, nlink: 0);
6573	} else {
6574	trace_btrfs_inode_request(inode: dir);
6575
6576	ret = btrfs_set_inode_index(BTRFS_I(dir), index: &BTRFS_I(inode)->dir_index);
6577	if (ret)
6578	goto out;
6579	}
6580
6581	if (S_ISDIR(inode->i_mode))
6582	BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX;
6583
6584	BTRFS_I(inode)->generation = trans->transid;
6585	inode->i_generation = BTRFS_I(inode)->generation;
6586
6587	/*
6588	* We don't have any capability xattrs set here yet, shortcut any
6589	* queries for the xattrs here. If we add them later via the inode
6590	* security init path or any other path this flag will be cleared.
6591	*/
6592	set_bit(nr: BTRFS_INODE_NO_CAP_XATTR, addr: &BTRFS_I(inode)->runtime_flags);
6593
6594	/*
6595	* Subvolumes don't inherit flags from their parent directory.
6596	* Originally this was probably by accident, but we probably can't
6597	* change it now without compatibility issues.
6598	*/
6599	if (!args->subvol)
6600	btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir));
6601
6602	btrfs_set_inode_mapping_order(BTRFS_I(inode));
6603	if (S_ISREG(inode->i_mode)) {
6604	if (btrfs_test_opt(fs_info, NODATASUM))
6605	BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6606	if (btrfs_test_opt(fs_info, NODATACOW))
6607	BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6608	BTRFS_INODE_NODATASUM;
6609	btrfs_update_inode_mapping_flags(BTRFS_I(inode));
6610	}
6611
6612	ret = btrfs_insert_inode_locked(inode);
6613	if (ret < 0) {
6614	if (!args->orphan)
6615	BTRFS_I(dir)->index_cnt--;
6616	goto out;
6617	}
6618
6619	/*
6620	* We could have gotten an inode number from somebody who was fsynced
6621	* and then removed in this same transaction, so let's just set full
6622	* sync since it will be a full sync anyway and this will blow away the
6623	* old info in the log.
6624	*/
6625	btrfs_set_inode_full_sync(BTRFS_I(inode));
6626
6627	key[0].objectid = objectid;
6628	key[0].type = BTRFS_INODE_ITEM_KEY;
6629	key[0].offset = 0;
6630
6631	sizes[0] = sizeof(struct btrfs_inode_item);
6632
6633	if (!args->orphan) {
6634	/*
6635	* Start new inodes with an inode_ref. This is slightly more
6636	* efficient for small numbers of hard links since they will
6637	* be packed into one item. Extended refs will kick in if we
6638	* add more hard links than can fit in the ref item.
6639	*/
6640	key[1].objectid = objectid;
6641	key[1].type = BTRFS_INODE_REF_KEY;
6642	if (args->subvol) {
6643	key[1].offset = objectid;
6644	sizes[1] = 2 + sizeof(*ref);
6645	} else {
6646	key[1].offset = btrfs_ino(BTRFS_I(dir));
6647	sizes[1] = name->len + sizeof(*ref);
6648	}
6649	}
6650
6651	batch.keys = &key[0];
6652	batch.data_sizes = &sizes[0];
6653	batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]);
6654	batch.nr = args->orphan ? 1 : 2;
6655	ret = btrfs_insert_empty_items(trans, root, path, batch: &batch);
6656	if (unlikely(ret != 0)) {
6657	btrfs_abort_transaction(trans, ret);
6658	goto discard;
6659	}
6660
6661	ts = simple_inode_init_ts(inode);
6662	BTRFS_I(inode)->i_otime_sec = ts.tv_sec;
6663	BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec;
6664
6665	/*
6666	* We're going to fill the inode item now, so at this point the inode
6667	* must be fully initialized.
6668	*/
6669
6670	inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6671	struct btrfs_inode_item);
6672	memzero_extent_buffer(eb: path->nodes[0], start: (unsigned long)inode_item,
6673	len: sizeof(*inode_item));
6674	fill_inode_item(trans, leaf: path->nodes[0], item: inode_item, inode);
6675
6676	if (!args->orphan) {
6677	ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6678	struct btrfs_inode_ref);
6679	ptr = (unsigned long)(ref + 1);
6680	if (args->subvol) {
6681	btrfs_set_inode_ref_name_len(eb: path->nodes[0], s: ref, val: 2);
6682	btrfs_set_inode_ref_index(eb: path->nodes[0], s: ref, val: 0);
6683	write_extent_buffer(eb: path->nodes[0], src: "..", start: ptr, len: 2);
6684	} else {
6685	btrfs_set_inode_ref_name_len(eb: path->nodes[0], s: ref,
6686	val: name->len);
6687	btrfs_set_inode_ref_index(eb: path->nodes[0], s: ref,
6688	BTRFS_I(inode)->dir_index);
6689	write_extent_buffer(eb: path->nodes[0], src: name->name, start: ptr,
6690	len: name->len);
6691	}
6692	}
6693
6694	/*
6695	* We don't need the path anymore, plus inheriting properties, adding
6696	* ACLs, security xattrs, orphan item or adding the link, will result in
6697	* allocating yet another path. So just free our path.
6698	*/
6699	btrfs_free_path(p: path);
6700	path = NULL;
6701
6702	if (args->subvol) {
6703	struct btrfs_inode *parent;
6704
6705	/*
6706	* Subvolumes inherit properties from their parent subvolume,
6707	* not the directory they were created in.
6708	*/
6709	parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root);
6710	if (IS_ERR(ptr: parent)) {
6711	ret = PTR_ERR(ptr: parent);
6712	} else {
6713	ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode),
6714	dir: parent);
6715	iput(&parent->vfs_inode);
6716	}
6717	} else {
6718	ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode),
6719	BTRFS_I(dir));
6720	}
6721	if (ret) {
6722	btrfs_err(fs_info,
6723	"error inheriting props for ino %llu (root %llu): %d",
6724	btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ret);
6725	}
6726
6727	/*
6728	* Subvolumes don't inherit ACLs or get passed to the LSM. This is
6729	* probably a bug.
6730	*/
6731	if (!args->subvol) {
6732	ret = btrfs_init_inode_security(trans, args);
6733	if (unlikely(ret)) {
6734	btrfs_abort_transaction(trans, ret);
6735	goto discard;
6736	}
6737	}
6738
6739	ret = btrfs_add_inode_to_root(BTRFS_I(inode), prealloc: false);
6740	if (WARN_ON(ret)) {
6741	/* Shouldn't happen, we used xa_reserve() before. */
6742	btrfs_abort_transaction(trans, ret);
6743	goto discard;
6744	}
6745
6746	trace_btrfs_inode_new(inode);
6747	btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
6748
6749	btrfs_update_root_times(trans, root);
6750
6751	if (args->orphan) {
6752	ret = btrfs_orphan_add(trans, BTRFS_I(inode));
6753	if (unlikely(ret)) {
6754	btrfs_abort_transaction(trans, ret);
6755	goto discard;
6756	}
6757	} else {
6758	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name,
6759	add_backref: 0, BTRFS_I(inode)->dir_index);
6760	if (unlikely(ret)) {
6761	btrfs_abort_transaction(trans, ret);
6762	goto discard;
6763	}
6764	}
6765
6766	return 0;
6767
6768	discard:
6769	/*
6770	* discard_new_inode() calls iput(), but the caller owns the reference
6771	* to the inode.
6772	*/
6773	ihold(inode);
6774	discard_new_inode(inode);
6775	out:
6776	if (xa_reserved)
6777	xa_release(xa: &root->inodes, index: objectid);
6778
6779	btrfs_free_path(p: path);
6780	return ret;
6781	}
6782
6783	/*
6784	* utility function to add 'inode' into 'parent_inode' with
6785	* a give name and a given sequence number.
6786	* if 'add_backref' is true, also insert a backref from the
6787	* inode to the parent directory.
6788	*/
6789	int btrfs_add_link(struct btrfs_trans_handle *trans,
6790	struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6791	const struct fscrypt_str *name, bool add_backref, u64 index)
6792	{
6793	int ret = 0;
6794	struct btrfs_key key;
6795	struct btrfs_root *root = parent_inode->root;
6796	u64 ino = btrfs_ino(inode);
6797	u64 parent_ino = btrfs_ino(inode: parent_inode);
6798
6799	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6800	memcpy(&key, &inode->root->root_key, sizeof(key));
6801	} else {
6802	key.objectid = ino;
6803	key.type = BTRFS_INODE_ITEM_KEY;
6804	key.offset = 0;
6805	}
6806
6807	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6808	ret = btrfs_add_root_ref(trans, root_id: key.objectid,
6809	ref_id: btrfs_root_id(root), dirid: parent_ino,
6810	sequence: index, name);
6811	} else if (add_backref) {
6812	ret = btrfs_insert_inode_ref(trans, root, name,
6813	inode_objectid: ino, ref_objectid: parent_ino, index);
6814	}
6815
6816	/* Nothing to clean up yet */
6817	if (ret)
6818	return ret;
6819
6820	ret = btrfs_insert_dir_item(trans, name, dir: parent_inode, location: &key,
6821	type: btrfs_inode_type(inode), index);
6822	if (ret == -EEXIST || ret == -EOVERFLOW)
6823	goto fail_dir_item;
6824	else if (unlikely(ret)) {
6825	btrfs_abort_transaction(trans, ret);
6826	return ret;
6827	}
6828
6829	btrfs_i_size_write(inode: parent_inode, size: parent_inode->vfs_inode.i_size +
6830	name->len * 2);
6831	inode_inc_iversion(inode: &parent_inode->vfs_inode);
6832	update_time_after_link_or_unlink(dir: parent_inode);
6833
6834	ret = btrfs_update_inode(trans, inode: parent_inode);
6835	if (ret)
6836	btrfs_abort_transaction(trans, ret);
6837	return ret;
6838
6839	fail_dir_item:
6840	if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6841	u64 local_index;
6842	int ret2;
6843
6844	ret2 = btrfs_del_root_ref(trans, root_id: key.objectid, ref_id: btrfs_root_id(root),
6845	dirid: parent_ino, sequence: &local_index, name);
6846	if (ret2)
6847	btrfs_abort_transaction(trans, ret2);
6848	} else if (add_backref) {
6849	int ret2;
6850
6851	ret2 = btrfs_del_inode_ref(trans, root, name, inode_objectid: ino, ref_objectid: parent_ino, NULL);
6852	if (ret2)
6853	btrfs_abort_transaction(trans, ret2);
6854	}
6855
6856	/* Return the original error code */
6857	return ret;
6858	}
6859
6860	static int btrfs_create_common(struct inode *dir, struct dentry *dentry,
6861	struct inode *inode)
6862	{
6863	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
6864	struct btrfs_root *root = BTRFS_I(dir)->root;
6865	struct btrfs_new_inode_args new_inode_args = {
6866	.dir = dir,
6867	.dentry = dentry,
6868	.inode = inode,
6869	};
6870	unsigned int trans_num_items;
6871	struct btrfs_trans_handle *trans;
6872	int ret;
6873
6874	ret = btrfs_new_inode_prepare(args: &new_inode_args, trans_num_items: &trans_num_items);
6875	if (ret)
6876	goto out_inode;
6877
6878	trans = btrfs_start_transaction(root, num_items: trans_num_items);
6879	if (IS_ERR(ptr: trans)) {
6880	ret = PTR_ERR(ptr: trans);
6881	goto out_new_inode_args;
6882	}
6883
6884	ret = btrfs_create_new_inode(trans, args: &new_inode_args);
6885	if (!ret) {
6886	if (S_ISDIR(inode->i_mode))
6887	inode->i_opflags |= IOP_FASTPERM_MAY_EXEC;
6888	d_instantiate_new(dentry, inode);
6889	}
6890
6891	btrfs_end_transaction(trans);
6892	btrfs_btree_balance_dirty(fs_info);
6893	out_new_inode_args:
6894	btrfs_new_inode_args_destroy(args: &new_inode_args);
6895	out_inode:
6896	if (ret)
6897	iput(inode);
6898	return ret;
6899	}
6900
6901	static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
6902	struct dentry *dentry, umode_t mode, dev_t rdev)
6903	{
6904	struct inode *inode;
6905
6906	inode = new_inode(sb: dir->i_sb);
6907	if (!inode)
6908	return -ENOMEM;
6909	inode_init_owner(idmap, inode, dir, mode);
6910	inode->i_op = &btrfs_special_inode_operations;
6911	init_special_inode(inode, inode->i_mode, rdev);
6912	return btrfs_create_common(dir, dentry, inode);
6913	}
6914
6915	static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir,
6916	struct dentry *dentry, umode_t mode, bool excl)
6917	{
6918	struct inode *inode;
6919
6920	inode = new_inode(sb: dir->i_sb);
6921	if (!inode)
6922	return -ENOMEM;
6923	inode_init_owner(idmap, inode, dir, mode);
6924	inode->i_fop = &btrfs_file_operations;
6925	inode->i_op = &btrfs_file_inode_operations;
6926	inode->i_mapping->a_ops = &btrfs_aops;
6927	return btrfs_create_common(dir, dentry, inode);
6928	}
6929
6930	static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6931	struct dentry *dentry)
6932	{
6933	struct btrfs_trans_handle *trans = NULL;
6934	struct btrfs_root *root = BTRFS_I(dir)->root;
6935	struct inode *inode = d_inode(dentry: old_dentry);
6936	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
6937	struct fscrypt_name fname;
6938	u64 index;
6939	int ret;
6940
6941	/* do not allow sys_link's with other subvols of the same device */
6942	if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root))
6943	return -EXDEV;
6944
6945	if (inode->i_nlink >= BTRFS_LINK_MAX)
6946	return -EMLINK;
6947
6948	ret = fscrypt_setup_filename(inode: dir, iname: &dentry->d_name, lookup: 0, fname: &fname);
6949	if (ret)
6950	goto fail;
6951
6952	ret = btrfs_set_inode_index(BTRFS_I(dir), index: &index);
6953	if (ret)
6954	goto fail;
6955
6956	/*
6957	* 2 items for inode and inode ref
6958	* 2 items for dir items
6959	* 1 item for parent inode
6960	* 1 item for orphan item deletion if O_TMPFILE
6961	*/
6962	trans = btrfs_start_transaction(root, num_items: inode->i_nlink ? 5 : 6);
6963	if (IS_ERR(ptr: trans)) {
6964	ret = PTR_ERR(ptr: trans);
6965	trans = NULL;
6966	goto fail;
6967	}
6968
6969	/* There are several dir indexes for this inode, clear the cache. */
6970	BTRFS_I(inode)->dir_index = 0ULL;
6971	inode_inc_iversion(inode);
6972	inode_set_ctime_current(inode);
6973
6974	ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6975	name: &fname.disk_name, add_backref: 1, index);
6976	if (ret)
6977	goto fail;
6978
6979	/* Link added now we update the inode item with the new link count. */
6980	inc_nlink(inode);
6981	ret = btrfs_update_inode(trans, BTRFS_I(inode));
6982	if (unlikely(ret)) {
6983	btrfs_abort_transaction(trans, ret);
6984	goto fail;
6985	}
6986
6987	if (inode->i_nlink == 1) {
6988	/*
6989	* If the new hard link count is 1, it's a file created with the
6990	* open(2) O_TMPFILE flag.
6991	*/
6992	ret = btrfs_orphan_del(trans, BTRFS_I(inode));
6993	if (unlikely(ret)) {
6994	btrfs_abort_transaction(trans, ret);
6995	goto fail;
6996	}
6997	}
6998
6999	/* Grab reference for the new dentry passed to d_instantiate(). */
7000	ihold(inode);
7001	d_instantiate(dentry, inode);
7002	btrfs_log_new_name(trans, old_dentry, NULL, old_dir_index: 0, parent: dentry->d_parent);
7003
7004	fail:
7005	fscrypt_free_filename(fname: &fname);
7006	if (trans)
7007	btrfs_end_transaction(trans);
7008	btrfs_btree_balance_dirty(fs_info);
7009	return ret;
7010	}
7011
7012	static struct dentry *btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
7013	struct dentry *dentry, umode_t mode)
7014	{
7015	struct inode *inode;
7016
7017	inode = new_inode(sb: dir->i_sb);
7018	if (!inode)
7019	return ERR_PTR(error: -ENOMEM);
7020	inode_init_owner(idmap, inode, dir, S_IFDIR | mode);
7021	inode->i_op = &btrfs_dir_inode_operations;
7022	inode->i_fop = &btrfs_dir_file_operations;
7023	return ERR_PTR(error: btrfs_create_common(dir, dentry, inode));
7024	}
7025
7026	static noinline int uncompress_inline(struct btrfs_path *path,
7027	struct folio *folio,
7028	struct btrfs_file_extent_item *item)
7029	{
7030	int ret;
7031	struct extent_buffer *leaf = path->nodes[0];
7032	const u32 blocksize = leaf->fs_info->sectorsize;
7033	char *tmp;
7034	size_t max_size;
7035	unsigned long inline_size;
7036	unsigned long ptr;
7037	int compress_type;
7038
7039	compress_type = btrfs_file_extent_compression(eb: leaf, s: item);
7040	max_size = btrfs_file_extent_ram_bytes(eb: leaf, s: item);
7041	inline_size = btrfs_file_extent_inline_item_len(eb: leaf, nr: path->slots[0]);
7042	tmp = kmalloc(inline_size, GFP_NOFS);
7043	if (!tmp)
7044	return -ENOMEM;
7045	ptr = btrfs_file_extent_inline_start(e: item);
7046
7047	read_extent_buffer(eb: leaf, dst: tmp, start: ptr, len: inline_size);
7048
7049	max_size = min_t(unsigned long, blocksize, max_size);
7050	ret = btrfs_decompress(type: compress_type, data_in: tmp, dest_folio: folio, dest_pgoff: 0, srclen: inline_size,
7051	destlen: max_size);
7052
7053	/*
7054	* decompression code contains a memset to fill in any space between the end
7055	* of the uncompressed data and the end of max_size in case the decompressed
7056	* data ends up shorter than ram_bytes. That doesn't cover the hole between
7057	* the end of an inline extent and the beginning of the next block, so we
7058	* cover that region here.
7059	*/
7060
7061	if (max_size < blocksize)
7062	folio_zero_range(folio, start: max_size, length: blocksize - max_size);
7063	kfree(objp: tmp);
7064	return ret;
7065	}
7066
7067	static int read_inline_extent(struct btrfs_path *path, struct folio *folio)
7068	{
7069	const u32 blocksize = path->nodes[0]->fs_info->sectorsize;
7070	struct btrfs_file_extent_item *fi;
7071	void *kaddr;
7072	size_t copy_size;
7073
7074	if (!folio || folio_test_uptodate(folio))
7075	return 0;
7076
7077	ASSERT(folio_pos(folio) == 0);
7078
7079	fi = btrfs_item_ptr(path->nodes[0], path->slots[0],
7080	struct btrfs_file_extent_item);
7081	if (btrfs_file_extent_compression(eb: path->nodes[0], s: fi) != BTRFS_COMPRESS_NONE)
7082	return uncompress_inline(path, folio, item: fi);
7083
7084	copy_size = min_t(u64, blocksize,
7085	btrfs_file_extent_ram_bytes(path->nodes[0], fi));
7086	kaddr = kmap_local_folio(folio, offset: 0);
7087	read_extent_buffer(eb: path->nodes[0], dst: kaddr,
7088	start: btrfs_file_extent_inline_start(e: fi), len: copy_size);
7089	kunmap_local(kaddr);
7090	if (copy_size < blocksize)
7091	folio_zero_range(folio, start: copy_size, length: blocksize - copy_size);
7092	return 0;
7093	}
7094
7095	/*
7096	* Lookup the first extent overlapping a range in a file.
7097	*
7098	* @inode: file to search in
7099	* @page: page to read extent data into if the extent is inline
7100	* @start: file offset
7101	* @len: length of range starting at @start
7102	*
7103	* Return the first &struct extent_map which overlaps the given range, reading
7104	* it from the B-tree and caching it if necessary. Note that there may be more
7105	* extents which overlap the given range after the returned extent_map.
7106	*
7107	* If @page is not NULL and the extent is inline, this also reads the extent
7108	* data directly into the page and marks the extent up to date in the io_tree.
7109	*
7110	* Return: ERR_PTR on error, non-NULL extent_map on success.
7111	*/
7112	struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
7113	struct folio *folio, u64 start, u64 len)
7114	{
7115	struct btrfs_fs_info *fs_info = inode->root->fs_info;
7116	int ret = 0;
7117	u64 extent_start = 0;
7118	u64 extent_end = 0;
7119	u64 objectid = btrfs_ino(inode);
7120	int extent_type = -1;
7121	struct btrfs_path *path = NULL;
7122	struct btrfs_root *root = inode->root;
7123	struct btrfs_file_extent_item *item;
7124	struct extent_buffer *leaf;
7125	struct btrfs_key found_key;
7126	struct extent_map *em = NULL;
7127	struct extent_map_tree *em_tree = &inode->extent_tree;
7128
7129	read_lock(&em_tree->lock);
7130	em = btrfs_lookup_extent_mapping(tree: em_tree, start, len);
7131	read_unlock(&em_tree->lock);
7132
7133	if (em) {
7134	if (em->start > start || em->start + em->len <= start)
7135	btrfs_free_extent_map(em);
7136	else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio)
7137	btrfs_free_extent_map(em);
7138	else
7139	goto out;
7140	}
7141	em = btrfs_alloc_extent_map();
7142	if (!em) {
7143	ret = -ENOMEM;
7144	goto out;
7145	}
7146	em->start = EXTENT_MAP_HOLE;
7147	em->disk_bytenr = EXTENT_MAP_HOLE;
7148	em->len = (u64)-1;
7149
7150	path = btrfs_alloc_path();
7151	if (!path) {
7152	ret = -ENOMEM;
7153	goto out;
7154	}
7155
7156	/* Chances are we'll be called again, so go ahead and do readahead */
7157	path->reada = READA_FORWARD;
7158
7159	/*
7160	* The same explanation in load_free_space_cache applies here as well,
7161	* we only read when we're loading the free space cache, and at that
7162	* point the commit_root has everything we need.
7163	*/
7164	if (btrfs_is_free_space_inode(inode)) {
7165	path->search_commit_root = true;
7166	path->skip_locking = true;
7167	}
7168
7169	ret = btrfs_lookup_file_extent(NULL, root, path, objectid, bytenr: start, mod: 0);
7170	if (ret < 0) {
7171	goto out;
7172	} else if (ret > 0) {
7173	if (path->slots[0] == 0)
7174	goto not_found;
7175	path->slots[0]--;
7176	ret = 0;
7177	}
7178
7179	leaf = path->nodes[0];
7180	item = btrfs_item_ptr(leaf, path->slots[0],
7181	struct btrfs_file_extent_item);
7182	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
7183	if (found_key.objectid != objectid ||
7184	found_key.type != BTRFS_EXTENT_DATA_KEY) {
7185	/*
7186	* If we backup past the first extent we want to move forward
7187	* and see if there is an extent in front of us, otherwise we'll
7188	* say there is a hole for our whole search range which can
7189	* cause problems.
7190	*/
7191	extent_end = start;
7192	goto next;
7193	}
7194
7195	extent_type = btrfs_file_extent_type(eb: leaf, s: item);
7196	extent_start = found_key.offset;
7197	extent_end = btrfs_file_extent_end(path);
7198	if (extent_type == BTRFS_FILE_EXTENT_REG ||
7199	extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7200	/* Only regular file could have regular/prealloc extent */
7201	if (unlikely(!S_ISREG(inode->vfs_inode.i_mode))) {
7202	ret = -EUCLEAN;
7203	btrfs_crit(fs_info,
7204	"regular/prealloc extent found for non-regular inode %llu",
7205	btrfs_ino(inode));
7206	goto out;
7207	}
7208	trace_btrfs_get_extent_show_fi_regular(bi: inode, l: leaf, fi: item,
7209	start: extent_start);
7210	} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7211	trace_btrfs_get_extent_show_fi_inline(bi: inode, l: leaf, fi: item,
7212	slot: path->slots[0],
7213	start: extent_start);
7214	}
7215	next:
7216	if (start >= extent_end) {
7217	path->slots[0]++;
7218	if (path->slots[0] >= btrfs_header_nritems(eb: leaf)) {
7219	ret = btrfs_next_leaf(root, path);
7220	if (ret < 0)
7221	goto out;
7222	else if (ret > 0)
7223	goto not_found;
7224
7225	leaf = path->nodes[0];
7226	}
7227	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &found_key, nr: path->slots[0]);
7228	if (found_key.objectid != objectid ||
7229	found_key.type != BTRFS_EXTENT_DATA_KEY)
7230	goto not_found;
7231	if (start + len <= found_key.offset)
7232	goto not_found;
7233	if (start > found_key.offset)
7234	goto next;
7235
7236	/* New extent overlaps with existing one */
7237	em->start = start;
7238	em->len = found_key.offset - start;
7239	em->disk_bytenr = EXTENT_MAP_HOLE;
7240	goto insert;
7241	}
7242
7243	btrfs_extent_item_to_extent_map(inode, path, fi: item, em);
7244
7245	if (extent_type == BTRFS_FILE_EXTENT_REG ||
7246	extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
7247	goto insert;
7248	} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
7249	/*
7250	* Inline extent can only exist at file offset 0. This is
7251	* ensured by tree-checker and inline extent creation path.
7252	* Thus all members representing file offsets should be zero.
7253	*/
7254	ASSERT(extent_start == 0);
7255	ASSERT(em->start == 0);
7256
7257	/*
7258	* btrfs_extent_item_to_extent_map() should have properly
7259	* initialized em members already.
7260	*
7261	* Other members are not utilized for inline extents.
7262	*/
7263	ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE);
7264	ASSERT(em->len == fs_info->sectorsize);
7265
7266	ret = read_inline_extent(path, folio);
7267	if (ret < 0)
7268	goto out;
7269	goto insert;
7270	}
7271	not_found:
7272	em->start = start;
7273	em->len = len;
7274	em->disk_bytenr = EXTENT_MAP_HOLE;
7275	insert:
7276	ret = 0;
7277	btrfs_release_path(p: path);
7278	if (unlikely(em->start > start || btrfs_extent_map_end(em) <= start)) {
7279	btrfs_err(fs_info,
7280	"bad extent! em: [%llu %llu] passed [%llu %llu]",
7281	em->start, em->len, start, len);
7282	ret = -EIO;
7283	goto out;
7284	}
7285
7286	write_lock(&em_tree->lock);
7287	ret = btrfs_add_extent_mapping(inode, em_in: &em, start, len);
7288	write_unlock(&em_tree->lock);
7289	out:
7290	btrfs_free_path(p: path);
7291
7292	trace_btrfs_get_extent(root, inode, map: em);
7293
7294	if (ret) {
7295	btrfs_free_extent_map(em);
7296	return ERR_PTR(error: ret);
7297	}
7298	return em;
7299	}
7300
7301	static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
7302	{
7303	struct btrfs_block_group *block_group;
7304	bool readonly = false;
7305
7306	block_group = btrfs_lookup_block_group(info: fs_info, bytenr);
7307	if (!block_group || block_group->ro)
7308	readonly = true;
7309	if (block_group)
7310	btrfs_put_block_group(cache: block_group);
7311	return readonly;
7312	}
7313
7314	/*
7315	* Check if we can do nocow write into the range [@offset, @offset + @len)
7316	*
7317	* @offset: File offset
7318	* @len: The length to write, will be updated to the nocow writeable
7319	* range
7320	* @orig_start: (optional) Return the original file offset of the file extent
7321	* @orig_len: (optional) Return the original on-disk length of the file extent
7322	* @ram_bytes: (optional) Return the ram_bytes of the file extent
7323	*
7324	* Return:
7325	* >0 and update @len if we can do nocow write
7326	* 0 if we can't do nocow write
7327	* <0 if error happened
7328	*
7329	* NOTE: This only checks the file extents, caller is responsible to wait for
7330	* any ordered extents.
7331	*/
7332	noinline int can_nocow_extent(struct btrfs_inode *inode, u64 offset, u64 *len,
7333	struct btrfs_file_extent *file_extent,
7334	bool nowait)
7335	{
7336	struct btrfs_root *root = inode->root;
7337	struct btrfs_fs_info *fs_info = root->fs_info;
7338	struct can_nocow_file_extent_args nocow_args = { 0 };
7339	BTRFS_PATH_AUTO_FREE(path);
7340	int ret;
7341	struct extent_buffer *leaf;
7342	struct extent_io_tree *io_tree = &inode->io_tree;
7343	struct btrfs_file_extent_item *fi;
7344	struct btrfs_key key;
7345	int found_type;
7346
7347	path = btrfs_alloc_path();
7348	if (!path)
7349	return -ENOMEM;
7350	path->nowait = nowait;
7351
7352	ret = btrfs_lookup_file_extent(NULL, root, path, objectid: btrfs_ino(inode),
7353	bytenr: offset, mod: 0);
7354	if (ret < 0)
7355	return ret;
7356
7357	if (ret == 1) {
7358	if (path->slots[0] == 0) {
7359	/* Can't find the item, must COW. */
7360	return 0;
7361	}
7362	path->slots[0]--;
7363	}
7364	ret = 0;
7365	leaf = path->nodes[0];
7366	btrfs_item_key_to_cpu(eb: leaf, cpu_key: &key, nr: path->slots[0]);
7367	if (key.objectid != btrfs_ino(inode) ||
7368	key.type != BTRFS_EXTENT_DATA_KEY) {
7369	/* Not our file or wrong item type, must COW. */
7370	return 0;
7371	}
7372
7373	if (key.offset > offset) {
7374	/* Wrong offset, must COW. */
7375	return 0;
7376	}
7377
7378	if (btrfs_file_extent_end(path) <= offset)
7379	return 0;
7380
7381	fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
7382	found_type = btrfs_file_extent_type(eb: leaf, s: fi);
7383
7384	nocow_args.start = offset;
7385	nocow_args.end = offset + *len - 1;
7386	nocow_args.free_path = true;
7387
7388	ret = can_nocow_file_extent(path, key: &key, inode, args: &nocow_args);
7389	/* can_nocow_file_extent() has freed the path. */
7390	path = NULL;
7391
7392	if (ret != 1) {
7393	/* Treat errors as not being able to NOCOW. */
7394	return 0;
7395	}
7396
7397	if (btrfs_extent_readonly(fs_info,
7398	bytenr: nocow_args.file_extent.disk_bytenr +
7399	nocow_args.file_extent.offset))
7400	return 0;
7401
7402	if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
7403	found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7404	u64 range_end;
7405
7406	range_end = round_up(offset + nocow_args.file_extent.num_bytes,
7407	root->fs_info->sectorsize) - 1;
7408	ret = btrfs_test_range_bit_exists(tree: io_tree, start: offset, end: range_end,
7409	bit: EXTENT_DELALLOC);
7410	if (ret)
7411	return -EAGAIN;
7412	}
7413
7414	if (file_extent)
7415	memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent));
7416
7417	*len = nocow_args.file_extent.num_bytes;
7418
7419	return 1;
7420	}
7421
7422	/* The callers of this must take lock_extent() */
7423	struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start,
7424	const struct btrfs_file_extent *file_extent,
7425	int type)
7426	{
7427	struct extent_map *em;
7428	int ret;
7429
7430	/*
7431	* Note the missing NOCOW type.
7432	*
7433	* For pure NOCOW writes, we should not create an io extent map, but
7434	* just reusing the existing one.
7435	* Only PREALLOC writes (NOCOW write into preallocated range) can
7436	* create an io extent map.
7437	*/
7438	ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7439	type == BTRFS_ORDERED_COMPRESSED ||
7440	type == BTRFS_ORDERED_REGULAR);
7441
7442	switch (type) {
7443	case BTRFS_ORDERED_PREALLOC:
7444	/* We're only referring part of a larger preallocated extent. */
7445	ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7446	break;
7447	case BTRFS_ORDERED_REGULAR:
7448	/* COW results a new extent matching our file extent size. */
7449	ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes);
7450	ASSERT(file_extent->ram_bytes == file_extent->num_bytes);
7451
7452	/* Since it's a new extent, we should not have any offset. */
7453	ASSERT(file_extent->offset == 0);
7454	break;
7455	case BTRFS_ORDERED_COMPRESSED:
7456	/* Must be compressed. */
7457	ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE);
7458
7459	/*
7460	* Encoded write can make us to refer to part of the
7461	* uncompressed extent.
7462	*/
7463	ASSERT(file_extent->num_bytes <= file_extent->ram_bytes);
7464	break;
7465	}
7466
7467	em = btrfs_alloc_extent_map();
7468	if (!em)
7469	return ERR_PTR(error: -ENOMEM);
7470
7471	em->start = start;
7472	em->len = file_extent->num_bytes;
7473	em->disk_bytenr = file_extent->disk_bytenr;
7474	em->disk_num_bytes = file_extent->disk_num_bytes;
7475	em->ram_bytes = file_extent->ram_bytes;
7476	em->generation = -1;
7477	em->offset = file_extent->offset;
7478	em->flags |= EXTENT_FLAG_PINNED;
7479	if (type == BTRFS_ORDERED_COMPRESSED)
7480	btrfs_extent_map_set_compression(em, type: file_extent->compression);
7481
7482	ret = btrfs_replace_extent_map_range(inode, new_em: em, modified: true);
7483	if (ret) {
7484	btrfs_free_extent_map(em);
7485	return ERR_PTR(error: ret);
7486	}
7487
7488	/* em got 2 refs now, callers needs to do btrfs_free_extent_map once. */
7489	return em;
7490	}
7491
7492	/*
7493	* For release_folio() and invalidate_folio() we have a race window where
7494	* folio_end_writeback() is called but the subpage spinlock is not yet released.
7495	* If we continue to release/invalidate the page, we could cause use-after-free
7496	* for subpage spinlock. So this function is to spin and wait for subpage
7497	* spinlock.
7498	*/
7499	static void wait_subpage_spinlock(struct folio *folio)
7500	{
7501	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
7502	struct btrfs_folio_state *bfs;
7503
7504	if (!btrfs_is_subpage(fs_info, folio))
7505	return;
7506
7507	ASSERT(folio_test_private(folio) && folio_get_private(folio));
7508	bfs = folio_get_private(folio);
7509
7510	/*
7511	* This may look insane as we just acquire the spinlock and release it,
7512	* without doing anything. But we just want to make sure no one is
7513	* still holding the subpage spinlock.
7514	* And since the page is not dirty nor writeback, and we have page
7515	* locked, the only possible way to hold a spinlock is from the endio
7516	* function to clear page writeback.
7517	*
7518	* Here we just acquire the spinlock so that all existing callers
7519	* should exit and we're safe to release/invalidate the page.
7520	*/
7521	spin_lock_irq(lock: &bfs->lock);
7522	spin_unlock_irq(lock: &bfs->lock);
7523	}
7524
7525	static int btrfs_launder_folio(struct folio *folio)
7526	{
7527	return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, start: folio_pos(folio),
7528	len: folio_size(folio), NULL);
7529	}
7530
7531	static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7532	{
7533	if (try_release_extent_mapping(folio, mask: gfp_flags)) {
7534	wait_subpage_spinlock(folio);
7535	clear_folio_extent_mapped(folio);
7536	return true;
7537	}
7538	return false;
7539	}
7540
7541	static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags)
7542	{
7543	if (folio_test_writeback(folio) || folio_test_dirty(folio))
7544	return false;
7545	return __btrfs_release_folio(folio, gfp_flags);
7546	}
7547
7548	#ifdef CONFIG_MIGRATION
7549	static int btrfs_migrate_folio(struct address_space *mapping,
7550	struct folio *dst, struct folio *src,
7551	enum migrate_mode mode)
7552	{
7553	int ret = filemap_migrate_folio(mapping, dst, src, mode);
7554
7555	if (ret)
7556	return ret;
7557
7558	if (folio_test_ordered(src)) {
7559	folio_clear_ordered(src);
7560	folio_set_ordered(dst);
7561	}
7562
7563	return 0;
7564	}
7565	#else
7566	#define btrfs_migrate_folio NULL
7567	#endif
7568
7569	static void btrfs_invalidate_folio(struct folio *folio, size_t offset,
7570	size_t length)
7571	{
7572	struct btrfs_inode *inode = folio_to_inode(folio);
7573	struct btrfs_fs_info *fs_info = inode->root->fs_info;
7574	struct extent_io_tree *tree = &inode->io_tree;
7575	struct extent_state *cached_state = NULL;
7576	u64 page_start = folio_pos(folio);
7577	u64 page_end = page_start + folio_size(folio) - 1;
7578	u64 cur;
7579	int inode_evicting = inode_state_read_once(inode: &inode->vfs_inode) & I_FREEING;
7580
7581	/*
7582	* We have folio locked so no new ordered extent can be created on this
7583	* page, nor bio can be submitted for this folio.
7584	*
7585	* But already submitted bio can still be finished on this folio.
7586	* Furthermore, endio function won't skip folio which has Ordered
7587	* already cleared, so it's possible for endio and
7588	* invalidate_folio to do the same ordered extent accounting twice
7589	* on one folio.
7590	*
7591	* So here we wait for any submitted bios to finish, so that we won't
7592	* do double ordered extent accounting on the same folio.
7593	*/
7594	folio_wait_writeback(folio);
7595	wait_subpage_spinlock(folio);
7596
7597	/*
7598	* For subpage case, we have call sites like
7599	* btrfs_punch_hole_lock_range() which passes range not aligned to
7600	* sectorsize.
7601	* If the range doesn't cover the full folio, we don't need to and
7602	* shouldn't clear page extent mapped, as folio->private can still
7603	* record subpage dirty bits for other part of the range.
7604	*
7605	* For cases that invalidate the full folio even the range doesn't
7606	* cover the full folio, like invalidating the last folio, we're
7607	* still safe to wait for ordered extent to finish.
7608	*/
7609	if (!(offset == 0 && length == folio_size(folio))) {
7610	btrfs_release_folio(folio, GFP_NOFS);
7611	return;
7612	}
7613
7614	if (!inode_evicting)
7615	btrfs_lock_extent(tree, start: page_start, end: page_end, cached: &cached_state);
7616
7617	cur = page_start;
7618	while (cur < page_end) {
7619	struct btrfs_ordered_extent *ordered;
7620	u64 range_end;
7621	u32 range_len;
7622	u32 extra_flags = 0;
7623
7624	ordered = btrfs_lookup_first_ordered_range(inode, file_offset: cur,
7625	len: page_end + 1 - cur);
7626	if (!ordered) {
7627	range_end = page_end;
7628	/*
7629	* No ordered extent covering this range, we are safe
7630	* to delete all extent states in the range.
7631	*/
7632	extra_flags = EXTENT_CLEAR_ALL_BITS;
7633	goto next;
7634	}
7635	if (ordered->file_offset > cur) {
7636	/*
7637	* There is a range between [cur, oe->file_offset) not
7638	* covered by any ordered extent.
7639	* We are safe to delete all extent states, and handle
7640	* the ordered extent in the next iteration.
7641	*/
7642	range_end = ordered->file_offset - 1;
7643	extra_flags = EXTENT_CLEAR_ALL_BITS;
7644	goto next;
7645	}
7646
7647	range_end = min(ordered->file_offset + ordered->num_bytes - 1,
7648	page_end);
7649	ASSERT(range_end + 1 - cur < U32_MAX);
7650	range_len = range_end + 1 - cur;
7651	if (!btrfs_folio_test_ordered(fs_info, folio, start: cur, len: range_len)) {
7652	/*
7653	* If Ordered is cleared, it means endio has
7654	* already been executed for the range.
7655	* We can't delete the extent states as
7656	* btrfs_finish_ordered_io() may still use some of them.
7657	*/
7658	goto next;
7659	}
7660	btrfs_folio_clear_ordered(fs_info, folio, start: cur, len: range_len);
7661
7662	/*
7663	* IO on this page will never be started, so we need to account
7664	* for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW
7665	* here, must leave that up for the ordered extent completion.
7666	*
7667	* This will also unlock the range for incoming
7668	* btrfs_finish_ordered_io().
7669	*/
7670	if (!inode_evicting)
7671	btrfs_clear_extent_bit(tree, start: cur, end: range_end,
7672	bits: EXTENT_DELALLOC |
7673	EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
7674	EXTENT_DEFRAG, cached: &cached_state);
7675
7676	spin_lock(lock: &inode->ordered_tree_lock);
7677	set_bit(nr: BTRFS_ORDERED_TRUNCATED, addr: &ordered->flags);
7678	ordered->truncated_len = min(ordered->truncated_len,
7679	cur - ordered->file_offset);
7680	spin_unlock(lock: &inode->ordered_tree_lock);
7681
7682	/*
7683	* If the ordered extent has finished, we're safe to delete all
7684	* the extent states of the range, otherwise
7685	* btrfs_finish_ordered_io() will get executed by endio for
7686	* other pages, so we can't delete extent states.
7687	*/
7688	if (btrfs_dec_test_ordered_pending(inode, cached: &ordered,
7689	file_offset: cur, io_size: range_end + 1 - cur)) {
7690	btrfs_finish_ordered_io(ordered);
7691	/*
7692	* The ordered extent has finished, now we're again
7693	* safe to delete all extent states of the range.
7694	*/
7695	extra_flags = EXTENT_CLEAR_ALL_BITS;
7696	}
7697	next:
7698	if (ordered)
7699	btrfs_put_ordered_extent(entry: ordered);
7700	/*
7701	* Qgroup reserved space handler
7702	* Sector(s) here will be either:
7703	*
7704	* 1) Already written to disk or bio already finished
7705	* Then its QGROUP_RESERVED bit in io_tree is already cleared.
7706	* Qgroup will be handled by its qgroup_record then.
7707	* btrfs_qgroup_free_data() call will do nothing here.
7708	*
7709	* 2) Not written to disk yet
7710	* Then btrfs_qgroup_free_data() call will clear the
7711	* QGROUP_RESERVED bit of its io_tree, and free the qgroup
7712	* reserved data space.
7713	* Since the IO will never happen for this page.
7714	*/
7715	btrfs_qgroup_free_data(inode, NULL, start: cur, len: range_end + 1 - cur, NULL);
7716	if (!inode_evicting)
7717	btrfs_clear_extent_bit(tree, start: cur, end: range_end, bits: EXTENT_LOCKED |
7718	EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
7719	EXTENT_DEFRAG | extra_flags,
7720	cached: &cached_state);
7721	cur = range_end + 1;
7722	}
7723	/*
7724	* We have iterated through all ordered extents of the page, the page
7725	* should not have Ordered anymore, or the above iteration
7726	* did something wrong.
7727	*/
7728	ASSERT(!folio_test_ordered(folio));
7729	btrfs_folio_clear_checked(fs_info, folio, start: folio_pos(folio), len: folio_size(folio));
7730	if (!inode_evicting)
7731	__btrfs_release_folio(folio, GFP_NOFS);
7732	clear_folio_extent_mapped(folio);
7733	}
7734
7735	static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback)
7736	{
7737	struct btrfs_truncate_control control = {
7738	.inode = inode,
7739	.ino = btrfs_ino(inode),
7740	.min_type = BTRFS_EXTENT_DATA_KEY,
7741	.clear_extent_range = true,
7742	.new_size = inode->vfs_inode.i_size,
7743	};
7744	struct btrfs_root *root = inode->root;
7745	struct btrfs_fs_info *fs_info = root->fs_info;
7746	struct btrfs_block_rsv rsv;
7747	int ret;
7748	struct btrfs_trans_handle *trans;
7749	const u64 min_size = btrfs_calc_metadata_size(fs_info, num_items: 1);
7750	const u64 lock_start = round_down(inode->vfs_inode.i_size, fs_info->sectorsize);
7751	const u64 i_size_up = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
7752
7753	/* Our inode is locked and the i_size can't be changed concurrently. */
7754	btrfs_assert_inode_locked(inode);
7755
7756	if (!skip_writeback) {
7757	ret = btrfs_wait_ordered_range(inode, start: lock_start, len: (u64)-1);
7758	if (ret)
7759	return ret;
7760	}
7761
7762	/*
7763	* Yes ladies and gentlemen, this is indeed ugly. We have a couple of
7764	* things going on here:
7765	*
7766	* 1) We need to reserve space to update our inode.
7767	*
7768	* 2) We need to have something to cache all the space that is going to
7769	* be free'd up by the truncate operation, but also have some slack
7770	* space reserved in case it uses space during the truncate (thank you
7771	* very much snapshotting).
7772	*
7773	* And we need these to be separate. The fact is we can use a lot of
7774	* space doing the truncate, and we have no earthly idea how much space
7775	* we will use, so we need the truncate reservation to be separate so it
7776	* doesn't end up using space reserved for updating the inode. We also
7777	* need to be able to stop the transaction and start a new one, which
7778	* means we need to be able to update the inode several times, and we
7779	* have no idea of knowing how many times that will be, so we can't just
7780	* reserve 1 item for the entirety of the operation, so that has to be
7781	* done separately as well.
7782	*
7783	* So that leaves us with
7784	*
7785	* 1) rsv - for the truncate reservation, which we will steal from the
7786	* transaction reservation.
7787	* 2) fs_info->trans_block_rsv - this will have 1 items worth left for
7788	* updating the inode.
7789	*/
7790	btrfs_init_metadata_block_rsv(fs_info, rsv: &rsv, type: BTRFS_BLOCK_RSV_TEMP);
7791	rsv.size = min_size;
7792	rsv.failfast = true;
7793
7794	/*
7795	* 1 for the truncate slack space
7796	* 1 for updating the inode.
7797	*/
7798	trans = btrfs_start_transaction(root, num_items: 2);
7799	if (IS_ERR(ptr: trans)) {
7800	ret = PTR_ERR(ptr: trans);
7801	goto out;
7802	}
7803
7804	/* Migrate the slack space for the truncate to our reserve */
7805	ret = btrfs_block_rsv_migrate(src_rsv: &fs_info->trans_block_rsv, dst_rsv: &rsv,
7806	num_bytes: min_size, update_size: false);
7807	/*
7808	* We have reserved 2 metadata units when we started the transaction and
7809	* min_size matches 1 unit, so this should never fail, but if it does,
7810	* it's not critical we just fail truncation.
7811	*/
7812	if (WARN_ON(ret)) {
7813	btrfs_end_transaction(trans);
7814	goto out;
7815	}
7816
7817	trans->block_rsv = &rsv;
7818
7819	while (1) {
7820	struct extent_state *cached_state = NULL;
7821
7822	btrfs_lock_extent(tree: &inode->io_tree, start: lock_start, end: (u64)-1, cached: &cached_state);
7823	/*
7824	* We want to drop from the next block forward in case this new
7825	* size is not block aligned since we will be keeping the last
7826	* block of the extent just the way it is.
7827	*/
7828	btrfs_drop_extent_map_range(inode, start: i_size_up, end: (u64)-1, skip_pinned: false);
7829
7830	ret = btrfs_truncate_inode_items(trans, root, control: &control);
7831
7832	inode_sub_bytes(inode: &inode->vfs_inode, bytes: control.sub_bytes);
7833	btrfs_inode_safe_disk_i_size_write(inode, new_i_size: control.last_size);
7834
7835	btrfs_unlock_extent(tree: &inode->io_tree, start: lock_start, end: (u64)-1, cached: &cached_state);
7836
7837	trans->block_rsv = &fs_info->trans_block_rsv;
7838	if (ret != -ENOSPC && ret != -EAGAIN)
7839	break;
7840
7841	ret = btrfs_update_inode(trans, inode);
7842	if (ret)
7843	break;
7844
7845	btrfs_end_transaction(trans);
7846	btrfs_btree_balance_dirty(fs_info);
7847
7848	trans = btrfs_start_transaction(root, num_items: 2);
7849	if (IS_ERR(ptr: trans)) {
7850	ret = PTR_ERR(ptr: trans);
7851	trans = NULL;
7852	break;
7853	}
7854
7855	btrfs_block_rsv_release(fs_info, block_rsv: &rsv, num_bytes: -1, NULL);
7856	ret = btrfs_block_rsv_migrate(src_rsv: &fs_info->trans_block_rsv,
7857	dst_rsv: &rsv, num_bytes: min_size, update_size: false);
7858	/*
7859	* We have reserved 2 metadata units when we started the
7860	* transaction and min_size matches 1 unit, so this should never
7861	* fail, but if it does, it's not critical we just fail truncation.
7862	*/
7863	if (WARN_ON(ret))
7864	break;
7865
7866	trans->block_rsv = &rsv;
7867	}
7868
7869	/*
7870	* We can't call btrfs_truncate_block inside a trans handle as we could
7871	* deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we
7872	* know we've truncated everything except the last little bit, and can
7873	* do btrfs_truncate_block and then update the disk_i_size.
7874	*/
7875	if (ret == BTRFS_NEED_TRUNCATE_BLOCK) {
7876	btrfs_end_transaction(trans);
7877	btrfs_btree_balance_dirty(fs_info);
7878
7879	ret = btrfs_truncate_block(inode, offset: inode->vfs_inode.i_size,
7880	start: inode->vfs_inode.i_size, end: (u64)-1);
7881	if (ret)
7882	goto out;
7883	trans = btrfs_start_transaction(root, num_items: 1);
7884	if (IS_ERR(ptr: trans)) {
7885	ret = PTR_ERR(ptr: trans);
7886	goto out;
7887	}
7888	btrfs_inode_safe_disk_i_size_write(inode, new_i_size: 0);
7889	}
7890
7891	if (trans) {
7892	int ret2;
7893
7894	trans->block_rsv = &fs_info->trans_block_rsv;
7895	ret2 = btrfs_update_inode(trans, inode);
7896	if (ret2 && !ret)
7897	ret = ret2;
7898
7899	ret2 = btrfs_end_transaction(trans);
7900	if (ret2 && !ret)
7901	ret = ret2;
7902	btrfs_btree_balance_dirty(fs_info);
7903	}
7904	out:
7905	btrfs_block_rsv_release(fs_info, block_rsv: &rsv, num_bytes: (u64)-1, NULL);
7906	/*
7907	* So if we truncate and then write and fsync we normally would just
7908	* write the extents that changed, which is a problem if we need to
7909	* first truncate that entire inode. So set this flag so we write out
7910	* all of the extents in the inode to the sync log so we're completely
7911	* safe.
7912	*
7913	* If no extents were dropped or trimmed we don't need to force the next
7914	* fsync to truncate all the inode's items from the log and re-log them
7915	* all. This means the truncate operation did not change the file size,
7916	* or changed it to a smaller size but there was only an implicit hole
7917	* between the old i_size and the new i_size, and there were no prealloc
7918	* extents beyond i_size to drop.
7919	*/
7920	if (control.extents_found > 0)
7921	btrfs_set_inode_full_sync(inode);
7922
7923	return ret;
7924	}
7925
7926	struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap,
7927	struct inode *dir)
7928	{
7929	struct inode *inode;
7930
7931	inode = new_inode(sb: dir->i_sb);
7932	if (inode) {
7933	/*
7934	* Subvolumes don't inherit the sgid bit or the parent's gid if
7935	* the parent's sgid bit is set. This is probably a bug.
7936	*/
7937	inode_init_owner(idmap, inode, NULL,
7938	S_IFDIR | (~current_umask() & S_IRWXUGO));
7939	inode->i_op = &btrfs_dir_inode_operations;
7940	inode->i_fop = &btrfs_dir_file_operations;
7941	}
7942	return inode;
7943	}
7944
7945	struct inode *btrfs_alloc_inode(struct super_block *sb)
7946	{
7947	struct btrfs_fs_info *fs_info = btrfs_sb(sb);
7948	struct btrfs_inode *ei;
7949	struct inode *inode;
7950
7951	ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL);
7952	if (!ei)
7953	return NULL;
7954
7955	ei->root = NULL;
7956	ei->generation = 0;
7957	ei->last_trans = 0;
7958	ei->last_sub_trans = 0;
7959	ei->logged_trans = 0;
7960	ei->delalloc_bytes = 0;
7961	/* new_delalloc_bytes and last_dir_index_offset are in a union. */
7962	ei->new_delalloc_bytes = 0;
7963	ei->defrag_bytes = 0;
7964	ei->disk_i_size = 0;
7965	ei->flags = 0;
7966	ei->ro_flags = 0;
7967	/*
7968	* ->index_cnt will be properly initialized later when creating a new
7969	* inode (btrfs_create_new_inode()) or when reading an existing inode
7970	* from disk (btrfs_read_locked_inode()).
7971	*/
7972	ei->csum_bytes = 0;
7973	ei->dir_index = 0;
7974	ei->last_unlink_trans = 0;
7975	ei->last_reflink_trans = 0;
7976	ei->last_log_commit = 0;
7977
7978	spin_lock_init(&ei->lock);
7979	ei->outstanding_extents = 0;
7980	if (sb->s_magic != BTRFS_TEST_MAGIC)
7981	btrfs_init_metadata_block_rsv(fs_info, rsv: &ei->block_rsv,
7982	type: BTRFS_BLOCK_RSV_DELALLOC);
7983	ei->runtime_flags = 0;
7984	ei->prop_compress = BTRFS_COMPRESS_NONE;
7985	ei->defrag_compress = BTRFS_COMPRESS_NONE;
7986
7987	ei->delayed_node = NULL;
7988
7989	ei->i_otime_sec = 0;
7990	ei->i_otime_nsec = 0;
7991
7992	inode = &ei->vfs_inode;
7993	btrfs_extent_map_tree_init(tree: &ei->extent_tree);
7994
7995	/* This io tree sets the valid inode. */
7996	btrfs_extent_io_tree_init(fs_info, tree: &ei->io_tree, owner: IO_TREE_INODE_IO);
7997	ei->io_tree.inode = ei;
7998
7999	ei->file_extent_tree = NULL;
8000
8001	mutex_init(&ei->log_mutex);
8002	spin_lock_init(&ei->ordered_tree_lock);
8003	ei->ordered_tree = RB_ROOT;
8004	ei->ordered_tree_last = NULL;
8005	INIT_LIST_HEAD(list: &ei->delalloc_inodes);
8006	INIT_LIST_HEAD(list: &ei->delayed_iput);
8007	init_rwsem(&ei->i_mmap_lock);
8008
8009	return inode;
8010	}
8011
8012	#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8013	void btrfs_test_destroy_inode(struct inode *inode)
8014	{
8015	btrfs_drop_extent_map_range(BTRFS_I(inode), start: 0, end: (u64)-1, skip_pinned: false);
8016	kfree(BTRFS_I(inode)->file_extent_tree);
8017	kmem_cache_free(s: btrfs_inode_cachep, BTRFS_I(inode));
8018	}
8019	#endif
8020
8021	void btrfs_free_inode(struct inode *inode)
8022	{
8023	kfree(BTRFS_I(inode)->file_extent_tree);
8024	kmem_cache_free(s: btrfs_inode_cachep, BTRFS_I(inode));
8025	}
8026
8027	void btrfs_destroy_inode(struct inode *vfs_inode)
8028	{
8029	struct btrfs_ordered_extent *ordered;
8030	struct btrfs_inode *inode = BTRFS_I(vfs_inode);
8031	struct btrfs_root *root = inode->root;
8032	bool freespace_inode;
8033
8034	WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
8035	WARN_ON(vfs_inode->i_data.nrpages);
8036	WARN_ON(inode->block_rsv.reserved);
8037	WARN_ON(inode->block_rsv.size);
8038	WARN_ON(inode->outstanding_extents);
8039	if (!S_ISDIR(vfs_inode->i_mode)) {
8040	WARN_ON(inode->delalloc_bytes);
8041	WARN_ON(inode->new_delalloc_bytes);
8042	WARN_ON(inode->csum_bytes);
8043	}
8044	if (!root || !btrfs_is_data_reloc_root(root))
8045	WARN_ON(inode->defrag_bytes);
8046
8047	/*
8048	* This can happen where we create an inode, but somebody else also
8049	* created the same inode and we need to destroy the one we already
8050	* created.
8051	*/
8052	if (!root)
8053	return;
8054
8055	/*
8056	* If this is a free space inode do not take the ordered extents lockdep
8057	* map.
8058	*/
8059	freespace_inode = btrfs_is_free_space_inode(inode);
8060
8061	while (1) {
8062	ordered = btrfs_lookup_first_ordered_extent(inode, file_offset: (u64)-1);
8063	if (!ordered)
8064	break;
8065	else {
8066	btrfs_err(root->fs_info,
8067	"found ordered extent %llu %llu on inode cleanup",
8068	ordered->file_offset, ordered->num_bytes);
8069
8070	if (!freespace_inode)
8071	btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent);
8072
8073	btrfs_remove_ordered_extent(btrfs_inode: inode, entry: ordered);
8074	btrfs_put_ordered_extent(entry: ordered);
8075	btrfs_put_ordered_extent(entry: ordered);
8076	}
8077	}
8078	btrfs_qgroup_check_reserved_leak(inode);
8079	btrfs_del_inode_from_root(inode);
8080	btrfs_drop_extent_map_range(inode, start: 0, end: (u64)-1, skip_pinned: false);
8081	btrfs_inode_clear_file_extent_range(inode, start: 0, len: (u64)-1);
8082	btrfs_put_root(root: inode->root);
8083	}
8084
8085	int btrfs_drop_inode(struct inode *inode)
8086	{
8087	struct btrfs_root *root = BTRFS_I(inode)->root;
8088
8089	if (root == NULL)
8090	return 1;
8091
8092	/* the snap/subvol tree is on deleting */
8093	if (btrfs_root_refs(s: &root->root_item) == 0)
8094	return 1;
8095	else
8096	return inode_generic_drop(inode);
8097	}
8098
8099	static void init_once(void *foo)
8100	{
8101	struct btrfs_inode *ei = foo;
8102
8103	inode_init_once(&ei->vfs_inode);
8104	#ifdef CONFIG_FS_VERITY
8105	ei->i_verity_info = NULL;
8106	#endif
8107	}
8108
8109	void __cold btrfs_destroy_cachep(void)
8110	{
8111	/*
8112	* Make sure all delayed rcu free inodes are flushed before we
8113	* destroy cache.
8114	*/
8115	rcu_barrier();
8116	kmem_cache_destroy(s: btrfs_inode_cachep);
8117	}
8118
8119	int __init btrfs_init_cachep(void)
8120	{
8121	btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
8122	sizeof(struct btrfs_inode), 0,
8123	SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
8124	init_once);
8125	if (!btrfs_inode_cachep)
8126	return -ENOMEM;
8127
8128	return 0;
8129	}
8130
8131	static int btrfs_getattr(struct mnt_idmap *idmap,
8132	const struct path *path, struct kstat *stat,
8133	u32 request_mask, unsigned int flags)
8134	{
8135	u64 delalloc_bytes;
8136	u64 inode_bytes;
8137	struct inode *inode = d_inode(dentry: path->dentry);
8138	u32 blocksize = btrfs_sb(sb: inode->i_sb)->sectorsize;
8139	u32 bi_flags = BTRFS_I(inode)->flags;
8140	u32 bi_ro_flags = BTRFS_I(inode)->ro_flags;
8141
8142	stat->result_mask |= STATX_BTIME;
8143	stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec;
8144	stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec;
8145	if (bi_flags & BTRFS_INODE_APPEND)
8146	stat->attributes |= STATX_ATTR_APPEND;
8147	if (bi_flags & BTRFS_INODE_COMPRESS)
8148	stat->attributes |= STATX_ATTR_COMPRESSED;
8149	if (bi_flags & BTRFS_INODE_IMMUTABLE)
8150	stat->attributes |= STATX_ATTR_IMMUTABLE;
8151	if (bi_flags & BTRFS_INODE_NODUMP)
8152	stat->attributes |= STATX_ATTR_NODUMP;
8153	if (bi_ro_flags & BTRFS_INODE_RO_VERITY)
8154	stat->attributes |= STATX_ATTR_VERITY;
8155
8156	stat->attributes_mask |= (STATX_ATTR_APPEND |
8157	STATX_ATTR_COMPRESSED |
8158	STATX_ATTR_IMMUTABLE |
8159	STATX_ATTR_NODUMP);
8160
8161	generic_fillattr(idmap, request_mask, inode, stat);
8162	stat->dev = BTRFS_I(inode)->root->anon_dev;
8163
8164	stat->subvol = btrfs_root_id(BTRFS_I(inode)->root);
8165	stat->result_mask |= STATX_SUBVOL;
8166
8167	spin_lock(lock: &BTRFS_I(inode)->lock);
8168	delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
8169	inode_bytes = inode_get_bytes(inode);
8170	spin_unlock(lock: &BTRFS_I(inode)->lock);
8171	stat->blocks = (ALIGN(inode_bytes, blocksize) +
8172	ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT;
8173	return 0;
8174	}
8175
8176	static int btrfs_rename_exchange(struct inode *old_dir,
8177	struct dentry *old_dentry,
8178	struct inode *new_dir,
8179	struct dentry *new_dentry)
8180	{
8181	struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
8182	struct btrfs_trans_handle *trans;
8183	unsigned int trans_num_items;
8184	struct btrfs_root *root = BTRFS_I(old_dir)->root;
8185	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
8186	struct inode *new_inode = new_dentry->d_inode;
8187	struct inode *old_inode = old_dentry->d_inode;
8188	struct btrfs_rename_ctx old_rename_ctx;
8189	struct btrfs_rename_ctx new_rename_ctx;
8190	u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
8191	u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
8192	u64 old_idx = 0;
8193	u64 new_idx = 0;
8194	int ret;
8195	int ret2;
8196	bool need_abort = false;
8197	bool logs_pinned = false;
8198	struct fscrypt_name old_fname, new_fname;
8199	struct fscrypt_str *old_name, *new_name;
8200
8201	/*
8202	* For non-subvolumes allow exchange only within one subvolume, in the
8203	* same inode namespace. Two subvolumes (represented as directory) can
8204	* be exchanged as they're a logical link and have a fixed inode number.
8205	*/
8206	if (root != dest &&
8207	(old_ino != BTRFS_FIRST_FREE_OBJECTID ||
8208	new_ino != BTRFS_FIRST_FREE_OBJECTID))
8209	return -EXDEV;
8210
8211	ret = fscrypt_setup_filename(inode: old_dir, iname: &old_dentry->d_name, lookup: 0, fname: &old_fname);
8212	if (ret)
8213	return ret;
8214
8215	ret = fscrypt_setup_filename(inode: new_dir, iname: &new_dentry->d_name, lookup: 0, fname: &new_fname);
8216	if (ret) {
8217	fscrypt_free_filename(fname: &old_fname);
8218	return ret;
8219	}
8220
8221	old_name = &old_fname.disk_name;
8222	new_name = &new_fname.disk_name;
8223
8224	/* close the race window with snapshot create/destroy ioctl */
8225	if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||
8226	new_ino == BTRFS_FIRST_FREE_OBJECTID)
8227	down_read(sem: &fs_info->subvol_sem);
8228
8229	/*
8230	* For each inode:
8231	* 1 to remove old dir item
8232	* 1 to remove old dir index
8233	* 1 to add new dir item
8234	* 1 to add new dir index
8235	* 1 to update parent inode
8236	*
8237	* If the parents are the same, we only need to account for one
8238	*/
8239	trans_num_items = (old_dir == new_dir ? 9 : 10);
8240	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8241	/*
8242	* 1 to remove old root ref
8243	* 1 to remove old root backref
8244	* 1 to add new root ref
8245	* 1 to add new root backref
8246	*/
8247	trans_num_items += 4;
8248	} else {
8249	/*
8250	* 1 to update inode item
8251	* 1 to remove old inode ref
8252	* 1 to add new inode ref
8253	*/
8254	trans_num_items += 3;
8255	}
8256	if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
8257	trans_num_items += 4;
8258	else
8259	trans_num_items += 3;
8260	trans = btrfs_start_transaction(root, num_items: trans_num_items);
8261	if (IS_ERR(ptr: trans)) {
8262	ret = PTR_ERR(ptr: trans);
8263	goto out_notrans;
8264	}
8265
8266	if (dest != root) {
8267	ret = btrfs_record_root_in_trans(trans, root: dest);
8268	if (ret)
8269	goto out_fail;
8270	}
8271
8272	/*
8273	* We need to find a free sequence number both in the source and
8274	* in the destination directory for the exchange.
8275	*/
8276	ret = btrfs_set_inode_index(BTRFS_I(new_dir), index: &old_idx);
8277	if (ret)
8278	goto out_fail;
8279	ret = btrfs_set_inode_index(BTRFS_I(old_dir), index: &new_idx);
8280	if (ret)
8281	goto out_fail;
8282
8283	BTRFS_I(old_inode)->dir_index = 0ULL;
8284	BTRFS_I(new_inode)->dir_index = 0ULL;
8285
8286	/* Reference for the source. */
8287	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8288	/* force full log commit if subvolume involved. */
8289	btrfs_set_log_full_commit(trans);
8290	} else {
8291	ret = btrfs_insert_inode_ref(trans, root: dest, name: new_name, inode_objectid: old_ino,
8292	ref_objectid: btrfs_ino(BTRFS_I(new_dir)),
8293	index: old_idx);
8294	if (ret)
8295	goto out_fail;
8296	need_abort = true;
8297	}
8298
8299	/* And now for the dest. */
8300	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8301	/* force full log commit if subvolume involved. */
8302	btrfs_set_log_full_commit(trans);
8303	} else {
8304	ret = btrfs_insert_inode_ref(trans, root, name: old_name, inode_objectid: new_ino,
8305	ref_objectid: btrfs_ino(BTRFS_I(old_dir)),
8306	index: new_idx);
8307	if (ret) {
8308	if (unlikely(need_abort))
8309	btrfs_abort_transaction(trans, ret);
8310	goto out_fail;
8311	}
8312	}
8313
8314	/* Update inode version and ctime/mtime. */
8315	inode_inc_iversion(inode: old_dir);
8316	inode_inc_iversion(inode: new_dir);
8317	inode_inc_iversion(inode: old_inode);
8318	inode_inc_iversion(inode: new_inode);
8319	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8320
8321	if (old_ino != BTRFS_FIRST_FREE_OBJECTID &&
8322	new_ino != BTRFS_FIRST_FREE_OBJECTID) {
8323	/*
8324	* If we are renaming in the same directory (and it's not for
8325	* root entries) pin the log early to prevent any concurrent
8326	* task from logging the directory after we removed the old
8327	* entries and before we add the new entries, otherwise that
8328	* task can sync a log without any entry for the inodes we are
8329	* renaming and therefore replaying that log, if a power failure
8330	* happens after syncing the log, would result in deleting the
8331	* inodes.
8332	*
8333	* If the rename affects two different directories, we want to
8334	* make sure the that there's no log commit that contains
8335	* updates for only one of the directories but not for the
8336	* other.
8337	*
8338	* If we are renaming an entry for a root, we don't care about
8339	* log updates since we called btrfs_set_log_full_commit().
8340	*/
8341	btrfs_pin_log_trans(root);
8342	btrfs_pin_log_trans(root: dest);
8343	logs_pinned = true;
8344	}
8345
8346	if (old_dentry->d_parent != new_dentry->d_parent) {
8347	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8348	BTRFS_I(old_inode), for_rename: true);
8349	btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
8350	BTRFS_I(new_inode), for_rename: true);
8351	}
8352
8353	/* src is a subvolume */
8354	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8355	ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), dentry: old_dentry);
8356	if (unlikely(ret)) {
8357	btrfs_abort_transaction(trans, ret);
8358	goto out_fail;
8359	}
8360	} else { /* src is an inode */
8361	ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8362	BTRFS_I(old_dentry->d_inode),
8363	name: old_name, rename_ctx: &old_rename_ctx);
8364	if (unlikely(ret)) {
8365	btrfs_abort_transaction(trans, ret);
8366	goto out_fail;
8367	}
8368	ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8369	if (unlikely(ret)) {
8370	btrfs_abort_transaction(trans, ret);
8371	goto out_fail;
8372	}
8373	}
8374
8375	/* dest is a subvolume */
8376	if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
8377	ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), dentry: new_dentry);
8378	if (unlikely(ret)) {
8379	btrfs_abort_transaction(trans, ret);
8380	goto out_fail;
8381	}
8382	} else { /* dest is an inode */
8383	ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8384	BTRFS_I(new_dentry->d_inode),
8385	name: new_name, rename_ctx: &new_rename_ctx);
8386	if (unlikely(ret)) {
8387	btrfs_abort_transaction(trans, ret);
8388	goto out_fail;
8389	}
8390	ret = btrfs_update_inode(trans, BTRFS_I(new_inode));
8391	if (unlikely(ret)) {
8392	btrfs_abort_transaction(trans, ret);
8393	goto out_fail;
8394	}
8395	}
8396
8397	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8398	name: new_name, add_backref: 0, index: old_idx);
8399	if (unlikely(ret)) {
8400	btrfs_abort_transaction(trans, ret);
8401	goto out_fail;
8402	}
8403
8404	ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
8405	name: old_name, add_backref: 0, index: new_idx);
8406	if (unlikely(ret)) {
8407	btrfs_abort_transaction(trans, ret);
8408	goto out_fail;
8409	}
8410
8411	if (old_inode->i_nlink == 1)
8412	BTRFS_I(old_inode)->dir_index = old_idx;
8413	if (new_inode->i_nlink == 1)
8414	BTRFS_I(new_inode)->dir_index = new_idx;
8415
8416	/*
8417	* Do the log updates for all inodes.
8418	*
8419	* If either entry is for a root we don't need to update the logs since
8420	* we've called btrfs_set_log_full_commit() before.
8421	*/
8422	if (logs_pinned) {
8423	btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8424	old_dir_index: old_rename_ctx.index, parent: new_dentry->d_parent);
8425	btrfs_log_new_name(trans, old_dentry: new_dentry, BTRFS_I(new_dir),
8426	old_dir_index: new_rename_ctx.index, parent: old_dentry->d_parent);
8427	}
8428
8429	out_fail:
8430	if (logs_pinned) {
8431	btrfs_end_log_trans(root);
8432	btrfs_end_log_trans(root: dest);
8433	}
8434	ret2 = btrfs_end_transaction(trans);
8435	ret = ret ? ret : ret2;
8436	out_notrans:
8437	if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||
8438	old_ino == BTRFS_FIRST_FREE_OBJECTID)
8439	up_read(sem: &fs_info->subvol_sem);
8440
8441	fscrypt_free_filename(fname: &new_fname);
8442	fscrypt_free_filename(fname: &old_fname);
8443	return ret;
8444	}
8445
8446	static struct inode *new_whiteout_inode(struct mnt_idmap *idmap,
8447	struct inode *dir)
8448	{
8449	struct inode *inode;
8450
8451	inode = new_inode(sb: dir->i_sb);
8452	if (inode) {
8453	inode_init_owner(idmap, inode, dir,
8454	S_IFCHR | WHITEOUT_MODE);
8455	inode->i_op = &btrfs_special_inode_operations;
8456	init_special_inode(inode, inode->i_mode, WHITEOUT_DEV);
8457	}
8458	return inode;
8459	}
8460
8461	static int btrfs_rename(struct mnt_idmap *idmap,
8462	struct inode *old_dir, struct dentry *old_dentry,
8463	struct inode *new_dir, struct dentry *new_dentry,
8464	unsigned int flags)
8465	{
8466	struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir);
8467	struct btrfs_new_inode_args whiteout_args = {
8468	.dir = old_dir,
8469	.dentry = old_dentry,
8470	};
8471	struct btrfs_trans_handle *trans;
8472	unsigned int trans_num_items;
8473	struct btrfs_root *root = BTRFS_I(old_dir)->root;
8474	struct btrfs_root *dest = BTRFS_I(new_dir)->root;
8475	struct inode *new_inode = d_inode(dentry: new_dentry);
8476	struct inode *old_inode = d_inode(dentry: old_dentry);
8477	struct btrfs_rename_ctx rename_ctx;
8478	u64 index = 0;
8479	int ret;
8480	int ret2;
8481	u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
8482	struct fscrypt_name old_fname, new_fname;
8483	bool logs_pinned = false;
8484
8485	if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
8486	return -EPERM;
8487
8488	/* we only allow rename subvolume link between subvolumes */
8489	if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
8490	return -EXDEV;
8491
8492	if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
8493	(new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
8494	return -ENOTEMPTY;
8495
8496	if (S_ISDIR(old_inode->i_mode) && new_inode &&
8497	new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
8498	return -ENOTEMPTY;
8499
8500	ret = fscrypt_setup_filename(inode: old_dir, iname: &old_dentry->d_name, lookup: 0, fname: &old_fname);
8501	if (ret)
8502	return ret;
8503
8504	ret = fscrypt_setup_filename(inode: new_dir, iname: &new_dentry->d_name, lookup: 0, fname: &new_fname);
8505	if (ret) {
8506	fscrypt_free_filename(fname: &old_fname);
8507	return ret;
8508	}
8509
8510	/* check for collisions, even if the name isn't there */
8511	ret = btrfs_check_dir_item_collision(root: dest, dir_ino: new_dir->i_ino, name: &new_fname.disk_name);
8512	if (ret) {
8513	if (ret == -EEXIST) {
8514	/* we shouldn't get
8515	* eexist without a new_inode */
8516	if (WARN_ON(!new_inode)) {
8517	goto out_fscrypt_names;
8518	}
8519	} else {
8520	/* maybe -EOVERFLOW */
8521	goto out_fscrypt_names;
8522	}
8523	}
8524	ret = 0;
8525
8526	/*
8527	* we're using rename to replace one file with another. Start IO on it
8528	* now so we don't add too much work to the end of the transaction
8529	*/
8530	if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
8531	filemap_flush(old_inode->i_mapping);
8532
8533	if (flags & RENAME_WHITEOUT) {
8534	whiteout_args.inode = new_whiteout_inode(idmap, dir: old_dir);
8535	if (!whiteout_args.inode) {
8536	ret = -ENOMEM;
8537	goto out_fscrypt_names;
8538	}
8539	ret = btrfs_new_inode_prepare(args: &whiteout_args, trans_num_items: &trans_num_items);
8540	if (ret)
8541	goto out_whiteout_inode;
8542	} else {
8543	/* 1 to update the old parent inode. */
8544	trans_num_items = 1;
8545	}
8546
8547	if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
8548	/* Close the race window with snapshot create/destroy ioctl */
8549	down_read(sem: &fs_info->subvol_sem);
8550	/*
8551	* 1 to remove old root ref
8552	* 1 to remove old root backref
8553	* 1 to add new root ref
8554	* 1 to add new root backref
8555	*/
8556	trans_num_items += 4;
8557	} else {
8558	/*
8559	* 1 to update inode
8560	* 1 to remove old inode ref
8561	* 1 to add new inode ref
8562	*/
8563	trans_num_items += 3;
8564	}
8565	/*
8566	* 1 to remove old dir item
8567	* 1 to remove old dir index
8568	* 1 to add new dir item
8569	* 1 to add new dir index
8570	*/
8571	trans_num_items += 4;
8572	/* 1 to update new parent inode if it's not the same as the old parent */
8573	if (new_dir != old_dir)
8574	trans_num_items++;
8575	if (new_inode) {
8576	/*
8577	* 1 to update inode
8578	* 1 to remove inode ref
8579	* 1 to remove dir item
8580	* 1 to remove dir index
8581	* 1 to possibly add orphan item
8582	*/
8583	trans_num_items += 5;
8584	}
8585	trans = btrfs_start_transaction(root, num_items: trans_num_items);
8586	if (IS_ERR(ptr: trans)) {
8587	ret = PTR_ERR(ptr: trans);
8588	goto out_notrans;
8589	}
8590
8591	if (dest != root) {
8592	ret = btrfs_record_root_in_trans(trans, root: dest);
8593	if (ret)
8594	goto out_fail;
8595	}
8596
8597	ret = btrfs_set_inode_index(BTRFS_I(new_dir), index: &index);
8598	if (ret)
8599	goto out_fail;
8600
8601	BTRFS_I(old_inode)->dir_index = 0ULL;
8602	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8603	/* force full log commit if subvolume involved. */
8604	btrfs_set_log_full_commit(trans);
8605	} else {
8606	ret = btrfs_insert_inode_ref(trans, root: dest, name: &new_fname.disk_name,
8607	inode_objectid: old_ino, ref_objectid: btrfs_ino(BTRFS_I(new_dir)),
8608	index);
8609	if (ret)
8610	goto out_fail;
8611	}
8612
8613	inode_inc_iversion(inode: old_dir);
8614	inode_inc_iversion(inode: new_dir);
8615	inode_inc_iversion(inode: old_inode);
8616	simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry);
8617
8618	if (old_ino != BTRFS_FIRST_FREE_OBJECTID) {
8619	/*
8620	* If we are renaming in the same directory (and it's not a
8621	* root entry) pin the log to prevent any concurrent task from
8622	* logging the directory after we removed the old entry and
8623	* before we add the new entry, otherwise that task can sync
8624	* a log without any entry for the inode we are renaming and
8625	* therefore replaying that log, if a power failure happens
8626	* after syncing the log, would result in deleting the inode.
8627	*
8628	* If the rename affects two different directories, we want to
8629	* make sure the that there's no log commit that contains
8630	* updates for only one of the directories but not for the
8631	* other.
8632	*
8633	* If we are renaming an entry for a root, we don't care about
8634	* log updates since we called btrfs_set_log_full_commit().
8635	*/
8636	btrfs_pin_log_trans(root);
8637	btrfs_pin_log_trans(root: dest);
8638	logs_pinned = true;
8639	}
8640
8641	if (old_dentry->d_parent != new_dentry->d_parent)
8642	btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
8643	BTRFS_I(old_inode), for_rename: true);
8644
8645	if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
8646	ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), dentry: old_dentry);
8647	if (unlikely(ret)) {
8648	btrfs_abort_transaction(trans, ret);
8649	goto out_fail;
8650	}
8651	} else {
8652	ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir),
8653	BTRFS_I(d_inode(old_dentry)),
8654	name: &old_fname.disk_name, rename_ctx: &rename_ctx);
8655	if (unlikely(ret)) {
8656	btrfs_abort_transaction(trans, ret);
8657	goto out_fail;
8658	}
8659	ret = btrfs_update_inode(trans, BTRFS_I(old_inode));
8660	if (unlikely(ret)) {
8661	btrfs_abort_transaction(trans, ret);
8662	goto out_fail;
8663	}
8664	}
8665
8666	if (new_inode) {
8667	inode_inc_iversion(inode: new_inode);
8668	if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
8669	BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
8670	ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), dentry: new_dentry);
8671	if (unlikely(ret)) {
8672	btrfs_abort_transaction(trans, ret);
8673	goto out_fail;
8674	}
8675	BUG_ON(new_inode->i_nlink == 0);
8676	} else {
8677	ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir),
8678	BTRFS_I(d_inode(new_dentry)),
8679	name: &new_fname.disk_name);
8680	if (unlikely(ret)) {
8681	btrfs_abort_transaction(trans, ret);
8682	goto out_fail;
8683	}
8684	}
8685	if (new_inode->i_nlink == 0) {
8686	ret = btrfs_orphan_add(trans,
8687	BTRFS_I(d_inode(new_dentry)));
8688	if (unlikely(ret)) {
8689	btrfs_abort_transaction(trans, ret);
8690	goto out_fail;
8691	}
8692	}
8693	}
8694
8695	ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
8696	name: &new_fname.disk_name, add_backref: 0, index);
8697	if (unlikely(ret)) {
8698	btrfs_abort_transaction(trans, ret);
8699	goto out_fail;
8700	}
8701
8702	if (old_inode->i_nlink == 1)
8703	BTRFS_I(old_inode)->dir_index = index;
8704
8705	if (logs_pinned)
8706	btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir),
8707	old_dir_index: rename_ctx.index, parent: new_dentry->d_parent);
8708
8709	if (flags & RENAME_WHITEOUT) {
8710	ret = btrfs_create_new_inode(trans, args: &whiteout_args);
8711	if (unlikely(ret)) {
8712	btrfs_abort_transaction(trans, ret);
8713	goto out_fail;
8714	} else {
8715	unlock_new_inode(whiteout_args.inode);
8716	iput(whiteout_args.inode);
8717	whiteout_args.inode = NULL;
8718	}
8719	}
8720	out_fail:
8721	if (logs_pinned) {
8722	btrfs_end_log_trans(root);
8723	btrfs_end_log_trans(root: dest);
8724	}
8725	ret2 = btrfs_end_transaction(trans);
8726	ret = ret ? ret : ret2;
8727	out_notrans:
8728	if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
8729	up_read(sem: &fs_info->subvol_sem);
8730	if (flags & RENAME_WHITEOUT)
8731	btrfs_new_inode_args_destroy(args: &whiteout_args);
8732	out_whiteout_inode:
8733	if (flags & RENAME_WHITEOUT)
8734	iput(whiteout_args.inode);
8735	out_fscrypt_names:
8736	fscrypt_free_filename(fname: &old_fname);
8737	fscrypt_free_filename(fname: &new_fname);
8738	return ret;
8739	}
8740
8741	static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir,
8742	struct dentry *old_dentry, struct inode *new_dir,
8743	struct dentry *new_dentry, unsigned int flags)
8744	{
8745	int ret;
8746
8747	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
8748	return -EINVAL;
8749
8750	if (flags & RENAME_EXCHANGE)
8751	ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir,
8752	new_dentry);
8753	else
8754	ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir,
8755	new_dentry, flags);
8756
8757	btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info);
8758
8759	return ret;
8760	}
8761
8762	struct btrfs_delalloc_work {
8763	struct inode *inode;
8764	struct completion completion;
8765	struct list_head list;
8766	struct btrfs_work work;
8767	};
8768
8769	static void btrfs_run_delalloc_work(struct btrfs_work *work)
8770	{
8771	struct btrfs_delalloc_work *delalloc_work;
8772	struct inode *inode;
8773
8774	delalloc_work = container_of(work, struct btrfs_delalloc_work,
8775	work);
8776	inode = delalloc_work->inode;
8777	filemap_flush(inode->i_mapping);
8778	if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8779	&BTRFS_I(inode)->runtime_flags))
8780	filemap_flush(inode->i_mapping);
8781
8782	iput(inode);
8783	complete(&delalloc_work->completion);
8784	}
8785
8786	static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode)
8787	{
8788	struct btrfs_delalloc_work *work;
8789
8790	work = kmalloc(sizeof(*work), GFP_NOFS);
8791	if (!work)
8792	return NULL;
8793
8794	init_completion(x: &work->completion);
8795	INIT_LIST_HEAD(list: &work->list);
8796	work->inode = inode;
8797	btrfs_init_work(work: &work->work, func: btrfs_run_delalloc_work, NULL);
8798
8799	return work;
8800	}
8801
8802	/*
8803	* some fairly slow code that needs optimization. This walks the list
8804	* of all the inodes with pending delalloc and forces them to disk.
8805	*/
8806	static int start_delalloc_inodes(struct btrfs_root *root, long *nr_to_write,
8807	bool snapshot, bool in_reclaim_context)
8808	{
8809	struct btrfs_delalloc_work *work, *next;
8810	LIST_HEAD(works);
8811	LIST_HEAD(splice);
8812	int ret = 0;
8813
8814	mutex_lock(&root->delalloc_mutex);
8815	spin_lock(lock: &root->delalloc_lock);
8816	list_splice_init(list: &root->delalloc_inodes, head: &splice);
8817	while (!list_empty(head: &splice)) {
8818	struct btrfs_inode *inode;
8819	struct inode *tmp_inode;
8820
8821	inode = list_first_entry(&splice, struct btrfs_inode, delalloc_inodes);
8822
8823	list_move_tail(list: &inode->delalloc_inodes, head: &root->delalloc_inodes);
8824
8825	if (in_reclaim_context &&
8826	test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags))
8827	continue;
8828
8829	tmp_inode = igrab(&inode->vfs_inode);
8830	if (!tmp_inode) {
8831	cond_resched_lock(&root->delalloc_lock);
8832	continue;
8833	}
8834	spin_unlock(lock: &root->delalloc_lock);
8835
8836	if (snapshot)
8837	set_bit(nr: BTRFS_INODE_SNAPSHOT_FLUSH, addr: &inode->runtime_flags);
8838	if (nr_to_write == NULL) {
8839	work = btrfs_alloc_delalloc_work(inode: tmp_inode);
8840	if (!work) {
8841	iput(tmp_inode);
8842	ret = -ENOMEM;
8843	goto out;
8844	}
8845	list_add_tail(new: &work->list, head: &works);
8846	btrfs_queue_work(wq: root->fs_info->flush_workers,
8847	work: &work->work);
8848	} else {
8849	ret = filemap_flush_nr(mapping: tmp_inode->i_mapping,
8850	nr_to_write);
8851	btrfs_add_delayed_iput(inode);
8852
8853	if (ret || *nr_to_write <= 0)
8854	goto out;
8855	}
8856	cond_resched();
8857	spin_lock(lock: &root->delalloc_lock);
8858	}
8859	spin_unlock(lock: &root->delalloc_lock);
8860
8861	out:
8862	list_for_each_entry_safe(work, next, &works, list) {
8863	list_del_init(entry: &work->list);
8864	wait_for_completion(&work->completion);
8865	kfree(objp: work);
8866	}
8867
8868	if (!list_empty(head: &splice)) {
8869	spin_lock(lock: &root->delalloc_lock);
8870	list_splice_tail(list: &splice, head: &root->delalloc_inodes);
8871	spin_unlock(lock: &root->delalloc_lock);
8872	}
8873	mutex_unlock(lock: &root->delalloc_mutex);
8874	return ret;
8875	}
8876
8877	int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context)
8878	{
8879	struct btrfs_fs_info *fs_info = root->fs_info;
8880
8881	if (BTRFS_FS_ERROR(fs_info))
8882	return -EROFS;
8883	return start_delalloc_inodes(root, NULL, snapshot: true, in_reclaim_context);
8884	}
8885
8886	int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr,
8887	bool in_reclaim_context)
8888	{
8889	long *nr_to_write = nr == LONG_MAX ? NULL : &nr;
8890	struct btrfs_root *root;
8891	LIST_HEAD(splice);
8892	int ret;
8893
8894	if (BTRFS_FS_ERROR(fs_info))
8895	return -EROFS;
8896
8897	mutex_lock(&fs_info->delalloc_root_mutex);
8898	spin_lock(lock: &fs_info->delalloc_root_lock);
8899	list_splice_init(list: &fs_info->delalloc_roots, head: &splice);
8900	while (!list_empty(head: &splice)) {
8901	root = list_first_entry(&splice, struct btrfs_root,
8902	delalloc_root);
8903	root = btrfs_grab_root(root);
8904	BUG_ON(!root);
8905	list_move_tail(list: &root->delalloc_root,
8906	head: &fs_info->delalloc_roots);
8907	spin_unlock(lock: &fs_info->delalloc_root_lock);
8908
8909	ret = start_delalloc_inodes(root, nr_to_write, snapshot: false,
8910	in_reclaim_context);
8911	btrfs_put_root(root);
8912	if (ret < 0 || nr <= 0)
8913	goto out;
8914	spin_lock(lock: &fs_info->delalloc_root_lock);
8915	}
8916	spin_unlock(lock: &fs_info->delalloc_root_lock);
8917
8918	ret = 0;
8919	out:
8920	if (!list_empty(head: &splice)) {
8921	spin_lock(lock: &fs_info->delalloc_root_lock);
8922	list_splice_tail(list: &splice, head: &fs_info->delalloc_roots);
8923	spin_unlock(lock: &fs_info->delalloc_root_lock);
8924	}
8925	mutex_unlock(lock: &fs_info->delalloc_root_mutex);
8926	return ret;
8927	}
8928
8929	static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
8930	struct dentry *dentry, const char *symname)
8931	{
8932	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
8933	struct btrfs_trans_handle *trans;
8934	struct btrfs_root *root = BTRFS_I(dir)->root;
8935	struct btrfs_path *path;
8936	struct btrfs_key key;
8937	struct inode *inode;
8938	struct btrfs_new_inode_args new_inode_args = {
8939	.dir = dir,
8940	.dentry = dentry,
8941	};
8942	unsigned int trans_num_items;
8943	int ret;
8944	int name_len;
8945	int datasize;
8946	unsigned long ptr;
8947	struct btrfs_file_extent_item *ei;
8948	struct extent_buffer *leaf;
8949
8950	name_len = strlen(symname);
8951	/*
8952	* Symlinks utilize uncompressed inline extent data, which should not
8953	* reach block size.
8954	*/
8955	if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(info: fs_info) ||
8956	name_len >= fs_info->sectorsize)
8957	return -ENAMETOOLONG;
8958
8959	inode = new_inode(sb: dir->i_sb);
8960	if (!inode)
8961	return -ENOMEM;
8962	inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO);
8963	inode->i_op = &btrfs_symlink_inode_operations;
8964	inode_nohighmem(inode);
8965	inode->i_mapping->a_ops = &btrfs_aops;
8966	btrfs_i_size_write(BTRFS_I(inode), size: name_len);
8967	inode_set_bytes(inode, bytes: name_len);
8968
8969	new_inode_args.inode = inode;
8970	ret = btrfs_new_inode_prepare(args: &new_inode_args, trans_num_items: &trans_num_items);
8971	if (ret)
8972	goto out_inode;
8973	/* 1 additional item for the inline extent */
8974	trans_num_items++;
8975
8976	trans = btrfs_start_transaction(root, num_items: trans_num_items);
8977	if (IS_ERR(ptr: trans)) {
8978	ret = PTR_ERR(ptr: trans);
8979	goto out_new_inode_args;
8980	}
8981
8982	ret = btrfs_create_new_inode(trans, args: &new_inode_args);
8983	if (ret)
8984	goto out;
8985
8986	path = btrfs_alloc_path();
8987	if (unlikely(!path)) {
8988	ret = -ENOMEM;
8989	btrfs_abort_transaction(trans, ret);
8990	discard_new_inode(inode);
8991	inode = NULL;
8992	goto out;
8993	}
8994	key.objectid = btrfs_ino(BTRFS_I(inode));
8995	key.type = BTRFS_EXTENT_DATA_KEY;
8996	key.offset = 0;
8997	datasize = btrfs_file_extent_calc_inline_size(datasize: name_len);
8998	ret = btrfs_insert_empty_item(trans, root, path, key: &key, data_size: datasize);
8999	if (unlikely(ret)) {
9000	btrfs_abort_transaction(trans, ret);
9001	btrfs_free_path(p: path);
9002	discard_new_inode(inode);
9003	inode = NULL;
9004	goto out;
9005	}
9006	leaf = path->nodes[0];
9007	ei = btrfs_item_ptr(leaf, path->slots[0],
9008	struct btrfs_file_extent_item);
9009	btrfs_set_file_extent_generation(eb: leaf, s: ei, val: trans->transid);
9010	btrfs_set_file_extent_type(eb: leaf, s: ei,
9011	val: BTRFS_FILE_EXTENT_INLINE);
9012	btrfs_set_file_extent_encryption(eb: leaf, s: ei, val: 0);
9013	btrfs_set_file_extent_compression(eb: leaf, s: ei, val: 0);
9014	btrfs_set_file_extent_other_encoding(eb: leaf, s: ei, val: 0);
9015	btrfs_set_file_extent_ram_bytes(eb: leaf, s: ei, val: name_len);
9016
9017	ptr = btrfs_file_extent_inline_start(e: ei);
9018	write_extent_buffer(eb: leaf, src: symname, start: ptr, len: name_len);
9019	btrfs_free_path(p: path);
9020
9021	d_instantiate_new(dentry, inode);
9022	ret = 0;
9023	out:
9024	btrfs_end_transaction(trans);
9025	btrfs_btree_balance_dirty(fs_info);
9026	out_new_inode_args:
9027	btrfs_new_inode_args_destroy(args: &new_inode_args);
9028	out_inode:
9029	if (ret)
9030	iput(inode);
9031	return ret;
9032	}
9033
9034	static struct btrfs_trans_handle *insert_prealloc_file_extent(
9035	struct btrfs_trans_handle *trans_in,
9036	struct btrfs_inode *inode,
9037	struct btrfs_key *ins,
9038	u64 file_offset)
9039	{
9040	struct btrfs_file_extent_item stack_fi;
9041	struct btrfs_replace_extent_info extent_info;
9042	struct btrfs_trans_handle *trans = trans_in;
9043	struct btrfs_path *path;
9044	u64 start = ins->objectid;
9045	u64 len = ins->offset;
9046	u64 qgroup_released = 0;
9047	int ret;
9048
9049	memset(&stack_fi, 0, sizeof(stack_fi));
9050
9051	btrfs_set_stack_file_extent_type(s: &stack_fi, val: BTRFS_FILE_EXTENT_PREALLOC);
9052	btrfs_set_stack_file_extent_disk_bytenr(s: &stack_fi, val: start);
9053	btrfs_set_stack_file_extent_disk_num_bytes(s: &stack_fi, val: len);
9054	btrfs_set_stack_file_extent_num_bytes(s: &stack_fi, val: len);
9055	btrfs_set_stack_file_extent_ram_bytes(s: &stack_fi, val: len);
9056	btrfs_set_stack_file_extent_compression(s: &stack_fi, val: BTRFS_COMPRESS_NONE);
9057	/* Encryption and other encoding is reserved and all 0 */
9058
9059	ret = btrfs_qgroup_release_data(inode, start: file_offset, len, released: &qgroup_released);
9060	if (ret < 0)
9061	return ERR_PTR(error: ret);
9062
9063	if (trans) {
9064	ret = insert_reserved_file_extent(trans, inode,
9065	file_pos: file_offset, stack_fi: &stack_fi,
9066	update_inode_bytes: true, qgroup_reserved: qgroup_released);
9067	if (ret)
9068	goto free_qgroup;
9069	return trans;
9070	}
9071
9072	extent_info.disk_offset = start;
9073	extent_info.disk_len = len;
9074	extent_info.data_offset = 0;
9075	extent_info.data_len = len;
9076	extent_info.file_offset = file_offset;
9077	extent_info.extent_buf = (char *)&stack_fi;
9078	extent_info.is_new_extent = true;
9079	extent_info.update_times = true;
9080	extent_info.qgroup_reserved = qgroup_released;
9081	extent_info.insertions = 0;
9082
9083	path = btrfs_alloc_path();
9084	if (!path) {
9085	ret = -ENOMEM;
9086	goto free_qgroup;
9087	}
9088
9089	ret = btrfs_replace_file_extents(inode, path, start: file_offset,
9090	end: file_offset + len - 1, extent_info: &extent_info,
9091	trans_out: &trans);
9092	btrfs_free_path(p: path);
9093	if (ret)
9094	goto free_qgroup;
9095	return trans;
9096
9097	free_qgroup:
9098	/*
9099	* We have released qgroup data range at the beginning of the function,
9100	* and normally qgroup_released bytes will be freed when committing
9101	* transaction.
9102	* But if we error out early, we have to free what we have released
9103	* or we leak qgroup data reservation.
9104	*/
9105	btrfs_qgroup_free_refroot(fs_info: inode->root->fs_info,
9106	ref_root: btrfs_root_id(root: inode->root), num_bytes: qgroup_released,
9107	type: BTRFS_QGROUP_RSV_DATA);
9108	return ERR_PTR(error: ret);
9109	}
9110
9111	static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
9112	u64 start, u64 num_bytes, u64 min_size,
9113	loff_t actual_len, u64 *alloc_hint,
9114	struct btrfs_trans_handle *trans)
9115	{
9116	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
9117	struct extent_map *em;
9118	struct btrfs_root *root = BTRFS_I(inode)->root;
9119	struct btrfs_key ins;
9120	u64 cur_offset = start;
9121	u64 clear_offset = start;
9122	u64 i_size;
9123	u64 cur_bytes;
9124	u64 last_alloc = (u64)-1;
9125	int ret = 0;
9126	bool own_trans = true;
9127	u64 end = start + num_bytes - 1;
9128
9129	if (trans)
9130	own_trans = false;
9131	while (num_bytes > 0) {
9132	cur_bytes = min_t(u64, num_bytes, SZ_256M);
9133	cur_bytes = max(cur_bytes, min_size);
9134	/*
9135	* If we are severely fragmented we could end up with really
9136	* small allocations, so if the allocator is returning small
9137	* chunks lets make its job easier by only searching for those
9138	* sized chunks.
9139	*/
9140	cur_bytes = min(cur_bytes, last_alloc);
9141	ret = btrfs_reserve_extent(root, ram_bytes: cur_bytes, num_bytes: cur_bytes,
9142	min_alloc_size: min_size, empty_size: 0, hint_byte: *alloc_hint, ins: &ins, is_data: true, delalloc: false);
9143	if (ret)
9144	break;
9145
9146	/*
9147	* We've reserved this space, and thus converted it from
9148	* ->bytes_may_use to ->bytes_reserved. Any error that happens
9149	* from here on out we will only need to clear our reservation
9150	* for the remaining unreserved area, so advance our
9151	* clear_offset by our extent size.
9152	*/
9153	clear_offset += ins.offset;
9154
9155	last_alloc = ins.offset;
9156	trans = insert_prealloc_file_extent(trans_in: trans, BTRFS_I(inode),
9157	ins: &ins, file_offset: cur_offset);
9158	/*
9159	* Now that we inserted the prealloc extent we can finally
9160	* decrement the number of reservations in the block group.
9161	* If we did it before, we could race with relocation and have
9162	* relocation miss the reserved extent, making it fail later.
9163	*/
9164	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
9165	if (IS_ERR(ptr: trans)) {
9166	ret = PTR_ERR(ptr: trans);
9167	btrfs_free_reserved_extent(fs_info, start: ins.objectid,
9168	len: ins.offset, is_delalloc: false);
9169	break;
9170	}
9171
9172	em = btrfs_alloc_extent_map();
9173	if (!em) {
9174	btrfs_drop_extent_map_range(BTRFS_I(inode), start: cur_offset,
9175	end: cur_offset + ins.offset - 1, skip_pinned: false);
9176	btrfs_set_inode_full_sync(BTRFS_I(inode));
9177	goto next;
9178	}
9179
9180	em->start = cur_offset;
9181	em->len = ins.offset;
9182	em->disk_bytenr = ins.objectid;
9183	em->offset = 0;
9184	em->disk_num_bytes = ins.offset;
9185	em->ram_bytes = ins.offset;
9186	em->flags |= EXTENT_FLAG_PREALLOC;
9187	em->generation = trans->transid;
9188
9189	ret = btrfs_replace_extent_map_range(BTRFS_I(inode), new_em: em, modified: true);
9190	btrfs_free_extent_map(em);
9191	next:
9192	num_bytes -= ins.offset;
9193	cur_offset += ins.offset;
9194	*alloc_hint = ins.objectid + ins.offset;
9195
9196	inode_inc_iversion(inode);
9197	inode_set_ctime_current(inode);
9198	BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
9199	if (!(mode & FALLOC_FL_KEEP_SIZE) &&
9200	(actual_len > inode->i_size) &&
9201	(cur_offset > inode->i_size)) {
9202	if (cur_offset > actual_len)
9203	i_size = actual_len;
9204	else
9205	i_size = cur_offset;
9206	i_size_write(inode, i_size);
9207	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), new_i_size: 0);
9208	}
9209
9210	ret = btrfs_update_inode(trans, BTRFS_I(inode));
9211
9212	if (unlikely(ret)) {
9213	btrfs_abort_transaction(trans, ret);
9214	if (own_trans)
9215	btrfs_end_transaction(trans);
9216	break;
9217	}
9218
9219	if (own_trans) {
9220	btrfs_end_transaction(trans);
9221	trans = NULL;
9222	}
9223	}
9224	if (clear_offset < end)
9225	btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, start: clear_offset,
9226	len: end - clear_offset + 1);
9227	return ret;
9228	}
9229
9230	int btrfs_prealloc_file_range(struct inode *inode, int mode,
9231	u64 start, u64 num_bytes, u64 min_size,
9232	loff_t actual_len, u64 *alloc_hint)
9233	{
9234	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9235	min_size, actual_len, alloc_hint,
9236	NULL);
9237	}
9238
9239	int btrfs_prealloc_file_range_trans(struct inode *inode,
9240	struct btrfs_trans_handle *trans, int mode,
9241	u64 start, u64 num_bytes, u64 min_size,
9242	loff_t actual_len, u64 *alloc_hint)
9243	{
9244	return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
9245	min_size, actual_len, alloc_hint, trans);
9246	}
9247
9248	/*
9249	* NOTE: in case you are adding MAY_EXEC check for directories:
9250	* we are marking them with IOP_FASTPERM_MAY_EXEC, allowing path lookup to
9251	* elide calls here.
9252	*/
9253	static int btrfs_permission(struct mnt_idmap *idmap,
9254	struct inode *inode, int mask)
9255	{
9256	struct btrfs_root *root = BTRFS_I(inode)->root;
9257	umode_t mode = inode->i_mode;
9258
9259	if (mask & MAY_WRITE &&
9260	(S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
9261	if (btrfs_root_readonly(root))
9262	return -EROFS;
9263	if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
9264	return -EACCES;
9265	}
9266	return generic_permission(idmap, inode, mask);
9267	}
9268
9269	static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir,
9270	struct file *file, umode_t mode)
9271	{
9272	struct btrfs_fs_info *fs_info = inode_to_fs_info(dir);
9273	struct btrfs_trans_handle *trans;
9274	struct btrfs_root *root = BTRFS_I(dir)->root;
9275	struct inode *inode;
9276	struct btrfs_new_inode_args new_inode_args = {
9277	.dir = dir,
9278	.dentry = file->f_path.dentry,
9279	.orphan = true,
9280	};
9281	unsigned int trans_num_items;
9282	int ret;
9283
9284	inode = new_inode(sb: dir->i_sb);
9285	if (!inode)
9286	return -ENOMEM;
9287	inode_init_owner(idmap, inode, dir, mode);
9288	inode->i_fop = &btrfs_file_operations;
9289	inode->i_op = &btrfs_file_inode_operations;
9290	inode->i_mapping->a_ops = &btrfs_aops;
9291
9292	new_inode_args.inode = inode;
9293	ret = btrfs_new_inode_prepare(args: &new_inode_args, trans_num_items: &trans_num_items);
9294	if (ret)
9295	goto out_inode;
9296
9297	trans = btrfs_start_transaction(root, num_items: trans_num_items);
9298	if (IS_ERR(ptr: trans)) {
9299	ret = PTR_ERR(ptr: trans);
9300	goto out_new_inode_args;
9301	}
9302
9303	ret = btrfs_create_new_inode(trans, args: &new_inode_args);
9304
9305	/*
9306	* We set number of links to 0 in btrfs_create_new_inode(), and here we
9307	* set it to 1 because d_tmpfile() will issue a warning if the count is
9308	* 0, through:
9309	*
9310	* d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
9311	*/
9312	set_nlink(inode, nlink: 1);
9313
9314	if (!ret) {
9315	d_tmpfile(file, inode);
9316	unlock_new_inode(inode);
9317	mark_inode_dirty(inode);
9318	}
9319
9320	btrfs_end_transaction(trans);
9321	btrfs_btree_balance_dirty(fs_info);
9322	out_new_inode_args:
9323	btrfs_new_inode_args_destroy(args: &new_inode_args);
9324	out_inode:
9325	if (ret)
9326	iput(inode);
9327	return finish_open_simple(file, error: ret);
9328	}
9329
9330	int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info,
9331	int compress_type)
9332	{
9333	switch (compress_type) {
9334	case BTRFS_COMPRESS_NONE:
9335	return BTRFS_ENCODED_IO_COMPRESSION_NONE;
9336	case BTRFS_COMPRESS_ZLIB:
9337	return BTRFS_ENCODED_IO_COMPRESSION_ZLIB;
9338	case BTRFS_COMPRESS_LZO:
9339	/*
9340	* The LZO format depends on the sector size. 64K is the maximum
9341	* sector size that we support.
9342	*/
9343	if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K)
9344	return -EINVAL;
9345	return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K +
9346	(fs_info->sectorsize_bits - 12);
9347	case BTRFS_COMPRESS_ZSTD:
9348	return BTRFS_ENCODED_IO_COMPRESSION_ZSTD;
9349	default:
9350	return -EUCLEAN;
9351	}
9352	}
9353
9354	static ssize_t btrfs_encoded_read_inline(
9355	struct kiocb *iocb,
9356	struct iov_iter *iter, u64 start,
9357	u64 lockend,
9358	struct extent_state **cached_state,
9359	u64 extent_start, size_t count,
9360	struct btrfs_ioctl_encoded_io_args *encoded,
9361	bool *unlocked)
9362	{
9363	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9364	struct btrfs_root *root = inode->root;
9365	struct btrfs_fs_info *fs_info = root->fs_info;
9366	struct extent_io_tree *io_tree = &inode->io_tree;
9367	BTRFS_PATH_AUTO_FREE(path);
9368	struct extent_buffer *leaf;
9369	struct btrfs_file_extent_item *item;
9370	u64 ram_bytes;
9371	unsigned long ptr;
9372	void *tmp;
9373	ssize_t ret;
9374	const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
9375
9376	path = btrfs_alloc_path();
9377	if (!path)
9378	return -ENOMEM;
9379
9380	path->nowait = nowait;
9381
9382	ret = btrfs_lookup_file_extent(NULL, root, path, objectid: btrfs_ino(inode),
9383	bytenr: extent_start, mod: 0);
9384	if (ret) {
9385	if (unlikely(ret > 0)) {
9386	/* The extent item disappeared? */
9387	return -EIO;
9388	}
9389	return ret;
9390	}
9391	leaf = path->nodes[0];
9392	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
9393
9394	ram_bytes = btrfs_file_extent_ram_bytes(eb: leaf, s: item);
9395	ptr = btrfs_file_extent_inline_start(e: item);
9396
9397	encoded->len = min_t(u64, extent_start + ram_bytes,
9398	inode->vfs_inode.i_size) - iocb->ki_pos;
9399	ret = btrfs_encoded_io_compression_from_extent(fs_info,
9400	compress_type: btrfs_file_extent_compression(eb: leaf, s: item));
9401	if (ret < 0)
9402	return ret;
9403	encoded->compression = ret;
9404	if (encoded->compression) {
9405	size_t inline_size;
9406
9407	inline_size = btrfs_file_extent_inline_item_len(eb: leaf,
9408	nr: path->slots[0]);
9409	if (inline_size > count)
9410	return -ENOBUFS;
9411
9412	count = inline_size;
9413	encoded->unencoded_len = ram_bytes;
9414	encoded->unencoded_offset = iocb->ki_pos - extent_start;
9415	} else {
9416	count = min_t(u64, count, encoded->len);
9417	encoded->len = count;
9418	encoded->unencoded_len = count;
9419	ptr += iocb->ki_pos - extent_start;
9420	}
9421
9422	tmp = kmalloc(count, GFP_NOFS);
9423	if (!tmp)
9424	return -ENOMEM;
9425
9426	read_extent_buffer(eb: leaf, dst: tmp, start: ptr, len: count);
9427	btrfs_release_path(p: path);
9428	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9429	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
9430	*unlocked = true;
9431
9432	ret = copy_to_iter(addr: tmp, bytes: count, i: iter);
9433	if (ret != count)
9434	ret = -EFAULT;
9435	kfree(objp: tmp);
9436
9437	return ret;
9438	}
9439
9440	struct btrfs_encoded_read_private {
9441	struct completion *sync_reads;
9442	void *uring_ctx;
9443	refcount_t pending_refs;
9444	blk_status_t status;
9445	};
9446
9447	static void btrfs_encoded_read_endio(struct btrfs_bio *bbio)
9448	{
9449	struct btrfs_encoded_read_private *priv = bbio->private;
9450
9451	if (bbio->bio.bi_status) {
9452	/*
9453	* The memory barrier implied by the refcount_dec_and_test() here
9454	* pairs with the memory barrier implied by the refcount_dec_and_test()
9455	* in btrfs_encoded_read_regular_fill_pages() to ensure that
9456	* this write is observed before the load of status in
9457	* btrfs_encoded_read_regular_fill_pages().
9458	*/
9459	WRITE_ONCE(priv->status, bbio->bio.bi_status);
9460	}
9461	if (refcount_dec_and_test(r: &priv->pending_refs)) {
9462	int err = blk_status_to_errno(READ_ONCE(priv->status));
9463
9464	if (priv->uring_ctx) {
9465	btrfs_uring_read_extent_endio(ctx: priv->uring_ctx, err);
9466	kfree(objp: priv);
9467	} else {
9468	complete(priv->sync_reads);
9469	}
9470	}
9471	bio_put(&bbio->bio);
9472	}
9473
9474	int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode,
9475	u64 disk_bytenr, u64 disk_io_size,
9476	struct page **pages, void *uring_ctx)
9477	{
9478	struct btrfs_encoded_read_private *priv, sync_priv;
9479	struct completion sync_reads;
9480	unsigned long i = 0;
9481	struct btrfs_bio *bbio;
9482	int ret;
9483
9484	/*
9485	* Fast path for synchronous reads which completes in this call, io_uring
9486	* needs longer time span.
9487	*/
9488	if (uring_ctx) {
9489	priv = kmalloc(sizeof(struct btrfs_encoded_read_private), GFP_NOFS);
9490	if (!priv)
9491	return -ENOMEM;
9492	} else {
9493	priv = &sync_priv;
9494	init_completion(x: &sync_reads);
9495	priv->sync_reads = &sync_reads;
9496	}
9497
9498	refcount_set(r: &priv->pending_refs, n: 1);
9499	priv->status = 0;
9500	priv->uring_ctx = uring_ctx;
9501
9502	bbio = btrfs_bio_alloc(BIO_MAX_VECS, opf: REQ_OP_READ, inode, file_offset: 0,
9503	end_io: btrfs_encoded_read_endio, private: priv);
9504	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9505
9506	do {
9507	size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE);
9508
9509	if (bio_add_page(bio: &bbio->bio, page: pages[i], len: bytes, off: 0) < bytes) {
9510	refcount_inc(r: &priv->pending_refs);
9511	btrfs_submit_bbio(bbio, mirror_num: 0);
9512
9513	bbio = btrfs_bio_alloc(BIO_MAX_VECS, opf: REQ_OP_READ, inode, file_offset: 0,
9514	end_io: btrfs_encoded_read_endio, private: priv);
9515	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
9516	continue;
9517	}
9518
9519	i++;
9520	disk_bytenr += bytes;
9521	disk_io_size -= bytes;
9522	} while (disk_io_size);
9523
9524	refcount_inc(r: &priv->pending_refs);
9525	btrfs_submit_bbio(bbio, mirror_num: 0);
9526
9527	if (uring_ctx) {
9528	if (refcount_dec_and_test(r: &priv->pending_refs)) {
9529	ret = blk_status_to_errno(READ_ONCE(priv->status));
9530	btrfs_uring_read_extent_endio(ctx: uring_ctx, err: ret);
9531	kfree(objp: priv);
9532	return ret;
9533	}
9534
9535	return -EIOCBQUEUED;
9536	} else {
9537	if (!refcount_dec_and_test(r: &priv->pending_refs))
9538	wait_for_completion_io(&sync_reads);
9539	/* See btrfs_encoded_read_endio() for ordering. */
9540	return blk_status_to_errno(READ_ONCE(priv->status));
9541	}
9542	}
9543
9544	ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter,
9545	u64 start, u64 lockend,
9546	struct extent_state **cached_state,
9547	u64 disk_bytenr, u64 disk_io_size,
9548	size_t count, bool compressed, bool *unlocked)
9549	{
9550	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9551	struct extent_io_tree *io_tree = &inode->io_tree;
9552	struct page **pages;
9553	unsigned long nr_pages, i;
9554	u64 cur;
9555	size_t page_offset;
9556	ssize_t ret;
9557
9558	nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE);
9559	pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
9560	if (!pages)
9561	return -ENOMEM;
9562	ret = btrfs_alloc_page_array(nr_pages, page_array: pages, nofail: false);
9563	if (ret) {
9564	ret = -ENOMEM;
9565	goto out;
9566	}
9567
9568	ret = btrfs_encoded_read_regular_fill_pages(inode, disk_bytenr,
9569	disk_io_size, pages, NULL);
9570	if (ret)
9571	goto out;
9572
9573	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9574	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
9575	*unlocked = true;
9576
9577	if (compressed) {
9578	i = 0;
9579	page_offset = 0;
9580	} else {
9581	i = (iocb->ki_pos - start) >> PAGE_SHIFT;
9582	page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1);
9583	}
9584	cur = 0;
9585	while (cur < count) {
9586	size_t bytes = min_t(size_t, count - cur,
9587	PAGE_SIZE - page_offset);
9588
9589	if (copy_page_to_iter(page: pages[i], offset: page_offset, bytes,
9590	i: iter) != bytes) {
9591	ret = -EFAULT;
9592	goto out;
9593	}
9594	i++;
9595	cur += bytes;
9596	page_offset = 0;
9597	}
9598	ret = count;
9599	out:
9600	for (i = 0; i < nr_pages; i++) {
9601	if (pages[i])
9602	__free_page(pages[i]);
9603	}
9604	kfree(objp: pages);
9605	return ret;
9606	}
9607
9608	ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter,
9609	struct btrfs_ioctl_encoded_io_args *encoded,
9610	struct extent_state **cached_state,
9611	u64 *disk_bytenr, u64 *disk_io_size)
9612	{
9613	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9614	struct btrfs_fs_info *fs_info = inode->root->fs_info;
9615	struct extent_io_tree *io_tree = &inode->io_tree;
9616	ssize_t ret;
9617	size_t count = iov_iter_count(i: iter);
9618	u64 start, lockend;
9619	struct extent_map *em;
9620	const bool nowait = (iocb->ki_flags & IOCB_NOWAIT);
9621	bool unlocked = false;
9622
9623	file_accessed(file: iocb->ki_filp);
9624
9625	ret = btrfs_inode_lock(inode,
9626	ilock_flags: BTRFS_ILOCK_SHARED | (nowait ? BTRFS_ILOCK_TRY : 0));
9627	if (ret)
9628	return ret;
9629
9630	if (iocb->ki_pos >= inode->vfs_inode.i_size) {
9631	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
9632	return 0;
9633	}
9634	start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize);
9635	/*
9636	* We don't know how long the extent containing iocb->ki_pos is, but if
9637	* it's compressed we know that it won't be longer than this.
9638	*/
9639	lockend = start + BTRFS_MAX_UNCOMPRESSED - 1;
9640
9641	if (nowait) {
9642	struct btrfs_ordered_extent *ordered;
9643
9644	if (filemap_range_needs_writeback(mapping: inode->vfs_inode.i_mapping,
9645	start_byte: start, end_byte: lockend)) {
9646	ret = -EAGAIN;
9647	goto out_unlock_inode;
9648	}
9649
9650	if (!btrfs_try_lock_extent(tree: io_tree, start, end: lockend, cached: cached_state)) {
9651	ret = -EAGAIN;
9652	goto out_unlock_inode;
9653	}
9654
9655	ordered = btrfs_lookup_ordered_range(inode, file_offset: start,
9656	len: lockend - start + 1);
9657	if (ordered) {
9658	btrfs_put_ordered_extent(entry: ordered);
9659	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9660	ret = -EAGAIN;
9661	goto out_unlock_inode;
9662	}
9663	} else {
9664	for (;;) {
9665	struct btrfs_ordered_extent *ordered;
9666
9667	ret = btrfs_wait_ordered_range(inode, start,
9668	len: lockend - start + 1);
9669	if (ret)
9670	goto out_unlock_inode;
9671
9672	btrfs_lock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9673	ordered = btrfs_lookup_ordered_range(inode, file_offset: start,
9674	len: lockend - start + 1);
9675	if (!ordered)
9676	break;
9677	btrfs_put_ordered_extent(entry: ordered);
9678	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9679	cond_resched();
9680	}
9681	}
9682
9683	em = btrfs_get_extent(inode, NULL, start, len: lockend - start + 1);
9684	if (IS_ERR(ptr: em)) {
9685	ret = PTR_ERR(ptr: em);
9686	goto out_unlock_extent;
9687	}
9688
9689	if (em->disk_bytenr == EXTENT_MAP_INLINE) {
9690	u64 extent_start = em->start;
9691
9692	/*
9693	* For inline extents we get everything we need out of the
9694	* extent item.
9695	*/
9696	btrfs_free_extent_map(em);
9697	em = NULL;
9698	ret = btrfs_encoded_read_inline(iocb, iter, start, lockend,
9699	cached_state, extent_start,
9700	count, encoded, unlocked: &unlocked);
9701	goto out_unlock_extent;
9702	}
9703
9704	/*
9705	* We only want to return up to EOF even if the extent extends beyond
9706	* that.
9707	*/
9708	encoded->len = min_t(u64, btrfs_extent_map_end(em),
9709	inode->vfs_inode.i_size) - iocb->ki_pos;
9710	if (em->disk_bytenr == EXTENT_MAP_HOLE ||
9711	(em->flags & EXTENT_FLAG_PREALLOC)) {
9712	*disk_bytenr = EXTENT_MAP_HOLE;
9713	count = min_t(u64, count, encoded->len);
9714	encoded->len = count;
9715	encoded->unencoded_len = count;
9716	} else if (btrfs_extent_map_is_compressed(em)) {
9717	*disk_bytenr = em->disk_bytenr;
9718	/*
9719	* Bail if the buffer isn't large enough to return the whole
9720	* compressed extent.
9721	*/
9722	if (em->disk_num_bytes > count) {
9723	ret = -ENOBUFS;
9724	goto out_em;
9725	}
9726	*disk_io_size = em->disk_num_bytes;
9727	count = em->disk_num_bytes;
9728	encoded->unencoded_len = em->ram_bytes;
9729	encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset);
9730	ret = btrfs_encoded_io_compression_from_extent(fs_info,
9731	compress_type: btrfs_extent_map_compression(em));
9732	if (ret < 0)
9733	goto out_em;
9734	encoded->compression = ret;
9735	} else {
9736	*disk_bytenr = btrfs_extent_map_block_start(em) + (start - em->start);
9737	if (encoded->len > count)
9738	encoded->len = count;
9739	/*
9740	* Don't read beyond what we locked. This also limits the page
9741	* allocations that we'll do.
9742	*/
9743	*disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start;
9744	count = start + *disk_io_size - iocb->ki_pos;
9745	encoded->len = count;
9746	encoded->unencoded_len = count;
9747	*disk_io_size = ALIGN(*disk_io_size, fs_info->sectorsize);
9748	}
9749	btrfs_free_extent_map(em);
9750	em = NULL;
9751
9752	if (*disk_bytenr == EXTENT_MAP_HOLE) {
9753	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9754	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
9755	unlocked = true;
9756	ret = iov_iter_zero(bytes: count, iter);
9757	if (ret != count)
9758	ret = -EFAULT;
9759	} else {
9760	ret = -EIOCBQUEUED;
9761	goto out_unlock_extent;
9762	}
9763
9764	out_em:
9765	btrfs_free_extent_map(em);
9766	out_unlock_extent:
9767	/* Leave inode and extent locked if we need to do a read. */
9768	if (!unlocked && ret != -EIOCBQUEUED)
9769	btrfs_unlock_extent(tree: io_tree, start, end: lockend, cached: cached_state);
9770	out_unlock_inode:
9771	if (!unlocked && ret != -EIOCBQUEUED)
9772	btrfs_inode_unlock(inode, ilock_flags: BTRFS_ILOCK_SHARED);
9773	return ret;
9774	}
9775
9776	ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from,
9777	const struct btrfs_ioctl_encoded_io_args *encoded)
9778	{
9779	struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp));
9780	struct btrfs_root *root = inode->root;
9781	struct btrfs_fs_info *fs_info = root->fs_info;
9782	struct extent_io_tree *io_tree = &inode->io_tree;
9783	struct extent_changeset *data_reserved = NULL;
9784	struct extent_state *cached_state = NULL;
9785	struct btrfs_ordered_extent *ordered;
9786	struct btrfs_file_extent file_extent;
9787	int compression;
9788	size_t orig_count;
9789	u64 start, end;
9790	u64 num_bytes, ram_bytes, disk_num_bytes;
9791	unsigned long nr_folios, i;
9792	struct folio **folios;
9793	struct btrfs_key ins;
9794	bool extent_reserved = false;
9795	struct extent_map *em;
9796	ssize_t ret;
9797
9798	switch (encoded->compression) {
9799	case BTRFS_ENCODED_IO_COMPRESSION_ZLIB:
9800	compression = BTRFS_COMPRESS_ZLIB;
9801	break;
9802	case BTRFS_ENCODED_IO_COMPRESSION_ZSTD:
9803	compression = BTRFS_COMPRESS_ZSTD;
9804	break;
9805	case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K:
9806	case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K:
9807	case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K:
9808	case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K:
9809	case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K:
9810	/* The sector size must match for LZO. */
9811	if (encoded->compression -
9812	BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 !=
9813	fs_info->sectorsize_bits)
9814	return -EINVAL;
9815	compression = BTRFS_COMPRESS_LZO;
9816	break;
9817	default:
9818	return -EINVAL;
9819	}
9820	if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE)
9821	return -EINVAL;
9822
9823	/*
9824	* Compressed extents should always have checksums, so error out if we
9825	* have a NOCOW file or inode was created while mounted with NODATASUM.
9826	*/
9827	if (inode->flags & BTRFS_INODE_NODATASUM)
9828	return -EINVAL;
9829
9830	orig_count = iov_iter_count(i: from);
9831
9832	/* The extent size must be sane. */
9833	if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED ||
9834	orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0)
9835	return -EINVAL;
9836
9837	/*
9838	* The compressed data must be smaller than the decompressed data.
9839	*
9840	* It's of course possible for data to compress to larger or the same
9841	* size, but the buffered I/O path falls back to no compression for such
9842	* data, and we don't want to break any assumptions by creating these
9843	* extents.
9844	*
9845	* Note that this is less strict than the current check we have that the
9846	* compressed data must be at least one sector smaller than the
9847	* decompressed data. We only want to enforce the weaker requirement
9848	* from old kernels that it is at least one byte smaller.
9849	*/
9850	if (orig_count >= encoded->unencoded_len)
9851	return -EINVAL;
9852
9853	/* The extent must start on a sector boundary. */
9854	start = iocb->ki_pos;
9855	if (!IS_ALIGNED(start, fs_info->sectorsize))
9856	return -EINVAL;
9857
9858	/*
9859	* The extent must end on a sector boundary. However, we allow a write
9860	* which ends at or extends i_size to have an unaligned length; we round
9861	* up the extent size and set i_size to the unaligned end.
9862	*/
9863	if (start + encoded->len < inode->vfs_inode.i_size &&
9864	!IS_ALIGNED(start + encoded->len, fs_info->sectorsize))
9865	return -EINVAL;
9866
9867	/* Finally, the offset in the unencoded data must be sector-aligned. */
9868	if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize))
9869	return -EINVAL;
9870
9871	num_bytes = ALIGN(encoded->len, fs_info->sectorsize);
9872	ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize);
9873	end = start + num_bytes - 1;
9874
9875	/*
9876	* If the extent cannot be inline, the compressed data on disk must be
9877	* sector-aligned. For convenience, we extend it with zeroes if it
9878	* isn't.
9879	*/
9880	disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize);
9881	nr_folios = DIV_ROUND_UP(disk_num_bytes, PAGE_SIZE);
9882	folios = kvcalloc(nr_folios, sizeof(struct folio *), GFP_KERNEL_ACCOUNT);
9883	if (!folios)
9884	return -ENOMEM;
9885	for (i = 0; i < nr_folios; i++) {
9886	size_t bytes = min_t(size_t, PAGE_SIZE, iov_iter_count(from));
9887	char *kaddr;
9888
9889	folios[i] = folio_alloc(GFP_KERNEL_ACCOUNT, 0);
9890	if (!folios[i]) {
9891	ret = -ENOMEM;
9892	goto out_folios;
9893	}
9894	kaddr = kmap_local_folio(folio: folios[i], offset: 0);
9895	if (copy_from_iter(addr: kaddr, bytes, i: from) != bytes) {
9896	kunmap_local(kaddr);
9897	ret = -EFAULT;
9898	goto out_folios;
9899	}
9900	if (bytes < PAGE_SIZE)
9901	memset(kaddr + bytes, 0, PAGE_SIZE - bytes);
9902	kunmap_local(kaddr);
9903	}
9904
9905	for (;;) {
9906	ret = btrfs_wait_ordered_range(inode, start, len: num_bytes);
9907	if (ret)
9908	goto out_folios;
9909	ret = invalidate_inode_pages2_range(mapping: inode->vfs_inode.i_mapping,
9910	start: start >> PAGE_SHIFT,
9911	end: end >> PAGE_SHIFT);
9912	if (ret)
9913	goto out_folios;
9914	btrfs_lock_extent(tree: io_tree, start, end, cached: &cached_state);
9915	ordered = btrfs_lookup_ordered_range(inode, file_offset: start, len: num_bytes);
9916	if (!ordered &&
9917	!filemap_range_has_page(inode->vfs_inode.i_mapping, lstart: start, lend: end))
9918	break;
9919	if (ordered)
9920	btrfs_put_ordered_extent(entry: ordered);
9921	btrfs_unlock_extent(tree: io_tree, start, end, cached: &cached_state);
9922	cond_resched();
9923	}
9924
9925	/*
9926	* We don't use the higher-level delalloc space functions because our
9927	* num_bytes and disk_num_bytes are different.
9928	*/
9929	ret = btrfs_alloc_data_chunk_ondemand(inode, bytes: disk_num_bytes);
9930	if (ret)
9931	goto out_unlock;
9932	ret = btrfs_qgroup_reserve_data(inode, reserved: &data_reserved, start, len: num_bytes);
9933	if (ret)
9934	goto out_free_data_space;
9935	ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes,
9936	noflush: false);
9937	if (ret)
9938	goto out_qgroup_free_data;
9939
9940	/* Try an inline extent first. */
9941	if (encoded->unencoded_len == encoded->len &&
9942	encoded->unencoded_offset == 0 &&
9943	can_cow_file_range_inline(inode, offset: start, size: encoded->len, compressed_size: orig_count)) {
9944	ret = __cow_file_range_inline(inode, size: encoded->len,
9945	compressed_size: orig_count, compress_type: compression, compressed_folio: folios[0],
9946	update_i_size: true);
9947	if (ret <= 0) {
9948	if (ret == 0)
9949	ret = orig_count;
9950	goto out_delalloc_release;
9951	}
9952	}
9953
9954	ret = btrfs_reserve_extent(root, ram_bytes: disk_num_bytes, num_bytes: disk_num_bytes,
9955	min_alloc_size: disk_num_bytes, empty_size: 0, hint_byte: 0, ins: &ins, is_data: true, delalloc: true);
9956	if (ret)
9957	goto out_delalloc_release;
9958	extent_reserved = true;
9959
9960	file_extent.disk_bytenr = ins.objectid;
9961	file_extent.disk_num_bytes = ins.offset;
9962	file_extent.num_bytes = num_bytes;
9963	file_extent.ram_bytes = ram_bytes;
9964	file_extent.offset = encoded->unencoded_offset;
9965	file_extent.compression = compression;
9966	em = btrfs_create_io_em(inode, start, file_extent: &file_extent, type: BTRFS_ORDERED_COMPRESSED);
9967	if (IS_ERR(ptr: em)) {
9968	ret = PTR_ERR(ptr: em);
9969	goto out_free_reserved;
9970	}
9971	btrfs_free_extent_map(em);
9972
9973	ordered = btrfs_alloc_ordered_extent(inode, file_offset: start, file_extent: &file_extent,
9974	flags: (1U << BTRFS_ORDERED_ENCODED) |
9975	(1U << BTRFS_ORDERED_COMPRESSED));
9976	if (IS_ERR(ptr: ordered)) {
9977	btrfs_drop_extent_map_range(inode, start, end, skip_pinned: false);
9978	ret = PTR_ERR(ptr: ordered);
9979	goto out_free_reserved;
9980	}
9981	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
9982
9983	if (start + encoded->len > inode->vfs_inode.i_size)
9984	i_size_write(inode: &inode->vfs_inode, i_size: start + encoded->len);
9985
9986	btrfs_unlock_extent(tree: io_tree, start, end, cached: &cached_state);
9987
9988	btrfs_delalloc_release_extents(inode, num_bytes);
9989
9990	btrfs_submit_compressed_write(ordered, compressed_folios: folios, nr_folios, write_flags: 0, writeback: false);
9991	ret = orig_count;
9992	goto out;
9993
9994	out_free_reserved:
9995	btrfs_dec_block_group_reservations(fs_info, start: ins.objectid);
9996	btrfs_free_reserved_extent(fs_info, start: ins.objectid, len: ins.offset, is_delalloc: true);
9997	out_delalloc_release:
9998	btrfs_delalloc_release_extents(inode, num_bytes);
9999	btrfs_delalloc_release_metadata(inode, num_bytes: disk_num_bytes, qgroup_free: ret < 0);
10000	out_qgroup_free_data:
10001	if (ret < 0)
10002	btrfs_qgroup_free_data(inode, reserved: data_reserved, start, len: num_bytes, NULL);
10003	out_free_data_space:
10004	/*
10005	* If btrfs_reserve_extent() succeeded, then we already decremented
10006	* bytes_may_use.
10007	*/
10008	if (!extent_reserved)
10009	btrfs_free_reserved_data_space_noquota(inode, len: disk_num_bytes);
10010	out_unlock:
10011	btrfs_unlock_extent(tree: io_tree, start, end, cached: &cached_state);
10012	out_folios:
10013	for (i = 0; i < nr_folios; i++) {
10014	if (folios[i])
10015	folio_put(folio: folios[i]);
10016	}
10017	kvfree(addr: folios);
10018	out:
10019	if (ret >= 0)
10020	iocb->ki_pos += encoded->len;
10021	return ret;
10022	}
10023
10024	#ifdef CONFIG_SWAP
10025	/*
10026	* Add an entry indicating a block group or device which is pinned by a
10027	* swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
10028	* negative errno on failure.
10029	*/
10030	static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr,
10031	bool is_block_group)
10032	{
10033	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10034	struct btrfs_swapfile_pin *sp, *entry;
10035	struct rb_node **p;
10036	struct rb_node *parent = NULL;
10037
10038	sp = kmalloc(sizeof(*sp), GFP_NOFS);
10039	if (!sp)
10040	return -ENOMEM;
10041	sp->ptr = ptr;
10042	sp->inode = inode;
10043	sp->is_block_group = is_block_group;
10044	sp->bg_extent_count = 1;
10045
10046	spin_lock(lock: &fs_info->swapfile_pins_lock);
10047	p = &fs_info->swapfile_pins.rb_node;
10048	while (*p) {
10049	parent = *p;
10050	entry = rb_entry(parent, struct btrfs_swapfile_pin, node);
10051	if (sp->ptr < entry->ptr ||
10052	(sp->ptr == entry->ptr && sp->inode < entry->inode)) {
10053	p = &(*p)->rb_left;
10054	} else if (sp->ptr > entry->ptr ||
10055	(sp->ptr == entry->ptr && sp->inode > entry->inode)) {
10056	p = &(*p)->rb_right;
10057	} else {
10058	if (is_block_group)
10059	entry->bg_extent_count++;
10060	spin_unlock(lock: &fs_info->swapfile_pins_lock);
10061	kfree(objp: sp);
10062	return 1;
10063	}
10064	}
10065	rb_link_node(node: &sp->node, parent, rb_link: p);
10066	rb_insert_color(&sp->node, &fs_info->swapfile_pins);
10067	spin_unlock(lock: &fs_info->swapfile_pins_lock);
10068	return 0;
10069	}
10070
10071	/* Free all of the entries pinned by this swapfile. */
10072	static void btrfs_free_swapfile_pins(struct inode *inode)
10073	{
10074	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
10075	struct btrfs_swapfile_pin *sp;
10076	struct rb_node *node, *next;
10077
10078	spin_lock(lock: &fs_info->swapfile_pins_lock);
10079	node = rb_first(root: &fs_info->swapfile_pins);
10080	while (node) {
10081	next = rb_next(node);
10082	sp = rb_entry(node, struct btrfs_swapfile_pin, node);
10083	if (sp->inode == inode) {
10084	rb_erase(&sp->node, &fs_info->swapfile_pins);
10085	if (sp->is_block_group) {
10086	btrfs_dec_block_group_swap_extents(bg: sp->ptr,
10087	amount: sp->bg_extent_count);
10088	btrfs_put_block_group(cache: sp->ptr);
10089	}
10090	kfree(objp: sp);
10091	}
10092	node = next;
10093	}
10094	spin_unlock(lock: &fs_info->swapfile_pins_lock);
10095	}
10096
10097	struct btrfs_swap_info {
10098	u64 start;
10099	u64 block_start;
10100	u64 block_len;
10101	u64 lowest_ppage;
10102	u64 highest_ppage;
10103	unsigned long nr_pages;
10104	int nr_extents;
10105	};
10106
10107	static int btrfs_add_swap_extent(struct swap_info_struct *sis,
10108	struct btrfs_swap_info *bsi)
10109	{
10110	unsigned long nr_pages;
10111	unsigned long max_pages;
10112	u64 first_ppage, first_ppage_reported, next_ppage;
10113	int ret;
10114
10115	/*
10116	* Our swapfile may have had its size extended after the swap header was
10117	* written. In that case activating the swapfile should not go beyond
10118	* the max size set in the swap header.
10119	*/
10120	if (bsi->nr_pages >= sis->max)
10121	return 0;
10122
10123	max_pages = sis->max - bsi->nr_pages;
10124	first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT;
10125	next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT;
10126
10127	if (first_ppage >= next_ppage)
10128	return 0;
10129	nr_pages = next_ppage - first_ppage;
10130	nr_pages = min(nr_pages, max_pages);
10131
10132	first_ppage_reported = first_ppage;
10133	if (bsi->start == 0)
10134	first_ppage_reported++;
10135	if (bsi->lowest_ppage > first_ppage_reported)
10136	bsi->lowest_ppage = first_ppage_reported;
10137	if (bsi->highest_ppage < (next_ppage - 1))
10138	bsi->highest_ppage = next_ppage - 1;
10139
10140	ret = add_swap_extent(sis, start_page: bsi->nr_pages, nr_pages, start_block: first_ppage);
10141	if (ret < 0)
10142	return ret;
10143	bsi->nr_extents += ret;
10144	bsi->nr_pages += nr_pages;
10145	return 0;
10146	}
10147
10148	static void btrfs_swap_deactivate(struct file *file)
10149	{
10150	struct inode *inode = file_inode(f: file);
10151
10152	btrfs_free_swapfile_pins(inode);
10153	atomic_dec(v: &BTRFS_I(inode)->root->nr_swapfiles);
10154	}
10155
10156	static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
10157	sector_t *span)
10158	{
10159	struct inode *inode = file_inode(f: file);
10160	struct btrfs_root *root = BTRFS_I(inode)->root;
10161	struct btrfs_fs_info *fs_info = root->fs_info;
10162	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
10163	struct extent_state *cached_state = NULL;
10164	struct btrfs_chunk_map *map = NULL;
10165	struct btrfs_device *device = NULL;
10166	struct btrfs_swap_info bsi = {
10167	.lowest_ppage = (sector_t)-1ULL,
10168	};
10169	struct btrfs_backref_share_check_ctx *backref_ctx = NULL;
10170	struct btrfs_path *path = NULL;
10171	int ret = 0;
10172	u64 isize;
10173	u64 prev_extent_end = 0;
10174
10175	/*
10176	* Acquire the inode's mmap lock to prevent races with memory mapped
10177	* writes, as they could happen after we flush delalloc below and before
10178	* we lock the extent range further below. The inode was already locked
10179	* up in the call chain.
10180	*/
10181	btrfs_assert_inode_locked(BTRFS_I(inode));
10182	down_write(sem: &BTRFS_I(inode)->i_mmap_lock);
10183
10184	/*
10185	* If the swap file was just created, make sure delalloc is done. If the
10186	* file changes again after this, the user is doing something stupid and
10187	* we don't really care.
10188	*/
10189	ret = btrfs_wait_ordered_range(BTRFS_I(inode), start: 0, len: (u64)-1);
10190	if (ret)
10191	goto out_unlock_mmap;
10192
10193	/*
10194	* The inode is locked, so these flags won't change after we check them.
10195	*/
10196	if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {
10197	btrfs_warn(fs_info, "swapfile must not be compressed");
10198	ret = -EINVAL;
10199	goto out_unlock_mmap;
10200	}
10201	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {
10202	btrfs_warn(fs_info, "swapfile must not be copy-on-write");
10203	ret = -EINVAL;
10204	goto out_unlock_mmap;
10205	}
10206	if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
10207	btrfs_warn(fs_info, "swapfile must not be checksummed");
10208	ret = -EINVAL;
10209	goto out_unlock_mmap;
10210	}
10211
10212	path = btrfs_alloc_path();
10213	backref_ctx = btrfs_alloc_backref_share_check_ctx();
10214	if (!path || !backref_ctx) {
10215	ret = -ENOMEM;
10216	goto out_unlock_mmap;
10217	}
10218
10219	/*
10220	* Balance or device remove/replace/resize can move stuff around from
10221	* under us. The exclop protection makes sure they aren't running/won't
10222	* run concurrently while we are mapping the swap extents, and
10223	* fs_info->swapfile_pins prevents them from running while the swap
10224	* file is active and moving the extents. Note that this also prevents
10225	* a concurrent device add which isn't actually necessary, but it's not
10226	* really worth the trouble to allow it.
10227	*/
10228	if (!btrfs_exclop_start(fs_info, type: BTRFS_EXCLOP_SWAP_ACTIVATE)) {
10229	btrfs_warn(fs_info,
10230	"cannot activate swapfile while exclusive operation is running");
10231	ret = -EBUSY;
10232	goto out_unlock_mmap;
10233	}
10234
10235	/*
10236	* Prevent snapshot creation while we are activating the swap file.
10237	* We do not want to race with snapshot creation. If snapshot creation
10238	* already started before we bumped nr_swapfiles from 0 to 1 and
10239	* completes before the first write into the swap file after it is
10240	* activated, than that write would fallback to COW.
10241	*/
10242	if (!btrfs_drew_try_write_lock(lock: &root->snapshot_lock)) {
10243	btrfs_exclop_finish(fs_info);
10244	btrfs_warn(fs_info,
10245	"cannot activate swapfile because snapshot creation is in progress");
10246	ret = -EINVAL;
10247	goto out_unlock_mmap;
10248	}
10249	/*
10250	* Snapshots can create extents which require COW even if NODATACOW is
10251	* set. We use this counter to prevent snapshots. We must increment it
10252	* before walking the extents because we don't want a concurrent
10253	* snapshot to run after we've already checked the extents.
10254	*
10255	* It is possible that subvolume is marked for deletion but still not
10256	* removed yet. To prevent this race, we check the root status before
10257	* activating the swapfile.
10258	*/
10259	spin_lock(lock: &root->root_item_lock);
10260	if (btrfs_root_dead(root)) {
10261	spin_unlock(lock: &root->root_item_lock);
10262
10263	btrfs_drew_write_unlock(lock: &root->snapshot_lock);
10264	btrfs_exclop_finish(fs_info);
10265	btrfs_warn(fs_info,
10266	"cannot activate swapfile because subvolume %llu is being deleted",
10267	btrfs_root_id(root));
10268	ret = -EPERM;
10269	goto out_unlock_mmap;
10270	}
10271	atomic_inc(v: &root->nr_swapfiles);
10272	spin_unlock(lock: &root->root_item_lock);
10273
10274	isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
10275
10276	btrfs_lock_extent(tree: io_tree, start: 0, end: isize - 1, cached: &cached_state);
10277	while (prev_extent_end < isize) {
10278	struct btrfs_key key;
10279	struct extent_buffer *leaf;
10280	struct btrfs_file_extent_item *ei;
10281	struct btrfs_block_group *bg;
10282	u64 logical_block_start;
10283	u64 physical_block_start;
10284	u64 extent_gen;
10285	u64 disk_bytenr;
10286	u64 len;
10287
10288	key.objectid = btrfs_ino(BTRFS_I(inode));
10289	key.type = BTRFS_EXTENT_DATA_KEY;
10290	key.offset = prev_extent_end;
10291
10292	ret = btrfs_search_slot(NULL, root, key: &key, p: path, ins_len: 0, cow: 0);
10293	if (ret < 0)
10294	goto out;
10295
10296	/*
10297	* If key not found it means we have an implicit hole (NO_HOLES
10298	* is enabled).
10299	*/
10300	if (ret > 0) {
10301	btrfs_warn(fs_info, "swapfile must not have holes");
10302	ret = -EINVAL;
10303	goto out;
10304	}
10305
10306	leaf = path->nodes[0];
10307	ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item);
10308
10309	if (btrfs_file_extent_type(eb: leaf, s: ei) == BTRFS_FILE_EXTENT_INLINE) {
10310	/*
10311	* It's unlikely we'll ever actually find ourselves
10312	* here, as a file small enough to fit inline won't be
10313	* big enough to store more than the swap header, but in
10314	* case something changes in the future, let's catch it
10315	* here rather than later.
10316	*/
10317	btrfs_warn(fs_info, "swapfile must not be inline");
10318	ret = -EINVAL;
10319	goto out;
10320	}
10321
10322	if (btrfs_file_extent_compression(eb: leaf, s: ei) != BTRFS_COMPRESS_NONE) {
10323	btrfs_warn(fs_info, "swapfile must not be compressed");
10324	ret = -EINVAL;
10325	goto out;
10326	}
10327
10328	disk_bytenr = btrfs_file_extent_disk_bytenr(eb: leaf, s: ei);
10329	if (disk_bytenr == 0) {
10330	btrfs_warn(fs_info, "swapfile must not have holes");
10331	ret = -EINVAL;
10332	goto out;
10333	}
10334
10335	logical_block_start = disk_bytenr + btrfs_file_extent_offset(eb: leaf, s: ei);
10336	extent_gen = btrfs_file_extent_generation(eb: leaf, s: ei);
10337	prev_extent_end = btrfs_file_extent_end(path);
10338
10339	if (prev_extent_end > isize)
10340	len = isize - key.offset;
10341	else
10342	len = btrfs_file_extent_num_bytes(eb: leaf, s: ei);
10343
10344	backref_ctx->curr_leaf_bytenr = leaf->start;
10345
10346	/*
10347	* Don't need the path anymore, release to avoid deadlocks when
10348	* calling btrfs_is_data_extent_shared() because when joining a
10349	* transaction it can block waiting for the current one's commit
10350	* which in turn may be trying to lock the same leaf to flush
10351	* delayed items for example.
10352	*/
10353	btrfs_release_path(p: path);
10354
10355	ret = btrfs_is_data_extent_shared(BTRFS_I(inode), bytenr: disk_bytenr,
10356	extent_gen, ctx: backref_ctx);
10357	if (ret < 0) {
10358	goto out;
10359	} else if (ret > 0) {
10360	btrfs_warn(fs_info,
10361	"swapfile must not be copy-on-write");
10362	ret = -EINVAL;
10363	goto out;
10364	}
10365
10366	map = btrfs_get_chunk_map(fs_info, logical: logical_block_start, length: len);
10367	if (IS_ERR(ptr: map)) {
10368	ret = PTR_ERR(ptr: map);
10369	goto out;
10370	}
10371
10372	if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
10373	btrfs_warn(fs_info,
10374	"swapfile must have single data profile");
10375	ret = -EINVAL;
10376	goto out;
10377	}
10378
10379	if (device == NULL) {
10380	device = map->stripes[0].dev;
10381	ret = btrfs_add_swapfile_pin(inode, ptr: device, is_block_group: false);
10382	if (ret == 1)
10383	ret = 0;
10384	else if (ret)
10385	goto out;
10386	} else if (device != map->stripes[0].dev) {
10387	btrfs_warn(fs_info, "swapfile must be on one device");
10388	ret = -EINVAL;
10389	goto out;
10390	}
10391
10392	physical_block_start = (map->stripes[0].physical +
10393	(logical_block_start - map->start));
10394	btrfs_free_chunk_map(map);
10395	map = NULL;
10396
10397	bg = btrfs_lookup_block_group(info: fs_info, bytenr: logical_block_start);
10398	if (!bg) {
10399	btrfs_warn(fs_info,
10400	"could not find block group containing swapfile");
10401	ret = -EINVAL;
10402	goto out;
10403	}
10404
10405	if (!btrfs_inc_block_group_swap_extents(bg)) {
10406	btrfs_warn(fs_info,
10407	"block group for swapfile at %llu is read-only%s",
10408	bg->start,
10409	atomic_read(&fs_info->scrubs_running) ?
10410	" (scrub running)" : "");
10411	btrfs_put_block_group(cache: bg);
10412	ret = -EINVAL;
10413	goto out;
10414	}
10415
10416	ret = btrfs_add_swapfile_pin(inode, ptr: bg, is_block_group: true);
10417	if (ret) {
10418	btrfs_put_block_group(cache: bg);
10419	if (ret == 1)
10420	ret = 0;
10421	else
10422	goto out;
10423	}
10424
10425	if (bsi.block_len &&
10426	bsi.block_start + bsi.block_len == physical_block_start) {
10427	bsi.block_len += len;
10428	} else {
10429	if (bsi.block_len) {
10430	ret = btrfs_add_swap_extent(sis, bsi: &bsi);
10431	if (ret)
10432	goto out;
10433	}
10434	bsi.start = key.offset;
10435	bsi.block_start = physical_block_start;
10436	bsi.block_len = len;
10437	}
10438
10439	if (fatal_signal_pending(current)) {
10440	ret = -EINTR;
10441	goto out;
10442	}
10443
10444	cond_resched();
10445	}
10446
10447	if (bsi.block_len)
10448	ret = btrfs_add_swap_extent(sis, bsi: &bsi);
10449
10450	out:
10451	if (!IS_ERR_OR_NULL(ptr: map))
10452	btrfs_free_chunk_map(map);
10453
10454	btrfs_unlock_extent(tree: io_tree, start: 0, end: isize - 1, cached: &cached_state);
10455
10456	if (ret)
10457	btrfs_swap_deactivate(file);
10458
10459	btrfs_drew_write_unlock(lock: &root->snapshot_lock);
10460
10461	btrfs_exclop_finish(fs_info);
10462
10463	out_unlock_mmap:
10464	up_write(sem: &BTRFS_I(inode)->i_mmap_lock);
10465	btrfs_free_backref_share_ctx(ctx: backref_ctx);
10466	btrfs_free_path(p: path);
10467	if (ret)
10468	return ret;
10469
10470	if (device)
10471	sis->bdev = device->bdev;
10472	*span = bsi.highest_ppage - bsi.lowest_ppage + 1;
10473	sis->max = bsi.nr_pages;
10474	sis->pages = bsi.nr_pages - 1;
10475	return bsi.nr_extents;
10476	}
10477	#else
10478	static void btrfs_swap_deactivate(struct file *file)
10479	{
10480	}
10481
10482	static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
10483	sector_t *span)
10484	{
10485	return -EOPNOTSUPP;
10486	}
10487	#endif
10488
10489	/*
10490	* Update the number of bytes used in the VFS' inode. When we replace extents in
10491	* a range (clone, dedupe, fallocate's zero range), we must update the number of
10492	* bytes used by the inode in an atomic manner, so that concurrent stat(2) calls
10493	* always get a correct value.
10494	*/
10495	void btrfs_update_inode_bytes(struct btrfs_inode *inode,
10496	const u64 add_bytes,
10497	const u64 del_bytes)
10498	{
10499	if (add_bytes == del_bytes)
10500	return;
10501
10502	spin_lock(lock: &inode->lock);
10503	if (del_bytes > 0)
10504	inode_sub_bytes(inode: &inode->vfs_inode, bytes: del_bytes);
10505	if (add_bytes > 0)
10506	inode_add_bytes(inode: &inode->vfs_inode, bytes: add_bytes);
10507	spin_unlock(lock: &inode->lock);
10508	}
10509
10510	/*
10511	* Verify that there are no ordered extents for a given file range.
10512	*
10513	* @inode: The target inode.
10514	* @start: Start offset of the file range, should be sector size aligned.
10515	* @end: End offset (inclusive) of the file range, its value +1 should be
10516	* sector size aligned.
10517	*
10518	* This should typically be used for cases where we locked an inode's VFS lock in
10519	* exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode,
10520	* we have flushed all delalloc in the range, we have waited for all ordered
10521	* extents in the range to complete and finally we have locked the file range in
10522	* the inode's io_tree.
10523	*/
10524	void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end)
10525	{
10526	struct btrfs_root *root = inode->root;
10527	struct btrfs_ordered_extent *ordered;
10528
10529	if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
10530	return;
10531
10532	ordered = btrfs_lookup_first_ordered_range(inode, file_offset: start, len: end + 1 - start);
10533	if (ordered) {
10534	btrfs_err(root->fs_info,
10535	"found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])",
10536	start, end, btrfs_ino(inode), btrfs_root_id(root),
10537	ordered->file_offset,
10538	ordered->file_offset + ordered->num_bytes - 1);
10539	btrfs_put_ordered_extent(entry: ordered);
10540	}
10541
10542	ASSERT(ordered == NULL);
10543	}
10544
10545	/*
10546	* Find the first inode with a minimum number.
10547	*
10548	* @root: The root to search for.
10549	* @min_ino: The minimum inode number.
10550	*
10551	* Find the first inode in the @root with a number >= @min_ino and return it.
10552	* Returns NULL if no such inode found.
10553	*/
10554	struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino)
10555	{
10556	struct btrfs_inode *inode;
10557	unsigned long from = min_ino;
10558
10559	xa_lock(&root->inodes);
10560	while (true) {
10561	inode = xa_find(xa: &root->inodes, index: &from, ULONG_MAX, XA_PRESENT);
10562	if (!inode)
10563	break;
10564	if (igrab(&inode->vfs_inode))
10565	break;
10566
10567	from = btrfs_ino(inode) + 1;
10568	cond_resched_lock(&root->inodes.xa_lock);
10569	}
10570	xa_unlock(&root->inodes);
10571
10572	return inode;
10573	}
10574
10575	static const struct inode_operations btrfs_dir_inode_operations = {
10576	.getattr = btrfs_getattr,
10577	.lookup = btrfs_lookup,
10578	.create = btrfs_create,
10579	.unlink = btrfs_unlink,
10580	.link = btrfs_link,
10581	.mkdir = btrfs_mkdir,
10582	.rmdir = btrfs_rmdir,
10583	.rename = btrfs_rename2,
10584	.symlink = btrfs_symlink,
10585	.setattr = btrfs_setattr,
10586	.mknod = btrfs_mknod,
10587	.listxattr = btrfs_listxattr,
10588	.permission = btrfs_permission,
10589	.get_inode_acl = btrfs_get_acl,
10590	.set_acl = btrfs_set_acl,
10591	.update_time = btrfs_update_time,
10592	.tmpfile = btrfs_tmpfile,
10593	.fileattr_get = btrfs_fileattr_get,
10594	.fileattr_set = btrfs_fileattr_set,
10595	};
10596
10597	static const struct file_operations btrfs_dir_file_operations = {
10598	.llseek = btrfs_dir_llseek,
10599	.read = generic_read_dir,
10600	.iterate_shared = btrfs_real_readdir,
10601	.open = btrfs_opendir,
10602	.unlocked_ioctl = btrfs_ioctl,
10603	#ifdef CONFIG_COMPAT
10604	.compat_ioctl = btrfs_compat_ioctl,
10605	#endif
10606	.release = btrfs_release_file,
10607	.fsync = btrfs_sync_file,
10608	};
10609
10610	/*
10611	* btrfs doesn't support the bmap operation because swapfiles
10612	* use bmap to make a mapping of extents in the file. They assume
10613	* these extents won't change over the life of the file and they
10614	* use the bmap result to do IO directly to the drive.
10615	*
10616	* the btrfs bmap call would return logical addresses that aren't
10617	* suitable for IO and they also will change frequently as COW
10618	* operations happen. So, swapfile + btrfs == corruption.
10619	*
10620	* For now we're avoiding this by dropping bmap.
10621	*/
10622	static const struct address_space_operations btrfs_aops = {
10623	.read_folio = btrfs_read_folio,
10624	.writepages = btrfs_writepages,
10625	.readahead = btrfs_readahead,
10626	.invalidate_folio = btrfs_invalidate_folio,
10627	.launder_folio = btrfs_launder_folio,
10628	.release_folio = btrfs_release_folio,
10629	.migrate_folio = btrfs_migrate_folio,
10630	.dirty_folio = filemap_dirty_folio,
10631	.error_remove_folio = generic_error_remove_folio,
10632	.swap_activate = btrfs_swap_activate,
10633	.swap_deactivate = btrfs_swap_deactivate,
10634	};
10635
10636	static const struct inode_operations btrfs_file_inode_operations = {
10637	.getattr = btrfs_getattr,
10638	.setattr = btrfs_setattr,
10639	.listxattr = btrfs_listxattr,
10640	.permission = btrfs_permission,
10641	.fiemap = btrfs_fiemap,
10642	.get_inode_acl = btrfs_get_acl,
10643	.set_acl = btrfs_set_acl,
10644	.update_time = btrfs_update_time,
10645	.fileattr_get = btrfs_fileattr_get,
10646	.fileattr_set = btrfs_fileattr_set,
10647	};
10648	static const struct inode_operations btrfs_special_inode_operations = {
10649	.getattr = btrfs_getattr,
10650	.setattr = btrfs_setattr,
10651	.permission = btrfs_permission,
10652	.listxattr = btrfs_listxattr,
10653	.get_inode_acl = btrfs_get_acl,
10654	.set_acl = btrfs_set_acl,
10655	.update_time = btrfs_update_time,
10656	};
10657	static const struct inode_operations btrfs_symlink_inode_operations = {
10658	.get_link = page_get_link,
10659	.getattr = btrfs_getattr,
10660	.setattr = btrfs_setattr,
10661	.permission = btrfs_permission,
10662	.listxattr = btrfs_listxattr,
10663	.update_time = btrfs_update_time,
10664	};
10665
10666	const struct dentry_operations btrfs_dentry_operations = {
10667	.d_delete = btrfs_dentry_delete,
10668	};
10669