Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc.
4 * All Rights Reserved.
5 */
6#include "xfs_platform.h"
7#include "xfs_fs.h"
8#include "xfs_shared.h"
9#include "xfs_format.h"
10#include "xfs_log_format.h"
11#include "xfs_trans_resv.h"
12#include "xfs_mount.h"
13#include "xfs_btree.h"
14#include "xfs_btree_staging.h"
15#include "xfs_alloc_btree.h"
16#include "xfs_alloc.h"
17#include "xfs_extent_busy.h"
18#include "xfs_error.h"
19#include "xfs_health.h"
20#include "xfs_trace.h"
21#include "xfs_trans.h"
22#include "xfs_ag.h"
23
24static struct kmem_cache *xfs_allocbt_cur_cache;
25
26STATIC struct xfs_btree_cur *
27xfs_bnobt_dup_cursor(
28 struct xfs_btree_cur *cur)
29{
30 return xfs_bnobt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp,
31 to_perag(cur->bc_group));
32}
33
34STATIC struct xfs_btree_cur *
35xfs_cntbt_dup_cursor(
36 struct xfs_btree_cur *cur)
37{
38 return xfs_cntbt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp,
39 to_perag(cur->bc_group));
40}
41
42STATIC void
43xfs_allocbt_set_root(
44 struct xfs_btree_cur *cur,
45 const union xfs_btree_ptr *ptr,
46 int inc)
47{
48 struct xfs_perag *pag = to_perag(cur->bc_group);
49 struct xfs_buf *agbp = cur->bc_ag.agbp;
50 struct xfs_agf *agf = agbp->b_addr;
51
52 ASSERT(ptr->s != 0);
53
54 if (xfs_btree_is_bno(cur->bc_ops)) {
55 agf->agf_bno_root = ptr->s;
56 be32_add_cpu(&agf->agf_bno_level, inc);
57 pag->pagf_bno_level += inc;
58 } else {
59 agf->agf_cnt_root = ptr->s;
60 be32_add_cpu(&agf->agf_cnt_level, inc);
61 pag->pagf_cnt_level += inc;
62 }
63
64 xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
65}
66
67STATIC int
68xfs_allocbt_alloc_block(
69 struct xfs_btree_cur *cur,
70 const union xfs_btree_ptr *start,
71 union xfs_btree_ptr *new,
72 int *stat)
73{
74 int error;
75 xfs_agblock_t bno;
76
77 /* Allocate the new block from the freelist. If we can't, give up. */
78 error = xfs_alloc_get_freelist(to_perag(cur->bc_group), cur->bc_tp,
79 cur->bc_ag.agbp, &bno, 1);
80 if (error)
81 return error;
82
83 if (bno == NULLAGBLOCK) {
84 *stat = 0;
85 return 0;
86 }
87
88 atomic64_inc(&cur->bc_mp->m_allocbt_blks);
89 xfs_extent_busy_reuse(cur->bc_group, bno, 1, false);
90
91 new->s = cpu_to_be32(bno);
92
93 *stat = 1;
94 return 0;
95}
96
97STATIC int
98xfs_allocbt_free_block(
99 struct xfs_btree_cur *cur,
100 struct xfs_buf *bp)
101{
102 struct xfs_buf *agbp = cur->bc_ag.agbp;
103 xfs_agblock_t bno;
104 int error;
105
106 bno = xfs_daddr_to_agbno(cur->bc_mp, xfs_buf_daddr(bp));
107 error = xfs_alloc_put_freelist(to_perag(cur->bc_group), cur->bc_tp,
108 agbp, NULL, bno, 1);
109 if (error)
110 return error;
111
112 atomic64_dec(&cur->bc_mp->m_allocbt_blks);
113 xfs_extent_busy_insert(cur->bc_tp, pag_group(agbp->b_pag), bno, 1,
114 XFS_EXTENT_BUSY_SKIP_DISCARD);
115 return 0;
116}
117
118STATIC int
119xfs_allocbt_get_minrecs(
120 struct xfs_btree_cur *cur,
121 int level)
122{
123 return cur->bc_mp->m_alloc_mnr[level != 0];
124}
125
126STATIC int
127xfs_allocbt_get_maxrecs(
128 struct xfs_btree_cur *cur,
129 int level)
130{
131 return cur->bc_mp->m_alloc_mxr[level != 0];
132}
133
134STATIC void
135xfs_allocbt_init_key_from_rec(
136 union xfs_btree_key *key,
137 const union xfs_btree_rec *rec)
138{
139 key->alloc.ar_startblock = rec->alloc.ar_startblock;
140 key->alloc.ar_blockcount = rec->alloc.ar_blockcount;
141}
142
143STATIC void
144xfs_bnobt_init_high_key_from_rec(
145 union xfs_btree_key *key,
146 const union xfs_btree_rec *rec)
147{
148 __u32 x;
149
150 x = be32_to_cpu(rec->alloc.ar_startblock);
151 x += be32_to_cpu(rec->alloc.ar_blockcount) - 1;
152 key->alloc.ar_startblock = cpu_to_be32(x);
153 key->alloc.ar_blockcount = 0;
154}
155
156STATIC void
157xfs_cntbt_init_high_key_from_rec(
158 union xfs_btree_key *key,
159 const union xfs_btree_rec *rec)
160{
161 key->alloc.ar_blockcount = rec->alloc.ar_blockcount;
162 key->alloc.ar_startblock = 0;
163}
164
165STATIC void
166xfs_allocbt_init_rec_from_cur(
167 struct xfs_btree_cur *cur,
168 union xfs_btree_rec *rec)
169{
170 rec->alloc.ar_startblock = cpu_to_be32(cur->bc_rec.a.ar_startblock);
171 rec->alloc.ar_blockcount = cpu_to_be32(cur->bc_rec.a.ar_blockcount);
172}
173
174STATIC void
175xfs_allocbt_init_ptr_from_cur(
176 struct xfs_btree_cur *cur,
177 union xfs_btree_ptr *ptr)
178{
179 struct xfs_agf *agf = cur->bc_ag.agbp->b_addr;
180
181 ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agf->agf_seqno));
182
183 if (xfs_btree_is_bno(cur->bc_ops))
184 ptr->s = agf->agf_bno_root;
185 else
186 ptr->s = agf->agf_cnt_root;
187}
188
189STATIC int
190xfs_bnobt_cmp_key_with_cur(
191 struct xfs_btree_cur *cur,
192 const union xfs_btree_key *key)
193{
194 struct xfs_alloc_rec_incore *rec = &cur->bc_rec.a;
195 const struct xfs_alloc_rec *kp = &key->alloc;
196
197 return cmp_int(be32_to_cpu(kp->ar_startblock),
198 rec->ar_startblock);
199}
200
201STATIC int
202xfs_cntbt_cmp_key_with_cur(
203 struct xfs_btree_cur *cur,
204 const union xfs_btree_key *key)
205{
206 struct xfs_alloc_rec_incore *rec = &cur->bc_rec.a;
207 const struct xfs_alloc_rec *kp = &key->alloc;
208
209 return cmp_int(be32_to_cpu(kp->ar_blockcount), rec->ar_blockcount) ?:
210 cmp_int(be32_to_cpu(kp->ar_startblock), rec->ar_startblock);
211}
212
213STATIC int
214xfs_bnobt_cmp_two_keys(
215 struct xfs_btree_cur *cur,
216 const union xfs_btree_key *k1,
217 const union xfs_btree_key *k2,
218 const union xfs_btree_key *mask)
219{
220 ASSERT(!mask || mask->alloc.ar_startblock);
221
222 return cmp_int(be32_to_cpu(k1->alloc.ar_startblock),
223 be32_to_cpu(k2->alloc.ar_startblock));
224}
225
226STATIC int
227xfs_cntbt_cmp_two_keys(
228 struct xfs_btree_cur *cur,
229 const union xfs_btree_key *k1,
230 const union xfs_btree_key *k2,
231 const union xfs_btree_key *mask)
232{
233 ASSERT(!mask || (mask->alloc.ar_blockcount &&
234 mask->alloc.ar_startblock));
235
236 return cmp_int(be32_to_cpu(k1->alloc.ar_blockcount),
237 be32_to_cpu(k2->alloc.ar_blockcount)) ?:
238 cmp_int(be32_to_cpu(k1->alloc.ar_startblock),
239 be32_to_cpu(k2->alloc.ar_startblock));
240}
241
242static xfs_failaddr_t
243xfs_allocbt_verify(
244 struct xfs_buf *bp)
245{
246 struct xfs_mount *mp = bp->b_mount;
247 struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp);
248 struct xfs_perag *pag = bp->b_pag;
249 xfs_failaddr_t fa;
250 unsigned int level;
251
252 if (!xfs_verify_magic(bp, block->bb_magic))
253 return __this_address;
254
255 if (xfs_has_crc(mp)) {
256 fa = xfs_btree_agblock_v5hdr_verify(bp);
257 if (fa)
258 return fa;
259 }
260
261 /*
262 * The perag may not be attached during grow operations or fully
263 * initialized from the AGF during log recovery. Therefore we can only
264 * check against maximum tree depth from those contexts.
265 *
266 * Otherwise check against the per-tree limit. Peek at one of the
267 * verifier magic values to determine the type of tree we're verifying
268 * against.
269 */
270 level = be16_to_cpu(block->bb_level);
271 if (pag && xfs_perag_initialised_agf(pag)) {
272 unsigned int maxlevel, repair_maxlevel = 0;
273
274 /*
275 * Online repair could be rewriting the free space btrees, so
276 * we'll validate against the larger of either tree while this
277 * is going on.
278 */
279 if (bp->b_ops->magic[0] == cpu_to_be32(XFS_ABTC_MAGIC)) {
280 maxlevel = pag->pagf_cnt_level;
281#ifdef CONFIG_XFS_ONLINE_REPAIR
282 repair_maxlevel = pag->pagf_repair_cnt_level;
283#endif
284 } else {
285 maxlevel = pag->pagf_bno_level;
286#ifdef CONFIG_XFS_ONLINE_REPAIR
287 repair_maxlevel = pag->pagf_repair_bno_level;
288#endif
289 }
290
291 if (level >= max(maxlevel, repair_maxlevel))
292 return __this_address;
293 } else if (level >= mp->m_alloc_maxlevels)
294 return __this_address;
295
296 return xfs_btree_agblock_verify(bp, mp->m_alloc_mxr[level != 0]);
297}
298
299static void
300xfs_allocbt_read_verify(
301 struct xfs_buf *bp)
302{
303 xfs_failaddr_t fa;
304
305 if (!xfs_btree_agblock_verify_crc(bp))
306 xfs_verifier_error(bp, -EFSBADCRC, __this_address);
307 else {
308 fa = xfs_allocbt_verify(bp);
309 if (fa)
310 xfs_verifier_error(bp, -EFSCORRUPTED, fa);
311 }
312
313 if (bp->b_error)
314 trace_xfs_btree_corrupt(bp, _RET_IP_);
315}
316
317static void
318xfs_allocbt_write_verify(
319 struct xfs_buf *bp)
320{
321 xfs_failaddr_t fa;
322
323 fa = xfs_allocbt_verify(bp);
324 if (fa) {
325 trace_xfs_btree_corrupt(bp, _RET_IP_);
326 xfs_verifier_error(bp, -EFSCORRUPTED, fa);
327 return;
328 }
329 xfs_btree_agblock_calc_crc(bp);
330
331}
332
333const struct xfs_buf_ops xfs_bnobt_buf_ops = {
334 .name = "xfs_bnobt",
335 .magic = { cpu_to_be32(XFS_ABTB_MAGIC),
336 cpu_to_be32(XFS_ABTB_CRC_MAGIC) },
337 .verify_read = xfs_allocbt_read_verify,
338 .verify_write = xfs_allocbt_write_verify,
339 .verify_struct = xfs_allocbt_verify,
340};
341
342const struct xfs_buf_ops xfs_cntbt_buf_ops = {
343 .name = "xfs_cntbt",
344 .magic = { cpu_to_be32(XFS_ABTC_MAGIC),
345 cpu_to_be32(XFS_ABTC_CRC_MAGIC) },
346 .verify_read = xfs_allocbt_read_verify,
347 .verify_write = xfs_allocbt_write_verify,
348 .verify_struct = xfs_allocbt_verify,
349};
350
351STATIC int
352xfs_bnobt_keys_inorder(
353 struct xfs_btree_cur *cur,
354 const union xfs_btree_key *k1,
355 const union xfs_btree_key *k2)
356{
357 return be32_to_cpu(k1->alloc.ar_startblock) <
358 be32_to_cpu(k2->alloc.ar_startblock);
359}
360
361STATIC int
362xfs_bnobt_recs_inorder(
363 struct xfs_btree_cur *cur,
364 const union xfs_btree_rec *r1,
365 const union xfs_btree_rec *r2)
366{
367 return be32_to_cpu(r1->alloc.ar_startblock) +
368 be32_to_cpu(r1->alloc.ar_blockcount) <=
369 be32_to_cpu(r2->alloc.ar_startblock);
370}
371
372STATIC int
373xfs_cntbt_keys_inorder(
374 struct xfs_btree_cur *cur,
375 const union xfs_btree_key *k1,
376 const union xfs_btree_key *k2)
377{
378 return be32_to_cpu(k1->alloc.ar_blockcount) <
379 be32_to_cpu(k2->alloc.ar_blockcount) ||
380 (k1->alloc.ar_blockcount == k2->alloc.ar_blockcount &&
381 be32_to_cpu(k1->alloc.ar_startblock) <
382 be32_to_cpu(k2->alloc.ar_startblock));
383}
384
385STATIC int
386xfs_cntbt_recs_inorder(
387 struct xfs_btree_cur *cur,
388 const union xfs_btree_rec *r1,
389 const union xfs_btree_rec *r2)
390{
391 return be32_to_cpu(r1->alloc.ar_blockcount) <
392 be32_to_cpu(r2->alloc.ar_blockcount) ||
393 (r1->alloc.ar_blockcount == r2->alloc.ar_blockcount &&
394 be32_to_cpu(r1->alloc.ar_startblock) <
395 be32_to_cpu(r2->alloc.ar_startblock));
396}
397
398STATIC enum xbtree_key_contig
399xfs_allocbt_keys_contiguous(
400 struct xfs_btree_cur *cur,
401 const union xfs_btree_key *key1,
402 const union xfs_btree_key *key2,
403 const union xfs_btree_key *mask)
404{
405 ASSERT(!mask || mask->alloc.ar_startblock);
406
407 return xbtree_key_contig(be32_to_cpu(key1->alloc.ar_startblock),
408 be32_to_cpu(key2->alloc.ar_startblock));
409}
410
411const struct xfs_btree_ops xfs_bnobt_ops = {
412 .name = "bno",
413 .type = XFS_BTREE_TYPE_AG,
414
415 .rec_len = sizeof(xfs_alloc_rec_t),
416 .key_len = sizeof(xfs_alloc_key_t),
417 .ptr_len = XFS_BTREE_SHORT_PTR_LEN,
418
419 .lru_refs = XFS_ALLOC_BTREE_REF,
420 .statoff = XFS_STATS_CALC_INDEX(xs_abtb_2),
421 .sick_mask = XFS_SICK_AG_BNOBT,
422
423 .dup_cursor = xfs_bnobt_dup_cursor,
424 .set_root = xfs_allocbt_set_root,
425 .alloc_block = xfs_allocbt_alloc_block,
426 .free_block = xfs_allocbt_free_block,
427 .get_minrecs = xfs_allocbt_get_minrecs,
428 .get_maxrecs = xfs_allocbt_get_maxrecs,
429 .init_key_from_rec = xfs_allocbt_init_key_from_rec,
430 .init_high_key_from_rec = xfs_bnobt_init_high_key_from_rec,
431 .init_rec_from_cur = xfs_allocbt_init_rec_from_cur,
432 .init_ptr_from_cur = xfs_allocbt_init_ptr_from_cur,
433 .cmp_key_with_cur = xfs_bnobt_cmp_key_with_cur,
434 .buf_ops = &xfs_bnobt_buf_ops,
435 .cmp_two_keys = xfs_bnobt_cmp_two_keys,
436 .keys_inorder = xfs_bnobt_keys_inorder,
437 .recs_inorder = xfs_bnobt_recs_inorder,
438 .keys_contiguous = xfs_allocbt_keys_contiguous,
439};
440
441const struct xfs_btree_ops xfs_cntbt_ops = {
442 .name = "cnt",
443 .type = XFS_BTREE_TYPE_AG,
444
445 .rec_len = sizeof(xfs_alloc_rec_t),
446 .key_len = sizeof(xfs_alloc_key_t),
447 .ptr_len = XFS_BTREE_SHORT_PTR_LEN,
448
449 .lru_refs = XFS_ALLOC_BTREE_REF,
450 .statoff = XFS_STATS_CALC_INDEX(xs_abtc_2),
451 .sick_mask = XFS_SICK_AG_CNTBT,
452
453 .dup_cursor = xfs_cntbt_dup_cursor,
454 .set_root = xfs_allocbt_set_root,
455 .alloc_block = xfs_allocbt_alloc_block,
456 .free_block = xfs_allocbt_free_block,
457 .get_minrecs = xfs_allocbt_get_minrecs,
458 .get_maxrecs = xfs_allocbt_get_maxrecs,
459 .init_key_from_rec = xfs_allocbt_init_key_from_rec,
460 .init_high_key_from_rec = xfs_cntbt_init_high_key_from_rec,
461 .init_rec_from_cur = xfs_allocbt_init_rec_from_cur,
462 .init_ptr_from_cur = xfs_allocbt_init_ptr_from_cur,
463 .cmp_key_with_cur = xfs_cntbt_cmp_key_with_cur,
464 .buf_ops = &xfs_cntbt_buf_ops,
465 .cmp_two_keys = xfs_cntbt_cmp_two_keys,
466 .keys_inorder = xfs_cntbt_keys_inorder,
467 .recs_inorder = xfs_cntbt_recs_inorder,
468 .keys_contiguous = NULL, /* not needed right now */
469};
470
471/*
472 * Allocate a new bnobt cursor.
473 *
474 * For staging cursors tp and agbp are NULL.
475 */
476struct xfs_btree_cur *
477xfs_bnobt_init_cursor(
478 struct xfs_mount *mp,
479 struct xfs_trans *tp,
480 struct xfs_buf *agbp,
481 struct xfs_perag *pag)
482{
483 struct xfs_btree_cur *cur;
484
485 cur = xfs_btree_alloc_cursor(mp, tp, &xfs_bnobt_ops,
486 mp->m_alloc_maxlevels, xfs_allocbt_cur_cache);
487 cur->bc_group = xfs_group_hold(pag_group(pag));
488 cur->bc_ag.agbp = agbp;
489 if (agbp) {
490 struct xfs_agf *agf = agbp->b_addr;
491
492 cur->bc_nlevels = be32_to_cpu(agf->agf_bno_level);
493 }
494 return cur;
495}
496
497/*
498 * Allocate a new cntbt cursor.
499 *
500 * For staging cursors tp and agbp are NULL.
501 */
502struct xfs_btree_cur *
503xfs_cntbt_init_cursor(
504 struct xfs_mount *mp,
505 struct xfs_trans *tp,
506 struct xfs_buf *agbp,
507 struct xfs_perag *pag)
508{
509 struct xfs_btree_cur *cur;
510
511 cur = xfs_btree_alloc_cursor(mp, tp, &xfs_cntbt_ops,
512 mp->m_alloc_maxlevels, xfs_allocbt_cur_cache);
513 cur->bc_group = xfs_group_hold(pag_group(pag));
514 cur->bc_ag.agbp = agbp;
515 if (agbp) {
516 struct xfs_agf *agf = agbp->b_addr;
517
518 cur->bc_nlevels = be32_to_cpu(agf->agf_cnt_level);
519 }
520 return cur;
521}
522
523/*
524 * Install a new free space btree root. Caller is responsible for invalidating
525 * and freeing the old btree blocks.
526 */
527void
528xfs_allocbt_commit_staged_btree(
529 struct xfs_btree_cur *cur,
530 struct xfs_trans *tp,
531 struct xfs_buf *agbp)
532{
533 struct xfs_agf *agf = agbp->b_addr;
534 struct xbtree_afakeroot *afake = cur->bc_ag.afake;
535
536 ASSERT(cur->bc_flags & XFS_BTREE_STAGING);
537
538 if (xfs_btree_is_bno(cur->bc_ops)) {
539 agf->agf_bno_root = cpu_to_be32(afake->af_root);
540 agf->agf_bno_level = cpu_to_be32(afake->af_levels);
541 } else {
542 agf->agf_cnt_root = cpu_to_be32(afake->af_root);
543 agf->agf_cnt_level = cpu_to_be32(afake->af_levels);
544 }
545 xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS);
546
547 xfs_btree_commit_afakeroot(cur, tp, agbp);
548}
549
550/* Calculate number of records in an alloc btree block. */
551static inline unsigned int
552xfs_allocbt_block_maxrecs(
553 unsigned int blocklen,
554 bool leaf)
555{
556 if (leaf)
557 return blocklen / sizeof(xfs_alloc_rec_t);
558 return blocklen / (sizeof(xfs_alloc_key_t) + sizeof(xfs_alloc_ptr_t));
559}
560
561/*
562 * Calculate number of records in an alloc btree block.
563 */
564unsigned int
565xfs_allocbt_maxrecs(
566 struct xfs_mount *mp,
567 unsigned int blocklen,
568 bool leaf)
569{
570 blocklen -= XFS_ALLOC_BLOCK_LEN(mp);
571 return xfs_allocbt_block_maxrecs(blocklen, leaf);
572}
573
574/* Free space btrees are at their largest when every other block is free. */
575#define XFS_MAX_FREESP_RECORDS ((XFS_MAX_AG_BLOCKS + 1) / 2)
576
577/* Compute the max possible height for free space btrees. */
578unsigned int
579xfs_allocbt_maxlevels_ondisk(void)
580{
581 unsigned int minrecs[2];
582 unsigned int blocklen;
583
584 blocklen = min(XFS_MIN_BLOCKSIZE - XFS_BTREE_SBLOCK_LEN,
585 XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN);
586
587 minrecs[0] = xfs_allocbt_block_maxrecs(blocklen, true) / 2;
588 minrecs[1] = xfs_allocbt_block_maxrecs(blocklen, false) / 2;
589
590 return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_FREESP_RECORDS);
591}
592
593/* Calculate the freespace btree size for some records. */
594xfs_extlen_t
595xfs_allocbt_calc_size(
596 struct xfs_mount *mp,
597 unsigned long long len)
598{
599 return xfs_btree_calc_size(mp->m_alloc_mnr, len);
600}
601
602int __init
603xfs_allocbt_init_cur_cache(void)
604{
605 xfs_allocbt_cur_cache = kmem_cache_create("xfs_bnobt_cur",
606 xfs_btree_cur_sizeof(xfs_allocbt_maxlevels_ondisk()),
607 0, 0, NULL);
608
609 if (!xfs_allocbt_cur_cache)
610 return -ENOMEM;
611 return 0;
612}
613
614void
615xfs_allocbt_destroy_cur_cache(void)
616{
617 kmem_cache_destroy(xfs_allocbt_cur_cache);
618 xfs_allocbt_cur_cache = NULL;
619}