tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

xprtrdma: Replace rpcrdma_mr_seg with xdr_buf cursor

The FRWR registration path converts data through three
representations: xdr_buf -> rpcrdma_mr_seg[] -> scatterlist[]
-> ib_map_mr_sg(). The rpcrdma_mr_seg intermediate is a relic
of when multiple registration strategies existed (FMR, physical,
FRWR). Only FRWR remains, so this indirection and the 6240-byte
rl_segments[260] array embedded in each rpcrdma_req serve no
purpose.

Introduce struct rpcrdma_xdr_cursor to track position within
an xdr_buf during iterative MR registration. Rewrite frwr_map to
populate scatterlist entries directly from the xdr_buf regions
(head kvec, page list, tail kvec). The boundary logic for
non-SG_GAPS devices is simpler because the xdr_buf structure
guarantees that page-region entries after the first start at
offset 0, and that head/tail kvecs are separate regions that
naturally break at MR boundaries.

Fix a pre-existing bug in rpcrdma_encode_write_list where the
write-pad statistics accumulator added mr->mr_length from the last
data MR rather than the write-pad MR. The refactored code uses
ep->re_write_pad_mr->mr_length.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

authored by

Chuck Lever and committed by
Trond Myklebust 7a079ab5 6f2e565f

+193 -157
+14 -14
include/trace/events/rpcrdma.h
··· 392 392 const struct rpc_task *task, 393 393 unsigned int pos, 394 394 struct rpcrdma_mr *mr, 395 - int nsegs 395 + bool is_last 396 396 ), 397 397 398 - TP_ARGS(task, pos, mr, nsegs), 398 + TP_ARGS(task, pos, mr, is_last), 399 399 400 400 TP_STRUCT__entry( 401 401 __field(unsigned int, task_id) ··· 405 405 __field(u32, handle) 406 406 __field(u32, length) 407 407 __field(u64, offset) 408 - __field(int, nsegs) 408 + __field(bool, is_last) 409 409 ), 410 410 411 411 TP_fast_assign( ··· 416 416 __entry->handle = mr->mr_handle; 417 417 __entry->length = mr->mr_length; 418 418 __entry->offset = mr->mr_offset; 419 - __entry->nsegs = nsegs; 419 + __entry->is_last = is_last; 420 420 ), 421 421 422 422 TP_printk(SUNRPC_TRACE_TASK_SPECIFIER ··· 424 424 __entry->task_id, __entry->client_id, 425 425 __entry->pos, __entry->length, 426 426 (unsigned long long)__entry->offset, __entry->handle, 427 - __entry->nents < __entry->nsegs ? "more" : "last" 427 + __entry->is_last ? "last" : "more" 428 428 ) 429 429 ); 430 430 ··· 434 434 const struct rpc_task *task, \ 435 435 unsigned int pos, \ 436 436 struct rpcrdma_mr *mr, \ 437 - int nsegs \ 437 + bool is_last \ 438 438 ), \ 439 - TP_ARGS(task, pos, mr, nsegs)) 439 + TP_ARGS(task, pos, mr, is_last)) 440 440 441 441 DECLARE_EVENT_CLASS(xprtrdma_wrch_event, 442 442 TP_PROTO( 443 443 const struct rpc_task *task, 444 444 struct rpcrdma_mr *mr, 445 - int nsegs 445 + bool is_last 446 446 ), 447 447 448 - TP_ARGS(task, mr, nsegs), 448 + TP_ARGS(task, mr, is_last), 449 449 450 450 TP_STRUCT__entry( 451 451 __field(unsigned int, task_id) ··· 454 454 __field(u32, handle) 455 455 __field(u32, length) 456 456 __field(u64, offset) 457 - __field(int, nsegs) 457 + __field(bool, is_last) 458 458 ), 459 459 460 460 TP_fast_assign( ··· 464 464 __entry->handle = mr->mr_handle; 465 465 __entry->length = mr->mr_length; 466 466 __entry->offset = mr->mr_offset; 467 - __entry->nsegs = nsegs; 467 + __entry->is_last = is_last; 468 468 ), 469 469 470 470 TP_printk(SUNRPC_TRACE_TASK_SPECIFIER ··· 472 472 __entry->task_id, __entry->client_id, 473 473 __entry->length, (unsigned long long)__entry->offset, 474 474 __entry->handle, 475 - __entry->nents < __entry->nsegs ? "more" : "last" 475 + __entry->is_last ? "last" : "more" 476 476 ) 477 477 ); 478 478 ··· 481 481 TP_PROTO( \ 482 482 const struct rpc_task *task, \ 483 483 struct rpcrdma_mr *mr, \ 484 - int nsegs \ 484 + bool is_last \ 485 485 ), \ 486 - TP_ARGS(task, mr, nsegs)) 486 + TP_ARGS(task, mr, is_last)) 487 487 488 488 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL); 489 489 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
+93 -24
net/sunrpc/xprtrdma/frwr_ops.c
··· 268 268 } 269 269 270 270 /** 271 - * frwr_map - Register a memory region 271 + * frwr_map - Register a memory region from an xdr_buf cursor 272 272 * @r_xprt: controlling transport 273 - * @seg: memory region co-ordinates 274 - * @nsegs: number of segments remaining 273 + * @cur: cursor tracking position within the xdr_buf 275 274 * @writing: true when RDMA Write will be used 276 275 * @xid: XID of RPC using the registered memory 277 276 * @mr: MR to fill in ··· 278 279 * Prepare a REG_MR Work Request to register a memory region 279 280 * for remote access via RDMA READ or RDMA WRITE. 280 281 * 281 - * Returns the next segment or a negative errno pointer. 282 - * On success, @mr is filled in. 282 + * Returns 0 on success (cursor advanced past consumed data, 283 + * @mr populated) or a negative errno on failure. 283 284 */ 284 - struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, 285 - struct rpcrdma_mr_seg *seg, 286 - int nsegs, bool writing, __be32 xid, 287 - struct rpcrdma_mr *mr) 285 + int frwr_map(struct rpcrdma_xprt *r_xprt, 286 + struct rpcrdma_xdr_cursor *cur, 287 + bool writing, __be32 xid, 288 + struct rpcrdma_mr *mr) 288 289 { 289 290 struct rpcrdma_ep *ep = r_xprt->rx_ep; 291 + const struct xdr_buf *xdrbuf = cur->xc_buf; 292 + bool sg_gaps = ep->re_mrtype == IB_MR_TYPE_SG_GAPS; 293 + unsigned int max_depth = ep->re_max_fr_depth; 290 294 struct ib_reg_wr *reg_wr; 291 295 int i, n, dma_nents; 292 296 struct ib_mr *ibmr; 293 297 u8 key; 294 298 295 - if (nsegs > ep->re_max_fr_depth) 296 - nsegs = ep->re_max_fr_depth; 297 - for (i = 0; i < nsegs;) { 298 - sg_set_page(&mr->mr_sg[i], seg->mr_page, 299 - seg->mr_len, seg->mr_offset); 299 + i = 0; 300 300 301 - ++seg; 302 - ++i; 303 - if (ep->re_mrtype == IB_MR_TYPE_SG_GAPS) 304 - continue; 305 - if ((i < nsegs && seg->mr_offset) || 306 - offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len)) 307 - break; 301 + /* Head kvec */ 302 + if (!(cur->xc_flags & XC_HEAD_DONE)) { 303 + const struct kvec *head = &xdrbuf->head[0]; 304 + 305 + sg_set_page(&mr->mr_sg[i], 306 + virt_to_page(head->iov_base), 307 + head->iov_len, 308 + offset_in_page(head->iov_base)); 309 + cur->xc_flags |= XC_HEAD_DONE; 310 + i++; 311 + /* Without sg-gap support, each non-contiguous region 312 + * must be registered as a separate MR. Returning 313 + * here after the head kvec causes the caller to 314 + * invoke frwr_map() again for the page list and 315 + * tail. 316 + */ 317 + if (!sg_gaps) 318 + goto finish; 308 319 } 320 + 321 + /* Page list */ 322 + if (!(cur->xc_flags & XC_PAGES_DONE) && xdrbuf->page_len) { 323 + unsigned int page_base, remaining; 324 + struct page **ppages; 325 + 326 + remaining = xdrbuf->page_len - cur->xc_page_offset; 327 + page_base = offset_in_page(xdrbuf->page_base + 328 + cur->xc_page_offset); 329 + ppages = xdrbuf->pages + 330 + ((xdrbuf->page_base + cur->xc_page_offset) 331 + >> PAGE_SHIFT); 332 + 333 + while (remaining > 0 && i < max_depth) { 334 + unsigned int len; 335 + 336 + len = min_t(unsigned int, 337 + PAGE_SIZE - page_base, remaining); 338 + sg_set_page(&mr->mr_sg[i], *ppages, 339 + len, page_base); 340 + cur->xc_page_offset += len; 341 + i++; 342 + ppages++; 343 + remaining -= len; 344 + 345 + if (!sg_gaps && remaining > 0 && 346 + offset_in_page(page_base + len)) 347 + goto finish; 348 + page_base = 0; 349 + } 350 + if (remaining == 0) 351 + cur->xc_flags |= XC_PAGES_DONE; 352 + } else if (!(cur->xc_flags & XC_PAGES_DONE)) { 353 + cur->xc_flags |= XC_PAGES_DONE; 354 + } 355 + 356 + /* Tail kvec */ 357 + if (!(cur->xc_flags & XC_TAIL_DONE) && xdrbuf->tail[0].iov_len && 358 + i < max_depth) { 359 + const struct kvec *tail = &xdrbuf->tail[0]; 360 + 361 + if (!sg_gaps && i > 0) { 362 + struct scatterlist *prev = &mr->mr_sg[i - 1]; 363 + 364 + if (offset_in_page(prev->offset + prev->length) || 365 + offset_in_page(tail->iov_base)) 366 + goto finish; 367 + } 368 + sg_set_page(&mr->mr_sg[i], 369 + virt_to_page(tail->iov_base), 370 + tail->iov_len, 371 + offset_in_page(tail->iov_base)); 372 + cur->xc_flags |= XC_TAIL_DONE; 373 + i++; 374 + } else if (!(cur->xc_flags & XC_TAIL_DONE) && 375 + !xdrbuf->tail[0].iov_len) { 376 + cur->xc_flags |= XC_TAIL_DONE; 377 + } 378 + 379 + finish: 309 380 mr->mr_dir = rpcrdma_data_dir(writing); 310 381 mr->mr_nents = i; 311 382 ··· 407 338 mr->mr_offset = ibmr->iova; 408 339 trace_xprtrdma_mr_map(mr); 409 340 410 - return seg; 341 + return 0; 411 342 412 343 out_dmamap_err: 413 344 trace_xprtrdma_frwr_sgerr(mr, i); 414 - return ERR_PTR(-EIO); 345 + return -EIO; 415 346 416 347 out_mapmr_err: 417 348 trace_xprtrdma_frwr_maperr(mr, n); 418 - return ERR_PTR(-EIO); 349 + return -EIO; 419 350 } 420 351 421 352 /**
+57 -106
net/sunrpc/xprtrdma/rpc_rdma.c
··· 200 200 return 0; 201 201 } 202 202 203 - /* Convert @vec to a single SGL element. 204 - * 205 - * Returns pointer to next available SGE, and bumps the total number 206 - * of SGEs consumed. 207 - */ 208 - static struct rpcrdma_mr_seg * 209 - rpcrdma_convert_kvec(struct kvec *vec, struct rpcrdma_mr_seg *seg, 210 - unsigned int *n) 203 + static void 204 + rpcrdma_xdr_cursor_init(struct rpcrdma_xdr_cursor *cur, 205 + const struct xdr_buf *xdrbuf, 206 + unsigned int pos, enum rpcrdma_chunktype type) 211 207 { 212 - seg->mr_page = virt_to_page(vec->iov_base); 213 - seg->mr_offset = offset_in_page(vec->iov_base); 214 - seg->mr_len = vec->iov_len; 215 - ++seg; 216 - ++(*n); 217 - return seg; 208 + cur->xc_buf = xdrbuf; 209 + cur->xc_page_offset = 0; 210 + cur->xc_flags = 0; 211 + 212 + if (pos != 0) 213 + cur->xc_flags |= XC_HEAD_DONE; 214 + if (!xdrbuf->page_len) 215 + cur->xc_flags |= XC_PAGES_DONE; 216 + if (type == rpcrdma_readch || type == rpcrdma_writech || 217 + !xdrbuf->tail[0].iov_len) 218 + cur->xc_flags |= XC_TAIL_DONE; 218 219 } 219 220 220 - /* Convert @xdrbuf into SGEs no larger than a page each. As they 221 - * are registered, these SGEs are then coalesced into RDMA segments 222 - * when the selected memreg mode supports it. 223 - * 224 - * Returns positive number of SGEs consumed, or a negative errno. 225 - */ 226 - 227 - static int 228 - rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf, 229 - unsigned int pos, enum rpcrdma_chunktype type, 230 - struct rpcrdma_mr_seg *seg) 221 + static bool 222 + rpcrdma_xdr_cursor_done(const struct rpcrdma_xdr_cursor *cur) 231 223 { 232 - unsigned long page_base; 233 - unsigned int len, n; 234 - struct page **ppages; 235 - 236 - n = 0; 237 - if (pos == 0) 238 - seg = rpcrdma_convert_kvec(&xdrbuf->head[0], seg, &n); 239 - 240 - len = xdrbuf->page_len; 241 - ppages = xdrbuf->pages + (xdrbuf->page_base >> PAGE_SHIFT); 242 - page_base = offset_in_page(xdrbuf->page_base); 243 - while (len) { 244 - seg->mr_page = *ppages; 245 - seg->mr_offset = page_base; 246 - seg->mr_len = min_t(u32, PAGE_SIZE - page_base, len); 247 - len -= seg->mr_len; 248 - ++ppages; 249 - ++seg; 250 - ++n; 251 - page_base = 0; 252 - } 253 - 254 - if (type == rpcrdma_readch || type == rpcrdma_writech) 255 - goto out; 256 - 257 - if (xdrbuf->tail[0].iov_len) 258 - rpcrdma_convert_kvec(&xdrbuf->tail[0], seg, &n); 259 - 260 - out: 261 - if (unlikely(n > RPCRDMA_MAX_SEGS)) 262 - return -EIO; 263 - return n; 224 + return (cur->xc_flags & (XC_HEAD_DONE | XC_PAGES_DONE | 225 + XC_TAIL_DONE)) == 226 + (XC_HEAD_DONE | XC_PAGES_DONE | XC_TAIL_DONE); 264 227 } 265 228 266 229 static int ··· 255 292 return 0; 256 293 } 257 294 258 - static struct rpcrdma_mr_seg *rpcrdma_mr_prepare(struct rpcrdma_xprt *r_xprt, 259 - struct rpcrdma_req *req, 260 - struct rpcrdma_mr_seg *seg, 261 - int nsegs, bool writing, 262 - struct rpcrdma_mr **mr) 295 + static int rpcrdma_mr_prepare(struct rpcrdma_xprt *r_xprt, 296 + struct rpcrdma_req *req, 297 + struct rpcrdma_xdr_cursor *cur, 298 + bool writing, struct rpcrdma_mr **mr) 263 299 { 264 300 *mr = rpcrdma_mr_pop(&req->rl_free_mrs); 265 301 if (!*mr) { ··· 269 307 } 270 308 271 309 rpcrdma_mr_push(*mr, &req->rl_registered); 272 - return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr); 310 + return frwr_map(r_xprt, cur, writing, req->rl_slot.rq_xid, *mr); 273 311 274 312 out_getmr_err: 275 313 trace_xprtrdma_nomrs_err(r_xprt, req); 276 314 xprt_wait_for_buffer_space(&r_xprt->rx_xprt); 277 315 rpcrdma_mrs_refresh(r_xprt); 278 - return ERR_PTR(-EAGAIN); 316 + return -EAGAIN; 279 317 } 280 318 281 319 /* Register and XDR encode the Read list. Supports encoding a list of read ··· 298 336 enum rpcrdma_chunktype rtype) 299 337 { 300 338 struct xdr_stream *xdr = &req->rl_stream; 301 - struct rpcrdma_mr_seg *seg; 339 + struct rpcrdma_xdr_cursor cur; 302 340 struct rpcrdma_mr *mr; 303 341 unsigned int pos; 304 - int nsegs; 342 + int ret; 305 343 306 344 if (rtype == rpcrdma_noch_pullup || rtype == rpcrdma_noch_mapped) 307 345 goto done; ··· 309 347 pos = rqst->rq_snd_buf.head[0].iov_len; 310 348 if (rtype == rpcrdma_areadch) 311 349 pos = 0; 312 - seg = req->rl_segments; 313 - nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_snd_buf, pos, 314 - rtype, seg); 315 - if (nsegs < 0) 316 - return nsegs; 350 + rpcrdma_xdr_cursor_init(&cur, &rqst->rq_snd_buf, pos, rtype); 317 351 318 352 do { 319 - seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, false, &mr); 320 - if (IS_ERR(seg)) 321 - return PTR_ERR(seg); 353 + ret = rpcrdma_mr_prepare(r_xprt, req, &cur, false, &mr); 354 + if (ret) 355 + return ret; 322 356 323 357 if (encode_read_segment(xdr, mr, pos) < 0) 324 358 return -EMSGSIZE; 325 359 326 - trace_xprtrdma_chunk_read(rqst->rq_task, pos, mr, nsegs); 360 + trace_xprtrdma_chunk_read(rqst->rq_task, pos, mr, 361 + rpcrdma_xdr_cursor_done(&cur)); 327 362 r_xprt->rx_stats.read_chunk_count++; 328 - nsegs -= mr->mr_nents; 329 - } while (nsegs); 363 + } while (!rpcrdma_xdr_cursor_done(&cur)); 330 364 331 365 done: 332 366 if (xdr_stream_encode_item_absent(xdr) < 0) ··· 352 394 { 353 395 struct xdr_stream *xdr = &req->rl_stream; 354 396 struct rpcrdma_ep *ep = r_xprt->rx_ep; 355 - struct rpcrdma_mr_seg *seg; 397 + struct rpcrdma_xdr_cursor cur; 356 398 struct rpcrdma_mr *mr; 357 - int nsegs, nchunks; 399 + int nchunks, ret; 358 400 __be32 *segcount; 359 401 360 402 if (wtype != rpcrdma_writech) 361 403 goto done; 362 404 363 - seg = req->rl_segments; 364 - nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 365 - rqst->rq_rcv_buf.head[0].iov_len, 366 - wtype, seg); 367 - if (nsegs < 0) 368 - return nsegs; 405 + rpcrdma_xdr_cursor_init(&cur, &rqst->rq_rcv_buf, 406 + rqst->rq_rcv_buf.head[0].iov_len, wtype); 369 407 370 408 if (xdr_stream_encode_item_present(xdr) < 0) 371 409 return -EMSGSIZE; ··· 372 418 373 419 nchunks = 0; 374 420 do { 375 - seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, true, &mr); 376 - if (IS_ERR(seg)) 377 - return PTR_ERR(seg); 421 + ret = rpcrdma_mr_prepare(r_xprt, req, &cur, true, &mr); 422 + if (ret) 423 + return ret; 378 424 379 425 if (encode_rdma_segment(xdr, mr) < 0) 380 426 return -EMSGSIZE; 381 427 382 - trace_xprtrdma_chunk_write(rqst->rq_task, mr, nsegs); 428 + trace_xprtrdma_chunk_write(rqst->rq_task, mr, 429 + rpcrdma_xdr_cursor_done(&cur)); 383 430 r_xprt->rx_stats.write_chunk_count++; 384 431 r_xprt->rx_stats.total_rdma_request += mr->mr_length; 385 432 nchunks++; 386 - nsegs -= mr->mr_nents; 387 - } while (nsegs); 433 + } while (!rpcrdma_xdr_cursor_done(&cur)); 388 434 389 435 if (xdr_pad_size(rqst->rq_rcv_buf.page_len)) { 390 436 if (encode_rdma_segment(xdr, ep->re_write_pad_mr) < 0) 391 437 return -EMSGSIZE; 392 438 393 439 trace_xprtrdma_chunk_wp(rqst->rq_task, ep->re_write_pad_mr, 394 - nsegs); 440 + true); 395 441 r_xprt->rx_stats.write_chunk_count++; 396 - r_xprt->rx_stats.total_rdma_request += mr->mr_length; 442 + r_xprt->rx_stats.total_rdma_request += 443 + ep->re_write_pad_mr->mr_length; 397 444 nchunks++; 398 - nsegs -= mr->mr_nents; 399 445 } 400 446 401 447 /* Update count of segments in this Write chunk */ ··· 425 471 enum rpcrdma_chunktype wtype) 426 472 { 427 473 struct xdr_stream *xdr = &req->rl_stream; 428 - struct rpcrdma_mr_seg *seg; 474 + struct rpcrdma_xdr_cursor cur; 429 475 struct rpcrdma_mr *mr; 430 - int nsegs, nchunks; 476 + int nchunks, ret; 431 477 __be32 *segcount; 432 478 433 479 if (wtype != rpcrdma_replych) { ··· 436 482 return 0; 437 483 } 438 484 439 - seg = req->rl_segments; 440 - nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); 441 - if (nsegs < 0) 442 - return nsegs; 485 + rpcrdma_xdr_cursor_init(&cur, &rqst->rq_rcv_buf, 0, wtype); 443 486 444 487 if (xdr_stream_encode_item_present(xdr) < 0) 445 488 return -EMSGSIZE; ··· 447 496 448 497 nchunks = 0; 449 498 do { 450 - seg = rpcrdma_mr_prepare(r_xprt, req, seg, nsegs, true, &mr); 451 - if (IS_ERR(seg)) 452 - return PTR_ERR(seg); 499 + ret = rpcrdma_mr_prepare(r_xprt, req, &cur, true, &mr); 500 + if (ret) 501 + return ret; 453 502 454 503 if (encode_rdma_segment(xdr, mr) < 0) 455 504 return -EMSGSIZE; 456 505 457 - trace_xprtrdma_chunk_reply(rqst->rq_task, mr, nsegs); 506 + trace_xprtrdma_chunk_reply(rqst->rq_task, mr, 507 + rpcrdma_xdr_cursor_done(&cur)); 458 508 r_xprt->rx_stats.reply_chunk_count++; 459 509 r_xprt->rx_stats.total_rdma_request += mr->mr_length; 460 510 nchunks++; 461 - nsegs -= mr->mr_nents; 462 - } while (nsegs); 511 + } while (!rpcrdma_xdr_cursor_done(&cur)); 463 512 464 513 /* Update count of segments in the Reply chunk */ 465 514 *segcount = cpu_to_be32(nchunks);
+29 -13
net/sunrpc/xprtrdma/xprt_rdma.h
··· 283 283 * registered or invalidated. Must handle a Reply chunk: 284 284 */ 285 285 enum { 286 - RPCRDMA_MAX_IOV_SEGS = 3, 286 + RPCRDMA_MAX_IOV_SEGS = 3, /* head, page-boundary, tail */ 287 287 RPCRDMA_MAX_DATA_SEGS = ((1 * 1024 * 1024) / PAGE_SIZE) + 1, 288 288 RPCRDMA_MAX_SEGS = RPCRDMA_MAX_DATA_SEGS + 289 289 RPCRDMA_MAX_IOV_SEGS, 290 290 }; 291 291 292 - /* Arguments for DMA mapping and registration */ 293 - struct rpcrdma_mr_seg { 294 - u32 mr_len; /* length of segment */ 295 - struct page *mr_page; /* underlying struct page */ 296 - u64 mr_offset; /* IN: page offset, OUT: iova */ 292 + /** 293 + * struct rpcrdma_xdr_cursor - tracks position within an xdr_buf 294 + * for iterative MR registration 295 + * @xc_buf: the xdr_buf being iterated 296 + * @xc_page_offset: byte offset into the page region consumed so far 297 + * @xc_flags: combination of XC_* bits 298 + * 299 + * Each XC_*_DONE flag indicates that this region has no 300 + * remaining MR registration work. That condition holds both when the region 301 + * has already been registered by a prior frwr_map() call and 302 + * when the region is excluded from this chunk type (pre-set 303 + * at init time by rpcrdma_xdr_cursor_init()). frwr_map() 304 + * treats the two cases identically: skip the region. 305 + */ 306 + struct rpcrdma_xdr_cursor { 307 + const struct xdr_buf *xc_buf; 308 + unsigned int xc_page_offset; 309 + unsigned int xc_flags; 297 310 }; 311 + 312 + #define XC_HEAD_DONE BIT(0) 313 + #define XC_PAGES_DONE BIT(1) 314 + #define XC_TAIL_DONE BIT(2) 298 315 299 316 /* The Send SGE array is provisioned to send a maximum size 300 317 * inline request: ··· 347 330 348 331 struct list_head rl_free_mrs; 349 332 struct list_head rl_registered; 350 - struct rpcrdma_mr_seg rl_segments[RPCRDMA_MAX_SEGS]; 351 333 }; 352 334 353 335 static inline struct rpcrdma_req * ··· 466 450 } 467 451 468 452 /* Setting this to 0 ensures interoperability with early servers. 469 - * Setting this to 1 enhances certain unaligned read/write performance. 470 - * Default is 0, see sysctl entry and rpc_rdma.c rpcrdma_convert_iovs() */ 453 + * Setting this to 1 enhances unaligned read/write performance. 454 + * Default is 0, see sysctl entry and rpc_rdma.c */ 471 455 extern int xprt_rdma_pad_optimize; 472 456 473 457 /* This setting controls the hunt for a supported memory ··· 551 535 int frwr_query_device(struct rpcrdma_ep *ep, const struct ib_device *device); 552 536 int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr); 553 537 void frwr_mr_release(struct rpcrdma_mr *mr); 554 - struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt, 555 - struct rpcrdma_mr_seg *seg, 556 - int nsegs, bool writing, __be32 xid, 557 - struct rpcrdma_mr *mr); 538 + int frwr_map(struct rpcrdma_xprt *r_xprt, 539 + struct rpcrdma_xdr_cursor *cur, 540 + bool writing, __be32 xid, 541 + struct rpcrdma_mr *mr); 558 542 int frwr_send(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); 559 543 void frwr_reminv(struct rpcrdma_rep *rep, struct list_head *mrs); 560 544 void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req);