23 #include <linux/version.h> 24 #if LINUX_VERSION_CODE <= KERNEL_VERSION(4,11,0) 25 #include <asm/uaccess.h> 27 #include <asm/atomic.h> 30 #include <linux/mount.h> 31 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) 32 #include <linux/uio.h> 33 #include <linux/aio.h> 36 #define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_M0T1FS 353 .
bt_name =
"io_request_bobtype",
360 .
bt_name =
"pargrp_iomap_bobtype",
367 .
bt_name =
"nw_xfer_request_bobtype",
391 #define INDEX(ivec, i) ((ivec)->iv_index[(i)]) 392 #define COUNT(ivec, i) ((ivec)->iv_vec.v_count[(i)]) 393 #define SEG_NR(ivec) ((ivec)->iv_vec.v_nr) 395 #define V_INDEX(ivec, i) (*(m0_bindex_t*)(m0_varr_ele_get(&(ivec)->iv_index, (i)))) 396 #define V_ADDR(bv, i) (*(void**) (m0_varr_ele_get(&(bv )->iv_index, (i)))) 397 #define V_COUNT(ivec, i) (*(m0_bcount_t*)(m0_varr_ele_get(&(ivec)->iv_count, (i)))) 398 #define V_SEG_NR(ivec) ((ivec)->iv_nr) 400 #define PA(pa, i) (*(enum page_attr*)(m0_varr_ele_get((pa), (i)))) 402 #define indexvec_dump(ivec) \ 405 for (seg = 0; seg < SEG_NR((ivec)); ++seg) { \ 406 M0_LOG(M0_DEBUG, "seg# %d: [pos, +len) = [%llu, +%llu)", \ 407 seg, INDEX((ivec), seg), COUNT((ivec), seg)); \ 411 #define indexvec_varr_dump(ivec) \ 414 for (seg = 0; seg < V_SEG_NR((ivec)); ++seg) { \ 415 M0_LOG(M0_DEBUG, "seg# %d: [pos, +len) = [%llu, +%llu)", \ 416 seg, V_INDEX((ivec), seg), V_COUNT((ivec), seg)); \ 437 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) 438 return file->f_path.dentry->d_inode;
440 return file->f_dentry->d_inode;
464 return M0T1FS_I(
inode);
474 return M0T1FS_SB(
m0inode->ci_inode.i_sb);
494 return size >> PAGE_SHIFT;
540 sum += *(uint64_t*)countp;
562 return index / dtsize;
590 static bool is_pver_dud(uint32_t fdev_nr, uint32_t dev_k, uint32_t fsvc_nr,
595 const uint64_t *
key = (uint64_t *)k;
602 const uint64_t *
k1 = (uint64_t *)key1;
603 const uint64_t *
k2 = (uint64_t *)key2;
658 M0_ENTRY(
"grpid = %llu, target_off = %llu",
map->pi_grpid, toff);
663 M0_LEAVE(
"global file offset = %llu", goff);
688 return offset >> PAGE_SHIFT;
696 #if !defined(round_down) 705 return (
val & (
size - 1)) == 0 ?
710 #if !defined(round_up) 719 return (
val & (
size - 1)) == 0 ?
775 M0_ENTRY(
"gid = %llu, row = %u, col = %u",
map->pi_grpid, row, col);
983 .sd_name =
"IO_initial",
988 .sd_name =
"IO_reading",
992 .sd_name =
"IO_read_complete",
998 .sd_name =
"IO_degraded_read",
1002 .sd_name =
"IO_degraded_write",
1006 .sd_name =
"IO_writing",
1010 .sd_name =
"IO_write_complete",
1016 .sd_name =
"IO_req_failed",
1021 .sd_name =
"IO_req_complete",
1026 .
scf_name =
"IO request state machine configuration",
1060 _0C(io_request_bob_check(
req)) &&
1068 nxr_tioreqs_hash))) &&
1072 nxr_tioreqs_hash))) &&
1092 return _0C(nw_xfer_request_bob_check(xfer)) &&
1114 data_buf_bob_check(db) &&
1125 for (row = 0; row <
rows_nr(play); ++row) {
1126 for (col = 0; col <
layout_n(play); ++col) {
1127 if (
map->pi_databufs[row][col] !=
NULL &&
1133 if (
map->pi_paritybufs !=
NULL) {
1134 for (row = 0; row <
rows_nr(play); ++row) {
1135 for (col = 0; col <
layout_k(play); ++col) {
1136 if (
map->pi_paritybufs[row][col] !=
NULL &&
1151 data_buf_bob_init(
buf);
1161 data_buf_bob_fini(
buf);
1168 _0C(io_req_fop_bob_check(
fop)) &&
1177 _0C(target_ioreq_bob_check(ti)) &&
1188 pargrp_iomap_bob_check(
map) &&
1212 M0_ENTRY(
"nw_xfer_request : %p", xfer);
1216 nw_xfer_request_bob_init(xfer);
1242 nw_xfer_request_bob_fini(xfer);
1253 "user_addr = %lx", user_addr);
1255 "dbuf->db_page = %p", dbuf->
db_page);
1266 down_read(¤t->mm->mmap_sem);
1267 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,0) 1268 rc = get_user_pages(user_addr, 1, FOLL_WRITE,
1271 rc = get_user_pages(current, current->mm, user_addr, 1, 1, 0,
1274 up_read(¤t->mm->mmap_sem);
1276 kmapped = kmap(dbuf->
db_page);
1277 rc = kmapped ==
NULL ? -EFAULT : 0;
1278 if (kmapped !=
NULL)
1289 set_page_dirty(dbuf->
db_page);
1297 struct iov_iter *
it,
1312 M0_ENTRY(
"Copy %s user-space, start = %8llu, end = %8llu",
1318 M0_PRE(
start >> PAGE_SHIFT == (end - 1) >> PAGE_SHIFT);
1322 dbuf =
map->pi_databufs[row][col];
1350 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 1351 bytes = iov_iter_copy_from_user_atomic(page,
it,
1355 bytes = iov_iter_copy_from_user(page,
it,
1361 bytes = end -
start;
1364 "user-space from offset %llu",
map->pi_ioreq,
1367 map->pi_ioreq->ir_copied_nr += bytes;
1376 (end & ~PAGE_MASK) == 0))
1379 if (bytes != end -
start)
1381 -EFAULT,
"[%p] Failed to" 1382 " copy_from_user: %" PRIu64 " !=" 1388 bytes = copy_to_user(
it->iov->iov_base +
it->iov_offset,
1390 (
start & ~PAGE_MASK),
1395 map->pi_ioreq->ir_copied_nr += end -
start - bytes;
1398 "offset %llu",
map->pi_ioreq, end -
start - bytes,
1403 "copy_to_user",
map->pi_ioreq);
1419 struct m0t1fs_sb *
csb;
1421 unsigned long zpage;
1428 if (!(
map->pi_ioreq->ir_type == IRT_READ &&
csb->csb_verify))
1434 zpage = get_zeroed_page(GFP_KERNEL);
1436 if (dbufs ==
NULL || pbufs ==
NULL || zpage == 0) {
1442 for (col = 0; col <
layout_k(play); ++col) {
1443 page = alloc_pages(GFP_KERNEL, 0);
1449 pbufs[col].
b_addr = (
void *)page_address(page);
1453 for (row = 0; row <
rows_nr(play); ++row) {
1455 for (col = 0; col <
layout_n(play); ++col) {
1456 if (
map->pi_databufs[row][col] !=
NULL) {
1458 map->pi_databufs[row][col]->db_buf;
1460 dbufs[col].
b_addr = (
void *)zpage;
1469 for (col = 0; col <
layout_k(play); ++col) {
1470 old_pbuf = &
map->pi_paritybufs[row][col]->db_buf;
1474 "failed for %llu [%u:%u], rc %d",
1475 map->pi_ioreq,
map->pi_grpid, row, col,
1481 "[%u:%u]",
map->pi_ioreq,
map->pi_grpid,
1488 if (pbufs !=
NULL) {
1489 for (col = 0; col <
layout_k(play); ++col) {
1491 free_page((
unsigned long)pbufs[col].
b_addr);
1519 if (dbufs ==
NULL || pbufs ==
NULL) {
1524 if ((
map->pi_ioreq->ir_type == IRT_WRITE &&
map->pi_rtype ==
PIR_NONE)
1527 unsigned long zpage;
1529 zpage = get_zeroed_page(GFP_KERNEL);
1535 for (row = 0; row <
rows_nr(play); ++row) {
1536 for (col = 0; col <
layout_n(play); ++col)
1537 if (
map->pi_databufs[row][col] !=
NULL) {
1538 dbufs[col] =
map->pi_databufs
1541 dbufs[col].
b_addr = (
void *)zpage;
1545 for (col = 0; col <
layout_k(play); ++col)
1546 pbufs[col] =
map->pi_paritybufs[row][col]->
1567 for (row = 0; row <
rows_nr(play); ++row) {
1568 for (col = 0; col <
layout_k(play); ++col)
1569 pbufs[col] =
map->pi_paritybufs[row][col]->
1572 for (col = 0; col <
layout_n(play); ++col) {
1586 if (
map->pi_databufs[row][col] ==
NULL ||
1587 map->pi_databufs[row][col]->
1588 db_auxbuf.b_addr ==
NULL)
1591 dbufs[col] =
map->pi_databufs[row][col]->db_buf;
1592 old[col] =
map->pi_databufs[row][col]->
1596 old, dbufs, pbufs, col);
1617 struct m0t1fs_sb *
csb;
1644 return rc != 0 ?
M0_ERR_INFO(
rc,
"[%p] Parity verification failed for " 1685 M0_PRE(*cursor < req->ir_iomap_nr);
1686 M0_ENTRY(
"[%p] group_id=%llu cursor=%llu",
req, grpid, *cursor);
1696 M0_POST(i < req->ir_iomap_nr);
1716 M0_ENTRY(
"[%p] %s user-space. filter = 0x%x",
1722 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 1746 count = pgend - pgstart;
1757 rc,
"[%p] Copy failed (pgstart=%" PRIu64 1759 req, pgstart, pgend);
1785 for (j =
i+1; j <
V_SEG_NR(ivec); ++j) {
1803 struct m0t1fs_sb *
csb;
1809 pargrp_iomap_bob_init(
map);
1812 map->pi_grpid = grpid;
1835 for (row = 0; row <
rows_nr(play); ++row) {
1837 if (
map->pi_databufs[row] ==
NULL)
1844 if (
map->pi_paritybufs ==
NULL)
1847 for (row = 0; row <
rows_nr(play); ++row) {
1850 if (
map->pi_paritybufs[row] ==
NULL)
1856 "databufs=[%u x %u] paritybufs=[%u x %u]",
1867 if (
map->pi_databufs !=
NULL) {
1868 for (row = 0; row <
rows_nr(play); ++row)
1872 if (
map->pi_paritybufs !=
NULL) {
1873 for (row = 0; row <
rows_nr(play); ++row)
1878 return M0_ERR_INFO(-ENOMEM,
"[%p] Memory allocation failed",
req);
1895 pargrp_iomap_bob_fini(
map);
1898 for (row = 0; row <
rows_nr(play); ++row) {
1899 for (col = 0; col <
layout_n(play); ++col) {
1900 if (
map->pi_databufs[row][col] !=
NULL) {
1902 pi_databufs[row][col]);
1903 map->pi_databufs[row][col] =
NULL;
1909 if (
map->pi_paritybufs !=
NULL) {
1910 for (row = 0; row <
rows_nr(play); ++row) {
1911 for (col = 0; col <
layout_k(play); ++col) {
1912 if (
map->pi_paritybufs[row][col] !=
NULL) {
1914 pi_paritybufs[row][col]);
1915 map->pi_paritybufs[row][col] =
NULL;
1935 bool spanned =
false;
1948 return M0_RC(!!spanned);
1958 M0_ENTRY(
"[%p] map %p, row %u col %u",
map->pi_ioreq,
map, row, col);
1961 return map->pi_databufs[row][col] ==
NULL ?
M0_ERR(-ENOMEM) : 0;
1987 rmw ?
"rmw" :
"aligned");
2013 (end < inode->i_size ||
2014 (
inode->i_size > 0 &&
2028 "%3llu row %u col %u f 0x%x addr %p",
2030 map->pi_databufs[row][col] !=
NULL ?
2031 map->pi_databufs[row][col]->db_buf.b_addr :
NULL);
2034 map->pi_databufs[row][col]->db_flags =
flags;
2039 for (row = 0; row <
rows_nr(play); ++row) {
2040 for (col = 0; col <
layout_n(play); ++col) {
2041 if (
map->pi_databufs[row][col] !=
NULL) {
2044 map->pi_databufs[row][col] =
NULL;
2063 for (row = 0; row <
rows_nr(play); ++row) {
2064 for (col = 0; col <
layout_n(play); ++col) {
2066 if (
map->pi_databufs[row][col] &&
2067 map->pi_databufs[row][col]->db_flags &
2084 map->pi_databufs[row][col]->db_auxbuf.b_addr = (
void *)
2085 get_zeroed_page(GFP_KERNEL);
2087 if (
map->pi_databufs[row][col]->db_auxbuf.b_addr ==
NULL)
2090 map->pi_databufs[row][col]->db_auxbuf.b_nob =
PAGE_SIZE;
2124 if (
map->pi_databufs[row][col] !=
NULL) {
2134 if (end < inode->i_size ||
2135 (
inode->i_size > 0 &&
2137 map->pi_databufs[row][col]->db_flags |=
2143 "failed",
map->pi_ioreq);
2225 ivec = &
map->pi_ivv;
2259 if (
map->pi_databufs[row][col] ==
NULL) {
2263 "failed",
map->pi_ioreq);
2265 if (end <= inode->i_size || (
inode->i_size > 0 &&
2267 map->pi_databufs[row][col]->db_flags |=
2281 struct m0t1fs_sb *
csb;
2291 for (row = 0; row <
rows_nr(play); ++row) {
2292 for (col = 0; col <
layout_k(play); ++col) {
2296 if (
map->pi_paritybufs[row][col] ==
NULL)
2298 dbuf =
map->pi_paritybufs[row][col];
2299 if (
map->pi_ioreq->ir_type == IRT_WRITE)
2302 irf =
map->pi_ioreq->ir_file;
2304 (
map->pi_ioreq->ir_type == IRT_READ &&
2305 csb->csb_verify)) &&
2313 for (row = 0; row <
rows_nr(play); ++row) {
2314 for (col = 0; col <
layout_k(play); ++col)
2317 return M0_ERR_INFO(-ENOMEM,
"[%p] Memory allocation failed for " 2318 "data_buf.",
map->pi_ioreq);
2382 grpstart = grpsize *
map->pi_grpid;
2383 grpend = grpstart + grpsize;
2392 if (
map->pi_ops->pi_spans_seg(
map,
2400 if (
map->pi_ioreq->ir_type == IRT_READ &&
2401 grpend >
inode->i_size) {
2488 uint64_t rr_pages_nr = data_pages_nr -
2489 map->pi_ops->pi_fullpages_find(
map);
2491 if (rr_pages_nr < ro_pages_nr) {
2494 rc =
map->pi_ops->pi_readrest(
map);
2500 rc =
map->pi_ops->pi_readold_auxbuf_alloc(
map);
2517 struct m0t1fs_sb *
csb;
2526 grpstart = grpsize *
map->pi_grpid;
2527 grpend = grpstart + grpsize;
2537 if (
map->pi_ioreq->ir_type == IRT_WRITE && grpstart < inode->i_size &&
2543 rmw ?
"rmw" :
"aligned");
2546 if (
map->pi_ioreq->ir_type == IRT_READ &&
csb->csb_verify) {
2555 rc =
map->pi_ops->pi_seg_process(
map, 0, rmw);
2571 if (
map->pi_ioreq->ir_type == IRT_WRITE ||
2572 (
map->pi_ioreq->ir_type == IRT_READ &&
csb->csb_verify))
2573 rc =
map->pi_ops->pi_paritybufs_alloc(
map);
2603 bufs =
map->pi_databufs;
2607 bufs =
map->pi_paritybufs;
2615 for (col = 0; col < col_nr; ++col) {
2616 for (row = 0; row < row_nr; ++row) {
2623 if (bufs[row][col] !=
NULL &&
2631 for (row = 0; row < row_nr; ++row) {
2632 if (bufs[row][col] ==
NULL) {
2634 if (bufs[row][col] ==
NULL) {
2669 uint32_t *spare_slot, uint32_t *spare_slot_prev,
2689 spare_slot, spare_slot_prev);
2695 spare.
sa_unit = *spare_slot_prev;
2707 uint32_t spare_slot;
2708 uint32_t spare_prev;
2711 M0_ENTRY(
"[%p] pid=%llu, row = %u, col=%u, type=0x%x",
2712 map->pi_ioreq,
map->pi_grpid, row, col, page_type);
2716 map->pi_paritybufs[row][col] !=
NULL));
2744 map->pi_databufs[row][col]->db_flags |=
2747 map->pi_paritybufs[row][col]->db_flags |=
2771 uint32_t spare_slot;
2772 uint32_t spare_slot_prev;
2777 M0_ENTRY(
"[%p] grpid = %llu, count = %u\n",
2798 map->pi_ioreq, spare_slot,
2799 spare_slot_prev, dev_state);
2827 return M0_ERR_INFO(
rc,
"[%p] Failed to mark pages from parity " 2834 if (
map->pi_paritybufs ==
NULL) {
2836 if (
map->pi_paritybufs ==
NULL)
2837 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate " 2838 "parity buffers",
req);
2840 for (row = 0; row <
rows_nr(play); ++row) {
2843 if (
map->pi_paritybufs[row] ==
NULL) {
2854 for (row = 0; row <
rows_nr(play); ++row)
2871 struct m0t1fs_sb *
csb;
2886 M0_ENTRY(
"[%p] parity group id %llu, map state = %d",
2898 for (col = 0; col <
layout_n(play); ++col) {
2899 for (row = 0; row <
rows_nr(play); ++row) {
2903 (
inode->i_size > 0 &&
2906 if (
map->pi_databufs[row][col] !=
NULL) {
2907 if (
map->pi_databufs[row][col]->db_flags &
2919 map->pi_databufs[row][col] =
2921 if (
map->pi_databufs[row][col] ==
2932 dbuf =
map->pi_databufs[row][col];
2982 for (row = 0; row <
rows_nr(play); ++row) {
2983 for (col = 0; col <
layout_k(play); ++col) {
2985 if (
map->pi_paritybufs[row][col] ==
NULL) {
2986 map->pi_paritybufs[row][col] =
2988 if (
map->pi_paritybufs[row][col] ==
NULL) {
2993 dbuf =
map->pi_paritybufs[row][col];
3008 rc == -ENOMEM ?
"Failed to allocate " 3009 "data buffer":
"Illegal device queried for status");
3020 for (col = 0; col <
layout_n(play); ++col) {
3021 if (
map->pi_databufs[0][col] !=
NULL &&
3022 map->pi_databufs[0][col]->db_flags &
3029 for (col = 0; col <
layout_k(play); ++col) {
3031 if (
map->pi_paritybufs[0][col]->db_flags &
3046 unsigned long zpage;
3060 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate memory" 3061 " for data buf",
map->pi_ioreq);
3066 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate memory" 3067 " for parity buf",
map->pi_ioreq);
3070 zpage = get_zeroed_page(GFP_KERNEL);
3074 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate page.",
3084 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate memory " 3085 "for m0_buf",
map->pi_ioreq);
3090 (
int)
map->pi_grpid);
3096 for (row = 0; row <
rows_nr(play); ++row) {
3097 for (col = 0; col <
layout_n(play); ++col) {
3099 if (
map->pi_databufs[row][col] ==
NULL) {
3100 data[col].b_addr = (
void *)zpage;
3103 data[col].b_addr =
map->pi_databufs[row][col]->
3106 for (col = 0; col <
layout_k(play); ++col) {
3108 parity[col].b_addr =
map->pi_paritybufs[row][col]->
3124 "in parity group exceeds the" 3125 "total number of parity units" 3126 "in a parity group %d.",
3127 (
int)
map->pi_grpid);
3137 uint64_t grparray_sz;
3149 if (grparray ==
NULL)
3150 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate memory" 3151 " for int array",
req);
3161 for (
grp = grpstart;
grp <= grpend; ++
grp) {
3170 if (grparray[
i] ==
grp)
3176 "[%p] nr=%llu size=%llu",
3177 req,
i , grparray_sz);
3203 M0_LOG(
M0_DEBUG,
"[%p] spanned_groups=%llu [N,K,us]=[%d,%d,%llu]",
3241 ir_iomaps[
map], &cursor);
3304 goto failed_free_dg;
3308 goto failed_free_iv;
3313 goto failed_free_bv;
3331 return M0_ERR_INFO(
rc,
"[%p] Dgmode read vector allocation failed",
3363 for (row = row_start; row <= row_end; ++row) {
3383 for (; row <
rows_nr(play); ++row) {
3418 struct m0t1fs_sb *
csb;
3420 M0_ENTRY(
"nw_xfer_request %p", xfer);
3466 "nxo_tioreq_map() failed, rc=%d",
3486 for (unit = 0; unit <
layout_k(play); ++unit) {
3497 "nxo_tioreq_map() failed, rc=%d",
3522 "nxo_tioreq_map() failed, rc=%d",
3552 M0_ENTRY(
"[%p] Waiting for %s -> %s, Pending fops %llu, " 3553 "Pending rdbulk %llu",
req,
3606 return pver->pv_fd_tol_vec[lv];
3619 uint64_t max_failures;
3620 uint64_t session_id;
3624 for (
i = 0;
i < max_failures; ++
i) {
3644 uint32_t fdev_nr = 0;
3645 uint32_t fsvc_nr = 0;
3649 uint64_t max_failures;
3663 "device state",
req);
3665 if (ti->
ti_rc == -ECANCELED) {
3677 M0_LOG(
M0_DEBUG,
"failed devices = %d\ttolerance=%d", (
int)fdev_nr,
3680 return M0_ERR_INFO(-EIO,
"[%p] Failed to recover data " 3681 "since number of failed data units " 3682 "(%lu) exceeds number of parity " 3683 "units in parity group (%lu) OR " 3684 "number of failed services (%lu) " 3685 "exceeds number of max failures " 3687 req, (
unsigned long)fdev_nr,
3689 (
unsigned long)fsvc_nr,
3690 (
unsigned long)max_failures);
3691 return M0_RC(fdev_nr);
3699 static bool is_pver_dud(uint32_t fdev_nr, uint32_t dev_k, uint32_t fsvc_nr,
3702 if (fdev_nr > 0 && dev_k == 0)
3704 if (fsvc_nr > 0 && svc_k == 0)
3706 return (svc_k + fsvc_nr > 0) ?
3707 (fdev_nr * svc_k + fsvc_nr * dev_k) > dev_k * svc_k :
3715 struct m0t1fs_sb *
csb;
3724 if (
csb->csb_oostore || M0_IN(xfer->
nxr_rc, (0, -E2BIG, -ESTALE)))
3789 return M0_ERR_INFO(
rc,
"[%p] Failed to redistribute file data " 3790 "between target_ioreq objects",
req);
3797 return M0_ERR_INFO(
rc,
"[%p] Failed to dispatch degraded mode" 3798 "write IO fops",
req);
3802 return M0_ERR_INFO(
rc,
"[%p] Degraded mode write IO failed",
3817 struct m0t1fs_sb *
csb;
3832 if (M0_IN(xfer->
nxr_rc, (0, -ENOENT, -ESTALE)) ||
3943 return M0_ERR_INFO(
rc,
"[%p] Failed to dispatch degraded mode " 3948 return M0_ERR_INFO(
rc,
"[%p] Degraded mode read IO failed.",
3953 "[%p] Degraded mode read IO failed.",
req);
3973 struct m0t1fs_inode *mi;
4026 struct m0t1fs_sb *
csb;
4067 "failed: rc=%d",
req,
rc);
4073 "failed: rc=%d",
req,
rc);
4106 "failed: rc=%d",
req,
rc);
4112 "failed: rc=%d",
req,
rc);
4119 "failed: rc=%d",
req,
rc);
4131 "failed: rc=%d",
req,
rc);
4147 if (read_pages > 0) {
4158 if (read_pages > 0) {
4162 "failed: rc=%d",
req,
rc);
4173 "failed: rc=%d",
req,
rc);
4211 if (read_pages > 0) {
4263 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) 4301 const struct iovec *iov,
4303 enum io_req_type rw)
4305 struct m0t1fs_inode *
ci;
4306 struct m0t1fs_sb *
csb;
4312 uint32_t max_failures;
4320 M0_PRE(M0_IN(rw, (IRT_READ, IRT_WRITE)));
4349 li =
ci->ci_layout_instance;
4359 io_request_bob_init(
req);
4363 "[%p] nw_xfer_req_init() failed",
req);
4370 "failed sessions.",
req);
4373 for (
i = 0;
i < max_failures; ++
i) {
4408 struct m0t1fs_sb *
csb;
4420 io_request_bob_fini(
req);
4535 M0_ENTRY(
"nw_xfer_request %p", xfer);
4573 uint32_t spare_slot;
4574 uint32_t spare_slot_prev;
4580 &spare_slot, &spare_slot_prev);
4591 spare.
sa_unit = spare_slot_prev;
4606 dev_state = dev_state_prev;
4626 (*tio)->ti_state = dev_state;
4633 const struct m0_fid *cobfid,
4649 ti, xfer,
FID_P(cobfid));
4673 tioreqht_tlink_init(ti);
4674 target_ioreq_bob_init(ti);
4704 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate memory in " 4705 "target_ioreq_init",
req);
4713 target_ioreq_bob_fini(ti);
4714 tioreqht_tlink_fini(ti);
4743 M0_ENTRY(
"nw_xfer_request %p, fid %p", xfer,
fid);
4778 return M0_ERR_INFO(-ENOMEM,
"[%p] Failed to allocate " 4779 "memory for target_ioreq",
req);
4807 addr = get_zeroed_page(GFP_KERNEL);
4832 free_page((
unsigned long)
buf->b_addr);
4845 else if (
buf->db_buf.b_addr !=
NULL)
4848 if (
buf->db_auxbuf.b_addr !=
NULL)
4880 M0_ENTRY(
"tio req %p, gob_offset %llu, count %llu frame %llu unit %llu",
4897 "[gpos %6llu, +%llu][%llu,%llu]->[%llu,%llu] %c",
4924 while (pgstart < toff +
count) {
4936 buf =
map->pi_databufs[row][col];
4948 buf->db_tioreq = ti;
4953 "[%llu, %llu] added to target_ioreq with "FID_F 4964 "[%p] ti %p, v_nr=%u, page_nr=%llu",
4978 M0_ENTRY(
"io_req_fop %p, target_ioreq %p",
fop, ti);
4983 io_req_fop_bob_init(
fop);
4984 iofops_tlink_init(
fop);
4985 fop->irf_pattr = pattr;
4986 fop->irf_tioreq = ti;
4987 fop->irf_reply_rc = 0;
5010 "rbulk %p",
req, &
fop->irf_iofop.if_fop,
5014 &
fop->irf_iofop.if_rbulk);
5030 iofops_tlink_fini(
fop);
5048 M0_ENTRY(
"io_req_fop %p, rbulk %p, fop %p, %p[%u]", irfop,
5073 const struct iovec *iov,
5075 enum io_req_type rw)
5080 struct m0t1fs_sb *
csb;
5083 M0_ENTRY(
"indexvec %p, rw %d", ivv, rw);
5087 M0_PRE(M0_IN(rw, (IRT_READ, IRT_WRITE)));
5093 return M0_ERR_INFO(-ENOMEM,
"Failed to allocate memory" 5114 "between target_ioreq objects, rc %d",
req,
rc);
5125 M0_LOG(
M0_INFO,
"[%p] nxr_bytes = %llu, copied_nr = %llu, count %lu, " 5146 const struct iovec *iov,
5178 pos += iov[
i].iov_len;
5186 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5197 written = __generic_file_write_iter(kcb, from);
5198 inode_unlock(
inode);
5201 written = generic_write_sync(kcb, written);
5208 const struct iovec *iov,
5218 BUG_ON(kcb->ki_pos != pos);
5220 mutex_lock(&
inode->i_mutex);
5221 written = __generic_file_aio_write(kcb, iov,
seg_nr, &kcb->ki_pos);
5222 mutex_unlock(&
inode->i_mutex);
5225 written = generic_write_sync(
file, pos, written) ?: written;
5233 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5234 static ssize_t
aio_write(
struct kiocb *kcb,
struct iov_iter *from)
5243 M0_ENTRY(
"struct iovec %p position %llu seg_nr %lu", from->iov, kcb->ki_pos, from->nr_segs);
5250 count = generic_write_checks(kcb, from);
5256 if (kcb->ki_filp->f_flags & O_DIRECT) {
5270 written =
m0t1fs_aio(kcb, from->iov, ivv, IRT_WRITE);
5274 kcb->ki_pos = kcb->ki_pos + written;
5283 static ssize_t
aio_write(
struct kiocb *kcb,
const struct iovec *iov,
5284 unsigned long seg_nr, loff_t pos)
5293 M0_ENTRY(
"struct iovec %p position %llu seg_nr %lu", iov, pos,
seg_nr);
5303 rc = generic_segment_checks(iov, &
seg_nr, &
count, VERIFY_READ);
5309 saved_count =
count;
5310 rc = generic_write_checks(kcb->ki_filp, &pos, &
count, 0);
5316 if (
count != saved_count)
5319 if (kcb->ki_filp->f_flags & O_DIRECT) {
5333 written =
m0t1fs_aio(kcb, iov, ivv, IRT_WRITE);
5337 kcb->ki_pos = pos + written;
5347 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5351 const struct iovec *iov,
5362 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5374 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5375 static ssize_t
aio_read(
struct kiocb *kcb,
struct iov_iter *from)
5387 M0_ENTRY(
"struct iovec %p position %llu", from->iov, kcb->ki_pos);
5389 filp = kcb->ki_filp;
5395 if (kcb->ki_pos >=
size)
5398 if (filp->f_flags & O_DIRECT) {
5399 res = generic_file_read_iter(kcb, from);
5404 count = iov_iter_count(from);
5440 M0_LOG(
M0_DEBUG,
"Read @%llu vec-count = %8llu return = %8llu(%d)",
5442 (
unsigned long long)
res, (
int)
res);
5445 kcb->ki_pos = kcb->ki_pos +
res;
5453 static ssize_t
aio_read(
struct kiocb *kcb,
const struct iovec *iov,
5454 unsigned long seg_nr, loff_t pos)
5464 M0_ENTRY(
"struct iovec %p position %llu", iov, pos);
5469 filp = kcb->ki_filp;
5478 if (filp->f_flags & O_DIRECT) {
5479 res = generic_file_aio_read(kcb, iov,
seg_nr, pos);
5488 res = generic_segment_checks(iov, &
seg_nr, &
count, VERIFY_WRITE);
5529 M0_LOG(
M0_DEBUG,
"Read @%llu vec-count = %8llu return = %8llu(%d)",
5531 (
unsigned long long)
res, (
int)
res);
5534 kcb->ki_pos = pos +
res;
5543 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5547 const struct iovec *iov,
5558 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5573 struct m0t1fs_inode *
ci = M0T1FS_I(
inode);
5574 struct m0t1fs_mdop
mo;
5579 M0_ENTRY(
"inode links:%d inode writecount = %d close size %d",
5580 (
unsigned int)
inode->i_nlink,
5581 atomic_read(&
inode->i_writecount),
5582 (
unsigned int)
inode->i_size);
5584 if (!
csb->csb_oostore ||
inode->i_nlink == 0 ||
5585 atomic_read(&
inode->i_writecount) == 0)
5590 mo.mo_attr.ca_size =
inode->i_size;
5591 mo.mo_attr.ca_nlink =
inode->i_nlink;
5593 mo.mo_attr.ca_lid =
ci->ci_layout_id;
5603 .llseek = generic_file_llseek,
5604 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) 5610 .read = do_sync_read,
5611 .write = do_sync_write,
5613 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,10,0) 5630 uint32_t req_sm_state;
5639 rbulk =
buf->bb_rbulk;
5645 M0_LOG(
M0_DEBUG,
"[%p] PASSIVE recv, e %p, status %d, len %llu, " 5650 "fop %p, item %p, "FID_F", rbulk %p",
5676 (M0_IN(req_sm_state,
5728 M0_LOG(
M0_DEBUG,
"[%p] IO fop %p, %p[%u], rbulk %p, submitted to rpc, " 5771 M0_LOG(
M0_DEBUG,
"[%p] fop %p, Pending fops %llu, Pending rdbulk %llu",
5790 size_t non_queued_buf_nr;
5794 buf_nr = rpcbulk_tlist_length(&rbulk->
rb_buflist);
5802 (
unsigned long long)buf_nr,
5803 (
unsigned long long)non_queued_buf_nr);
5815 if (buf_nr > non_queued_buf_nr) {
5832 M0_LOG(
M0_DEBUG,
"[%p] fop %p, Pending fops %llu, Pending rdbulk %llu",
5845 io_req_fop_bob_fini(reqfop);
5884 struct m0t1fs_inode *
inode;
5885 struct m0t1fs_sb *
csb;
5898 if (reply_item !=
NULL) {
5902 if (
rc < 0 || reply_item ==
NULL) {
5912 rc =
rc ? M0_IN(
reply->cor_rc, (0, -EEXIST)) ? 0 :
reply->cor_rc : 0;
5919 csb = M0T1FS_SB(
inode->ci_inode.i_sb);
5925 if (ti->
ti_rc == 0 &&
rc != 0)
5941 struct m0t1fs_sb *
csb;
5942 struct m0t1fs_inode *
inode;
5946 csb = M0T1FS_SB(
inode->ci_inode.i_sb);
6037 struct m0t1fs_inode *
inode;
6038 struct m0t1fs_sb *
csb;
6040 uint64_t actual_bytes = 0;
6065 "req item %p[%u], ri_error %d",
req,
6071 if (reply_item !=
NULL) {
6074 if (
rc < 0 || reply_item ==
NULL) {
6088 "sns state = %d",
req, req_item,
6096 csb = M0T1FS_SB(
inode->ci_inode.i_sb);
6113 if (tioreq->
ti_rc == 0)
6117 if (
rc == -ESTALE || (xfer->
nxr_rc == 0 &&
rc != 0)) {
6120 "nwxfer rc = %d @"FID_F,
6130 M0_LOG(
M0_INFO,
"[%p] fop %p, Returned no of bytes = %llu, " 6131 "expected = %llu",
req, &iofop->
if_fop, actual_bytes,
6148 "Pending rdbulk %llu",
req, req_item,
6161 struct m0t1fs_sb *
csb;
6162 uint64_t nr_dispatched = 0;
6205 ", fops length of ti %u, total fops nr %llu",
req,
6207 (
int)iofops_tlist_length(&ti->
ti_iofops),
6224 FID_F"@%p, item %p, fops nr=%llu, rc=%d, " 6236 else if (post_error == 0)
6237 post_error = ri_error;
6243 if (
rc == 0 && nr_dispatched == 0 && post_error == 0) {
6258 "nr_dispatched %llu",
req,
6261 (
unsigned long long)nr_dispatched);
6273 struct m0t1fs_inode *
inode;
6274 struct m0t1fs_sb *
csb;
6276 M0_ENTRY(
"nw_xfer_request %p, rmw %s", xfer,
6277 rmw ? (
char *)
"true" : (
char *)
"false");
6283 csb = M0T1FS_SB(
inode->ci_inode.i_sb);
6289 rmw ? (
char *)
"true" : (
char *)
"false");
6311 "item %p[%u], ri_error %d, ri_state %d",
6347 "nxr_rdbulk_nr %llu",
req, xfer->
nxr_rc,
6422 rc =
map->pi_ops->pi_dgmode_process(
map,
6427 "Parity group dgmode process failed",
6480 M0_ENTRY(
"io_req_fop %p net_domain %p delta_size %d",
6481 irfop,
dom, *delta);
6503 "rpc_bulk_buffer",
req);
6597 M0_ENTRY(
"[%p] prepare io fops for target ioreq %p filter 0x%x, tfid " 6601 if (
rc != 0 &&
rc != -ECANCELED)
6636 "filter = 0x%x, rw = 0x%x",
6642 if (irfop ==
NULL) {
6672 "filter=0x%x, rw=0x%x",
req,
seg,
6689 if (
rc == -EMSGSIZE) {
6704 &rbuf, &delta, maxsize);
6747 iofops_tlist_add(&ti->
ti_iofops, irfop);
6750 " pending io fops = %llu, pending read bulks = %llu " 6757 (
int)iofops_tlist_length(&ti->
ti_iofops));
6773 .getattr = m0t1fs_getattr,
6774 #if LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0) 6783 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,7,0) 6785 struct iov_iter *from)
6795 rw = iov_iter_rw(from);
6797 "addr=%p len=%lu", rw == READ ?
"READ" :
"WRITE",
6798 (
long long)kcb->ki_pos, from->nr_segs, from->iov->iov_base,
6799 from->iov->iov_len);
6801 M0_PRE(M0_IN(rw, (READ, WRITE)));
6817 retval =
m0t1fs_aio(kcb, from->iov, ivv, rw == READ ? IRT_READ : IRT_WRITE);
6838 const struct iovec *iov,
6850 "addr=%p len=%lu", rw == READ ?
"READ" :
"WRITE",
6851 (
long long)pos,
seg_nr, iov->iov_base, iov->iov_len);
6853 M0_PRE(M0_IN(rw, (READ, WRITE)));
6869 retval =
m0t1fs_aio(kcb, iov, ivv, rw == READ ? IRT_READ : IRT_WRITE);
6891 #undef M0_TRACE_SUBSYSTEM
M0_INTERNAL long m0t1fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
ssize_t m0t1fs_listxattr(struct dentry *dentry, char *buffer, size_t size)
static void m0_atomic64_inc(struct m0_atomic64 *a)
M0_INTERNAL int m0_rpc_post(struct m0_rpc_item *item)
static int io_request_init(struct io_request *req, struct file *file, const struct iovec *iov, struct m0_indexvec_varr *ivv, enum io_req_type rw)
static ssize_t aio_read(struct kiocb *kcb, struct iov_iter *from)
static int ioreq_file_lock(struct io_request *req)
static int user_data_copy(struct pargrp_iomap *map, m0_bindex_t start, m0_bindex_t end, struct iov_iter *it, enum copy_direction dir, enum page_attr filter)
uint32_t m0_fop_opcode(const struct m0_fop *fop)
static uint32_t iomap_dgmode_recov_prepare(struct pargrp_iomap *map, uint8_t *failed)
M0_INTERNAL void m0_chan_wait(struct m0_clink *link)
enum sns_repair_state ir_sns_state
M0_INTERNAL bool m0_ivec_varr_cursor_move_to(struct m0_ivec_varr_cursor *cur, m0_bindex_t dest)
m0_time_t ri_resend_interval
static int ioreq_dgmode_recover(struct io_request *req)
#define M0_ALLOC_ARR(arr, nr)
M0_TL_DECLARE(rpcbulk, M0_INTERNAL, struct m0_rpc_bulk_buf)
M0_INTERNAL void m0_sm_fail(struct m0_sm *mach, int fail_state, int32_t rc)
static const struct io_request_ops ioreq_oostore_ops
static struct m0_fid target_fid(const struct io_request *req, struct m0_pdclust_tgt_addr *tgt)
M0_INTERNAL m0_bcount_t m0_ext_length(const struct m0_ext *ext)
M0_INTERNAL m0_bcount_t m0_io_fop_byte_count(struct m0_io_fop *iofop)
static m0_bindex_t seg_set(struct pargrp_iomap *map, uint32_t seg, struct m0_ivec_varr_cursor *cur, m0_bindex_t grpend)
M0_INTERNAL void m0_mutex_unlock(struct m0_mutex *mutex)
static bool target_ioreq_invariant(struct target_ioreq *ti)
static const struct io_request_ops ioreq_ops
M0_INTERNAL struct m0t1fs_sb * m0_fop_to_sb(struct m0_fop *fop)
M0_INTERNAL int m0t1fs_inode_layout_init(struct m0t1fs_inode *ci)
int m0t1fs_removexattr(struct dentry *dentry, const char *name)
M0_INTERNAL struct m0_fop_cob_common * m0_cobfop_common_get(struct m0_fop *fop)
#define m0_htable_for(name, var, htable)
static void parity_page_pos_get(struct pargrp_iomap *map, m0_bindex_t index, uint32_t *row, uint32_t *col)
int const char const void size_t int flags
static m0_bcount_t seg_endpos(const struct m0_indexvec *ivec, uint32_t i)
M0_INTERNAL void m0_clink_init(struct m0_clink *link, m0_chan_cb_t cb)
int(* iro_parity_recalc)(struct io_request *req)
static int dgmode_rwvec_alloc_init(struct target_ioreq *ti)
struct m0_atomic64 nxr_rdbulk_nr
struct m0_indexvec_varr pi_ivv
M0_INTERNAL int m0_rpc_bulk_store(struct m0_rpc_bulk *rbulk, const struct m0_rpc_conn *conn, struct m0_net_buf_desc_data *to_desc, const struct m0_net_buffer_callbacks *bulk_cb)
M0_INTERNAL void m0_clink_del_lock(struct m0_clink *link)
struct m0_pool_version * l_pver
static void io_req_fop_fini(struct io_req_fop *fop)
static void pargrp_iomap_fini(struct pargrp_iomap *map)
static struct buffer * cur(struct m0_addb2_mach *mach, m0_bcount_t space)
struct m0_bufvec nb_buffer
int(* iro_dgmode_write)(struct io_request *req, bool rmw)
static int ioreq_iomaps_prepare(struct io_request *req)
void(* sa_cb)(struct m0_sm_group *grp, struct m0_sm_ast *)
static bool is_page_read(struct data_buf *dbuf)
static int target_cob_create_fop_prepare(struct target_ioreq *ti)
#define M0_MEMBER_SIZE(type, member)
const m0_time_t M0_TIME_NEVER
M0_INTERNAL struct m0_pool_version * m0_pool_version_find(struct m0_pools_common *pc, const struct m0_fid *id)
static struct io_request req
static struct m0_sm_group * grp
M0_INTERNAL void m0_fop_init(struct m0_fop *fop, struct m0_fop_type *fopt, void *data, void(*fop_release)(struct m0_ref *))
M0_INTERNAL int m0_rpc_bulk_buf_databuf_add(struct m0_rpc_bulk_buf *rbuf, void *buf, m0_bcount_t count, m0_bindex_t index, struct m0_net_domain *netdom)
struct data_buf *** pi_paritybufs
struct m0_poolmach pv_mach
static void data_buf_fini(struct data_buf *buf)
static int nw_xfer_tioreq_map(struct nw_xfer_request *xfer, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt, struct target_ioreq **tio)
#define M0_LOG(level,...)
static void nw_xfer_req_complete(struct nw_xfer_request *xfer, bool rmw)
M0_INTERNAL void m0_sm_ast_post(struct m0_sm_group *grp, struct m0_sm_ast *ast)
static uint32_t layout_k(const struct m0_pdclust_layout *play)
static ssize_t file_aio_read(struct kiocb *kcb, struct iov_iter *from)
static void m0_atomic64_sub(struct m0_atomic64 *a, int64_t num)
int(* nxo_tioreq_map)(struct nw_xfer_request *xfer, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt, struct target_ioreq **tio)
static struct m0t1fs_sb * file_to_sb(const struct file *file)
M0_INTERNAL const struct m0_fid * m0t1fs_inode_fid(const struct m0t1fs_inode *ci)
struct m0_layout_instance pi_base
M0_INTERNAL int m0_sns_repair_spare_map(struct m0_poolmach *pm, const struct m0_fid *fid, struct m0_pdclust_layout *pl, struct m0_pdclust_instance *pi, uint64_t group, uint64_t unit, uint32_t *spare_slot_out, uint32_t *spare_slot_out_prev)
uint64_t(* do_out_shift)(const struct m0_file *file)
static m0_bcount_t v_seg_endpos(struct m0_indexvec_varr *ivec, uint32_t i)
static ssize_t aio_write(struct kiocb *kcb, struct iov_iter *from)
static const struct m0_bob_type ioreq_bobtype
static bool m0_is_po2(uint64_t val)
struct m0_rpc_bulk if_rbulk
M0_INTERNAL void m0_buf_init(struct m0_buf *buf, void *data, uint32_t nob)
void(* nxo_complete)(struct nw_xfer_request *xfer, bool rmw)
const struct address_space_operations m0t1fs_aops
static uint32_t io_seg_size(void)
M0_INTERNAL m0_bindex_t m0_ivec_varr_cursor_conti(const struct m0_ivec_varr_cursor *cur, m0_bindex_t dest)
M0_INTERNAL m0_bcount_t m0_ivec_varr_cursor_step(const struct m0_ivec_varr_cursor *cur)
static void buf_page_free(struct m0_buf *buf)
int(* pi_dgmode_recover)(struct pargrp_iomap *map)
static void seg_idx_inc_round(struct pargrp_iomap *map, uint32_t seg, uint64_t sz)
enum target_ioreq_type ti_req_type
struct m0_indexvec_varr ti_bufvec
struct m0_net_domain * ntm_dom
struct m0_net_buf_desc_data * id_descs
void * m0_fop_data(const struct m0_fop *fop)
static struct m0_be_emap_cursor it
M0_HT_DESCR_DEFINE(tioreqht, "Hash of target_ioreq objects", static, struct target_ioreq, ti_link, ti_magic, M0_T1FS_TIOREQ_MAGIC, M0_T1FS_TLIST_HEAD_MAGIC, ti_fid.f_container, tioreqs_hash_func, tioreq_key_eq)
M0_INTERNAL void m0_file_lock(struct m0_rm_owner *owner, struct m0_rm_incoming *req)
static struct m0_bob_type iofop_bobtype
struct m0_varr ti_pageattrs
static void io_rpc_item_cb(struct m0_rpc_item *item)
M0_INTERNAL int m0_poolmach_device_state(struct m0_poolmach *pm, uint32_t device_index, enum m0_pool_nd_state *state_out)
M0_INTERNAL int m0_parity_math_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
int m0t1fs_flush(struct file *file, fl_owner_t id)
static uint64_t round_up(uint64_t val, uint64_t size)
#define container_of(ptr, type, member)
struct m0_rm_credit rin_want
M0_INTERNAL void m0_mutex_lock(struct m0_mutex *mutex)
static void ioreq_sm_state_set(struct io_request *req, int state)
M0_ADDB2_ADD(M0_AVI_FS_CREATE, new_fid.f_container, new_fid.f_key, mode, rc)
static int pargrp_iomap_parity_verify(struct pargrp_iomap *map)
int(* iro_parity_verify)(struct io_request *req)
M0_INTERNAL int m0_parity_math_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
static int io_req_fop_init(struct io_req_fop *fop, struct target_ioreq *ti, enum page_attr pattr)
struct m0_net_buffer * nbe_buffer
M0_INTERNAL int m0_fid_cmp(const struct m0_fid *fid0, const struct m0_fid *fid1)
static int io_req_fop_dgmode_read(struct io_req_fop *irfop)
static struct m0_rpc_item * item
struct m0_pdclust_attr pl_attr
static void databufs_set_dgw_mode(struct pargrp_iomap *iomap, struct m0_ext *ext)
struct target_ioreq * irf_tioreq
static struct inode * iomap_to_inode(const struct pargrp_iomap *map)
M0_INTERNAL uint64_t m0_round_up(uint64_t val, uint64_t size)
static ssize_t file_aio_write(struct kiocb *kcb, struct iov_iter *from)
static int bulk_buffer_add(struct io_req_fop *irfop, struct m0_net_domain *dom, struct m0_rpc_bulk_buf **rbuf, uint32_t *delta, uint32_t maxsize)
M0_INTERNAL bool m0_tlist_is_empty(const struct m0_tl_descr *d, const struct m0_tl *list)
M0_INTERNAL void m0_rpc_bulk_buflist_empty(struct m0_rpc_bulk *rbulk)
static int ioreq_sm_timedwait(struct io_request *req, uint64_t state)
struct target_ioreq * dr_tioreq
enum m0_pool_nd_state ti_state
M0_INTERNAL int m0_sm_timedwait(struct m0_sm *mach, uint64_t states, m0_time_t deadline)
M0_INTERNAL uint64_t m0_round_down(uint64_t val, uint64_t size)
static int pargrp_iomap_dgmode_recover(struct pargrp_iomap *map)
M0_INTERNAL void m0_parity_math_calculate(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity)
#define M0_ASSERT_EX(cond)
static uint32_t unit_size
static uint32_t io_di_size(const struct io_request *req)
M0_INTERNAL int m0_pagesize_get(void)
static struct m0_sm_ast ast[NR]
M0_INTERNAL void m0_sm_group_unlock(struct m0_sm_group *grp)
M0_INTERNAL bool m0t1fs_inode_bob_check(struct m0t1fs_inode *bob)
int32_t m0_rpc_item_generic_reply_rc(const struct m0_rpc_item *reply)
static const struct m0_sm_conf io_sm_conf
static struct m0t1fs_inode m0inode
void m0_fop_put0_lock(struct m0_fop *fop)
void m0_fop_rpc_machine_set(struct m0_fop *fop, struct m0_rpc_machine *mach)
M0_INTERNAL m0_bcount_t m0_rpc_session_get_max_item_payload_size(const struct m0_rpc_session *session)
M0_INTERNAL int m0_indexvec_varr_alloc(struct m0_indexvec_varr *ivec, uint32_t len)
struct m0_rpc_machine * c_rpc_machine
static m0_bindex_t gfile_offset(m0_bindex_t toff, const struct pargrp_iomap *map, const struct m0_pdclust_layout *play, const struct m0_pdclust_src_addr *src)
static uint32_t rows_nr(struct m0_pdclust_layout *play)
static void cc_fop_release(struct m0_ref *ref)
M0_INTERNAL bool m0_ext_is_valid(const struct m0_ext *ext)
static int ioreq_dgmode_read(struct io_request *req, bool rmw)
struct m0_rpc_machine * m0_fop_rpc_machine(const struct m0_fop *fop)
#define M0_ERR_INFO(rc, fmt,...)
int(* nxo_distribute)(struct nw_xfer_request *xfer)
static int io_spare_map(const struct pargrp_iomap *map, const struct m0_pdclust_src_addr *src, uint32_t *spare_slot, uint32_t *spare_slot_prev, enum m0_pool_nd_state *eff_state)
static void client_passive_recv(const struct m0_net_buffer_event *evt)
return M0_ERR(-EOPNOTSUPP)
struct io_mem_stats iommstats
static void cc_rpc_item_cb(struct m0_rpc_item *item)
static void target_ioreq_type_set(struct target_ioreq *ti, enum target_ioreq_type type)
M0_INTERNAL void m0_rpc_machine_unlock(struct m0_rpc_machine *machine)
M0_INTERNAL struct m0_fop_cob_rw_reply * io_rw_rep_get(struct m0_fop *fop)
M0_INTERNAL void m0_rpc_bulk_default_cb(const struct m0_net_buffer_event *evt)
static void mark_page_as_read_failed(struct pargrp_iomap *map, uint32_t row, uint32_t col, enum page_attr page_type)
static m0_bindex_t data_page_offset_get(struct pargrp_iomap *map, uint32_t row, uint32_t col)
int(* pi_populate)(struct pargrp_iomap *iomap, struct m0_ivec_varr_cursor *cursor)
void m0_addb2_push(uint64_t id, int n, const uint64_t *value)
static void ioreq_no_unlock(struct io_request *req)
M0_INTERNAL m0_bindex_t m0_ivec_varr_cursor_index(const struct m0_ivec_varr_cursor *cur)
M0_INTERNAL struct m0_file * m0_fop_to_file(struct m0_fop *fop)
#define m0_tl_teardown(name, head, obj)
int(* tio_cc_fops_prepare)(struct target_ioreq *ti)
static bool io_request_invariant(struct io_request *req)
struct m0_net_buffer * bb_nbuf
struct m0_parity_math pi_math
M0_INTERNAL struct m0t1fs_inode * m0t1fs_inode_to_m0inode(const struct inode *inode)
static int ioreq_no_lock(struct io_request *req)
enum pargrp_iomap_state pi_state
void(* tio_seg_add)(struct target_ioreq *ti, const struct m0_pdclust_src_addr *src, const struct m0_pdclust_tgt_addr *tgt, m0_bindex_t gob_offset, m0_bcount_t count, struct pargrp_iomap *map)
static uint64_t page_nr(m0_bcount_t size)
M0_INTERNAL size_t m0_io_fop_size_get(struct m0_fop *fop)
struct m0_net_transfer_mc rm_tm
struct m0_io_descs crw_desc
static uint64_t page_id(m0_bindex_t offset)
struct nw_xfer_request ir_nwxfer
struct m0_rpc_item_header2 ri_header
void m0_sm_state_set(struct m0_sm *mach, int state)
struct m0_rpc_machine * m0_fop_session_machine(const struct m0_rpc_session *s)
M0_INTERNAL struct m0t1fs_inode * m0t1fs_file_to_m0inode(const struct file *file)
static void data_buf_dealloc_fini(struct data_buf *buf)
static struct m0_pdclust_instance * pdlayout_instance(const struct m0_layout_instance *li)
const struct nw_xfer_ops * nxr_ops
M0_INTERNAL bool m0_is_io_fop_rep(const struct m0_fop *fop)
static struct m0_fop reply_fop
#define M0_ADDB2_OBJ(obj)
#define m0_htable_forall(name, var, htable,...)
M0_INTERNAL int m0t1fs_ref_get_lock(struct m0t1fs_sb *csb)
static ssize_t file_dio_write(struct kiocb *kcb, struct iov_iter *from)
static void ioreq_file_unlock(struct io_request *req)
#define bob_of(ptr, type, field, bt)
static void m0_atomic64_dec(struct m0_atomic64 *a)
M0_INTERNAL struct m0_poolmach * m0t1fs_file_to_poolmach(const struct file *file)
struct m0_atomic64 nxr_ccfop_nr
static int unit_state(const struct m0_pdclust_src_addr *src, const struct io_request *req, enum m0_pool_nd_state *state)
static struct m0_bufvec bvec
static void dgmode_rwvec_dealloc_fini(struct dgmode_rwvec *dg)
int m0t1fs_setxattr(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, const void *value, size_t size, int flags) int m0t1fs_setxattr(struct dentry *dentry
M0_INTERNAL int m0_varr_init(struct m0_varr *arr, uint64_t nr, size_t size, size_t bufsize)
static int ioreq_parity_recalc(struct io_request *req)
int m0t1fs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
static struct m0_stob_domain * dom
struct m0_varr dr_pageattrs
void(* iro_iomaps_destroy)(struct io_request *req)
M0_INTERNAL struct m0_pdclust_layout * m0_layout_to_pdl(const struct m0_layout *l)
struct m0_fop * m0_fop_get(struct m0_fop *fop)
const struct m0_rpc_item_type * ri_type
M0_INTERNAL ssize_t m0t1fs_aio(struct kiocb *kcb, const struct iovec *iov, struct m0_indexvec_varr *ivv, enum io_req_type rw)
static uint64_t layout_unit_size(const struct m0_pdclust_layout *play)
struct m0_rpc_item * ri_reply
void * m0_alloc(size_t size)
struct m0_semaphore m0t1fs_cpus_sem
struct m0_fop_mod_rep rwr_mod_rep
struct m0_sm_group * sm_grp
M0_INTERNAL uint32_t m0_fid_cob_device_id(const struct m0_fid *cob_fid)
M0_INTERNAL void m0_mutex_init(struct m0_mutex *mutex)
enum pargrp_iomap_rmwtype pi_rtype
struct m0_0vec bb_zerovec
static bool nw_xfer_request_invariant(const struct nw_xfer_request *xfer)
M0_TL_DEFINE(iofops, static, struct io_req_fop)
int(* pi_dgmode_postprocess)(struct pargrp_iomap *map)
static void ioreq_failed_fini(struct io_request *req, int rc)
static struct m0_sm_group * file_to_smgroup(const struct file *file)
M0_INTERNAL int m0t1fs_setattr(struct dentry *dentry, struct iattr *attr)
static int nw_xfer_req_dispatch(struct nw_xfer_request *xfer)
static int pargrp_iomap_select_ro_rr(struct pargrp_iomap *map, m0_bcount_t data_pages_nr, m0_bcount_t parity_pages_nr)
m0_net_buffer_cb_proc_t nbc_cb[M0_NET_QT_NR]
static uint64_t tolerance_of_level(struct io_request *req, uint64_t lv)
static bool pargrp_iomap_spans_seg(struct pargrp_iomap *map, m0_bindex_t index, m0_bcount_t count)
static m0_bindex_t offset
static void pargrp_src_addr(m0_bindex_t index, const struct io_request *req, const struct target_ioreq *tio_req, struct m0_pdclust_src_addr *src)
M0_INTERNAL void m0_fd_bwd_map(struct m0_pdclust_instance *pi, const struct m0_pdclust_tgt_addr *tgt, struct m0_pdclust_src_addr *src)
static int pargrp_iomap_paritybufs_alloc(struct pargrp_iomap *map)
static const struct m0_bob_type nwxfer_bobtype
int(* pi_parity_verify)(struct pargrp_iomap *map)
struct m0_indexvec_varr dr_bufvec
struct m0_htable nxr_tioreqs_hash
M0_INTERNAL int m0_fop_data_alloc(struct m0_fop *fop)
static void target_ioreq_fini(struct target_ioreq *ti)
M0_INTERNAL void m0_fop_fini(struct m0_fop *fop)
struct m0_rpc_session * ti_session
struct m0_indexvec_varr ir_ivv
static bool pargrp_iomap_invariant(struct pargrp_iomap *map)
static struct m0_clink clink[RDWR_REQUEST_MAX]
static uint64_t min64u(uint64_t a, uint64_t b)
static int pargrp_iomap_populate_pi_ivec(struct pargrp_iomap *map, struct m0_ivec_varr_cursor *cursor, bool rmw)
static int pargrp_iomap_dgmode_postprocess(struct pargrp_iomap *map)
void(* iro_file_unlock)(struct io_request *req)
struct m0_indexvec_varr dr_ivec_varr
M0_INTERNAL int m0t1fs_cob_setattr(struct inode *inode, struct m0t1fs_mdop *mo)
static void page_pos_get(struct pargrp_iomap *map, m0_bindex_t index, uint32_t *row, uint32_t *col)
static struct fdmi_ctx ctx
M0_INTERNAL void m0_indexvec_varr_free(struct m0_indexvec_varr *ivec)
static uint64_t data_size(const struct m0_pdclust_layout *play)
void m0_addb2_pop(uint64_t id)
static const struct m0_rpc_item_ops io_item_ops
M0_TL_DESCR_DECLARE(rpcbulk, M0_EXTERN)
const struct target_ioreq_ops * ti_ops
int(* iro_dgmode_recover)(struct io_request *req)
static void irfop_fini(struct io_req_fop *irfop)
static uint64_t round_down(uint64_t val, uint64_t size)
static int nw_xfer_io_distribute(struct nw_xfer_request *xfer)
M0_INTERNAL const char * m0_rpc_item_opname(const struct m0_rpc_item *item)
static uint32_t layout_n(const struct m0_pdclust_layout *play)
static int ioreq_parity_verify(struct io_request *req)
const struct iovec * ir_iovec
static ssize_t m0t1fs_direct_IO(struct kiocb *kcb, struct iov_iter *from)
static struct m0_pdclust_layout * pdlayout_get(const struct io_request *req)
static int64_t m0_atomic64_get(const struct m0_atomic64 *a)
static int pargrp_iomap_dgmode_process(struct pargrp_iomap *map, struct target_ioreq *tio, m0_bindex_t *index, uint32_t count)
void(* rio_replied)(struct m0_rpc_item *item)
static int pargrp_iomap_parity_recalc(struct pargrp_iomap *map)
#define m0_forall(var, nr,...)
static int ioreq_dgmode_write(struct io_request *req, bool rmw)
M0_INTERNAL int m0_rpc_session_validate(struct m0_rpc_session *session)
const struct m0_net_buffer_callbacks client_buf_bulk_cb
struct m0_fop_type m0_fop_cob_readv_fopt
M0_INTERNAL void m0_ivec_varr_cursor_init(struct m0_ivec_varr_cursor *cur, struct m0_indexvec_varr *ivec)
M0_INTERNAL size_t m0_rpc_bulk_buf_length(struct m0_rpc_bulk *rbulk)
static const struct m0_bob_type tioreq_bobtype
M0_INTERNAL size_t m0_rpc_bulk_store_del_unqueued(struct m0_rpc_bulk *rbulk)
static bool should_spare_be_mapped(struct io_request *req, enum m0_pool_nd_state dev_state)
static int pargrp_iomap_readold_auxbuf_alloc(struct pargrp_iomap *map)
struct m0_pdclust_tgt_addr tgt
static uint8_t fail[DATA_UNIT_COUNT_MAX+PARITY_UNIT_COUNT_MAX]
static const struct m0_rpc_item_ops cc_item_ops
static struct m0_parity_math * parity_math(struct io_request *req)
static void paritybufs_set_dgw_mode(struct pargrp_iomap *iomap, struct m0_pdclust_layout *play, uint64_t unit)
M0_INTERNAL int64_t m0_ref_read(const struct m0_ref *ref)
M0_BOB_DEFINE(static, &tioreq_bobtype, target_ioreq)
static void cc_bottom_half(struct m0_sm_group *grp, struct m0_sm_ast *ast)
static const struct m0_bob_type pgiomap_bobtype
static void io_bottom_half(struct m0_sm_group *grp, struct m0_sm_ast *ast)
M0_INTERNAL void m0_varr_fini(struct m0_varr *arr)
M0_INTERNAL void m0_file_unlock(struct m0_rm_incoming *req)
static int ioreq_iosm_handle(struct io_request *req)
static int pargrp_iomap_seg_process(struct pargrp_iomap *map, uint64_t seg, bool rmw)
#define indexvec_varr_dump(ivec)
static void nw_xfer_request_fini(struct nw_xfer_request *xfer)
#define M0_FI_ENABLED(tag)
static int ioreq_iomaps_parity_groups_cal(struct io_request *req)
static bool tioreq_key_eq(const void *key1, const void *key2)
M0_INTERNAL void m0_sm_init(struct m0_sm *mach, const struct m0_sm_conf *conf, uint32_t state, struct m0_sm_group *grp)
M0_INTERNAL void m0_rpc_machine_lock(struct m0_rpc_machine *machine)
static struct m0_rpc_session * target_session(struct io_request *req, struct m0_fid tfid)
static uint64_t indexvec_varr_count(struct m0_indexvec_varr *varr)
static struct m0_layout_instance * layout_instance(const struct io_request *req)
static bool io_req_fop_invariant(const struct io_req_fop *fop)
static int pargrp_iomap_init(struct pargrp_iomap *map, struct io_request *req, uint64_t grpid)
#define M0_ALLOC_PTR(ptr)
struct cc_req_fop ti_cc_fop
M0_INTERNAL void m0_clink_add(struct m0_chan *chan, struct m0_clink *link)
static int ioreq_user_data_copy(struct io_request *req, enum copy_direction dir, enum page_attr filter)
static bool pargrp_iomap_invariant_nr(struct io_request *req)
const struct m0_rpc_item_ops * ri_ops
static void ioreq_sm_state_set_nolock(struct io_request *req, int state)
int(* nxo_dispatch)(struct nw_xfer_request *xfer)
int(* iro_dgmode_read)(struct io_request *req, bool rmw)
M0_INTERNAL m0_bcount_t m0_net_domain_get_max_buffer_desc_size(struct m0_net_domain *dom)
const struct m0_uint128 m0_rm_m0t1fs_group
enum nw_xfer_state nxr_state
struct m0_indexvec_varr ti_ivv
static uint64_t pargrp_iomap_auxbuf_alloc(struct pargrp_iomap *map, uint32_t row, uint32_t col)
struct m0_rpc_session * ri_session
static int target_ioreq_init(struct target_ioreq *ti, struct nw_xfer_request *xfer, const struct m0_fid *cobfid, uint64_t ta_obj, struct m0_rpc_session *session, uint64_t size)
static bool data_buf_invariant_nr(const struct pargrp_iomap *map)
static void io_req_fop_release(struct m0_ref *ref)
static void target_ioreq_seg_add(struct target_ioreq *ti, const struct m0_pdclust_src_addr *src, const struct m0_pdclust_tgt_addr *tgt, m0_bindex_t gob_offset, m0_bcount_t count, struct pargrp_iomap *map)
struct m0_fop_type m0_fop_cob_create_fopt
struct m0_rpc_item * m0_fop_to_rpc_item(const struct m0_fop *fop)
static void ioreq_sm_failed(struct io_request *req, int rc)
static int pargrp_iomap_populate(struct pargrp_iomap *map, struct m0_ivec_varr_cursor *cursor)
M0_INTERNAL enum m0_pdclust_unit_type m0_pdclust_unit_classify(const struct m0_pdclust_layout *pl, int unit)
static void nw_xfer_request_init(struct nw_xfer_request *xfer)
M0_TL_DESCR_DEFINE(iofops, "List of IO fops", static, struct io_req_fop, irf_link, irf_magic, M0_T1FS_IOFOP_MAGIC, M0_T1FS_TIOREQ_MAGIC)
static uint64_t parity_units_page_nr(const struct m0_pdclust_layout *play)
static int start(struct m0_fom *fom)
struct data_buf *** pi_databufs
M0_INTERNAL void m0_mutex_fini(struct m0_mutex *mutex)
M0_INTERNAL void m0_clink_fini(struct m0_clink *link)
static int nw_xfer_tioreq_get(struct nw_xfer_request *xfer, const struct m0_fid *fid, uint64_t ta_obj, struct m0_rpc_session *session, uint64_t size, struct target_ioreq **out)
void m0_fop_put_lock(struct m0_fop *fop)
M0_INTERNAL bool m0_rpc_bulk_is_empty(struct m0_rpc_bulk *rbulk)
static void ioreq_pgiomap_find(struct io_request *req, uint64_t grpid, uint64_t *cursor, struct pargrp_iomap **out)
const struct io_request_ops * ir_ops
struct m0_atomic64 nxr_iofop_nr
static struct m0_fop * fop
static void user_page_unmap(struct data_buf *dbuf, bool set_dirty)
M0_INTERNAL int32_t m0_net_domain_get_max_buffer_segments(struct m0_net_domain *dom)
static void indexvec_sort(struct m0_indexvec_varr *ivec)
struct m0_io_fop irf_iofop
M0_INTERNAL void m0_sm_group_lock(struct m0_sm_group *grp)
struct pargrp_iomap ** ir_iomaps
struct m0_fop * m0_rpc_item_to_fop(const struct m0_rpc_item *item)
static int target_ioreq_iofops_prepare(struct target_ioreq *ti, enum page_attr filter)
static const struct m0_bob_type dtbuf_bobtype
int(* pi_parity_recalc)(struct pargrp_iomap *map)
M0_INTERNAL void m0_rm_owner_unlock(struct m0_rm_owner *owner)
M0_INTERNAL int user_page_map(struct data_buf *dbuf, unsigned long user_addr)
static uint64_t group_id(m0_bindex_t index, m0_bcount_t dtsize)
M0_INTERNAL struct m0_rpc_session * m0t1fs_container_id_to_session(const struct m0_pool_version *pver, uint64_t container_id)
static struct m0_be_seg * seg
int(* iro_iomaps_prepare)(struct io_request *req)
M0_INTERNAL void iov_iter_advance(struct iov_iter *i, size_t bytes)
static uint32_t ioreq_sm_state(const struct io_request *req)
#define M0_ASSERT_INFO(cond, fmt,...)
#define m0_varr_for(arr, type, idx, obj)
static void data_buf_init(struct data_buf *buf, void *addr, uint64_t flags)
static uint64_t tioreqs_hash_func(const struct m0_htable *htable, const void *k)
M0_INTERNAL void m0_io_fop_fini(struct m0_io_fop *iofop)
M0_INTERNAL int m0_io_fop_init(struct m0_io_fop *iofop, const struct m0_fid *gfid, struct m0_fop_type *ftype, void(*fop_release)(struct m0_ref *))
static void device_state_reset(struct nw_xfer_request *xfer, bool rmw)
static const struct pargrp_iomap_ops iomap_ops
static bool is_session_marked(struct io_request *req, struct m0_rpc_session *session)
M0_INTERNAL void m0_ext_intersection(const struct m0_ext *e0, const struct m0_ext *e1, struct m0_ext *result)
M0_INTERNAL struct m0_pdclust_instance * m0_layout_instance_to_pdi(const struct m0_layout_instance *li)
M0_INTERNAL void m0_semaphore_down(struct m0_semaphore *semaphore)
struct nw_xfer_request * ti_nwxfer
struct m0_rm_incoming ir_in
M0_INTERNAL void m0_rpc_bulk_store_del(struct m0_rpc_bulk *rbulk)
M0_INTERNAL bool m0_is_read_fop(const struct m0_fop *fop)
static uint32_t target_ioreq_type_get(struct target_ioreq *ti)
static uint64_t pargrp_id_find(m0_bindex_t index, const struct io_request *req, const struct io_req_fop *ir_fop)
M0_INTERNAL void m0_rm_owner_lock(struct m0_rm_owner *owner)
M0_INTERNAL void m0_semaphore_up(struct m0_semaphore *semaphore)
struct m0_uint128 cr_group_id
static void seg_align(struct pargrp_iomap *map, uint32_t seg, m0_bindex_t end, uint64_t sz)
M0_INTERNAL struct m0_fop_cob_rw * io_rw_get(struct m0_fop *fop)
static bool should_req_sm_complete(struct io_request *req)
static int32_t min32(int32_t a, int32_t b)
struct target_ioreq * db_tioreq
M0_INTERNAL bool m0_fid_is_valid(const struct m0_fid *fid)
static uint64_t iomap_page_nr(struct pargrp_iomap *map)
M0_INTERNAL void m0_fd_fwd_map(struct m0_pdclust_instance *pi, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt)
static void ioreq_iomaps_destroy(struct io_request *req)
M0_INTERNAL int m0_io_fop_prepare(struct m0_fop *fop)
struct m0_rpc_machine * ri_rmachine
int(* iro_iosm_handle)(struct io_request *req)
static struct m0_dtm_oper_descr reply
static void m0_atomic64_add(struct m0_atomic64 *a, int64_t num)
M0_INTERNAL int m0_rpc_bulk_buf_add(struct m0_rpc_bulk *rbulk, uint32_t segs_nr, m0_bcount_t length, struct m0_net_domain *netdom, struct m0_net_buffer *nb, struct m0_rpc_bulk_buf **out)
const struct inode_operations m0t1fs_reg_inode_operations
M0_INTERNAL struct m0_pool_version * m0t1fs_file_to_pver(const struct file *file)
struct m0_fop_type m0_fop_cob_writev_fopt
static const struct target_ioreq_ops tioreq_ops
#define m0_tl_for(name, head, obj)
static int device_check(struct io_request *req)
uint64_t * ir_failed_session
struct m0_rpc_item f_item
int(* iro_user_data_copy)(struct io_request *req, enum copy_direction dir, enum page_attr filter)
static int iofop_async_submit(struct m0_io_fop *iofop, struct m0_rpc_session *session)
static bool is_pver_dud(uint32_t fdev_nr, uint32_t dev_k, uint32_t fsvc_nr, uint32_t svc_k)
struct m0_pdclust_src_addr src
static bool data_buf_invariant(const struct data_buf *db)
struct dgmode_rwvec * ti_dgvec
M0_INTERNAL void m0_bob_type_tlist_init(struct m0_bob_type *bt, const struct m0_tl_descr *td)
static uint32_t io_desc_size(struct m0_net_domain *ndom)
M0_INTERNAL struct m0t1fs_sb * m0inode_to_sb(const struct m0t1fs_inode *m0inode)
const struct pargrp_iomap_ops * pi_ops
M0_INTERNAL void m0t1fs_ref_put_lock(struct m0t1fs_sb *csb)
#define offsetof(typ, memb)
const struct file_operations m0t1fs_reg_file_operations
M0_INTERNAL void m0_poolmach_gob2cob(struct m0_poolmach *pm, const struct m0_fid *gfid, uint32_t idx, struct m0_fid *cob_fid)
struct m0_rpc_conn * s_conn
M0_HT_DEFINE(tioreqht, static, struct target_ioreq, uint64_t)
static void io_request_fini(struct io_request *req)
struct m0_be_tx_remid fmr_remid
static uint64_t target_offset(uint64_t frame, struct m0_pdclust_layout *play, m0_bindex_t gob_offset)
M0_INTERNAL int m0t1fs_size_update(struct dentry *dentry, uint64_t newsize)
static struct m0_sm_state_descr io_states[]
int(* tio_iofops_prepare)(struct target_ioreq *ti, enum page_attr filter)
static uint8_t parity[DATA_UNIT_COUNT_MAX][UNIT_BUFF_SIZE_MAX]
void m0t1fs_fsync_record_update(struct m0_reqh_service_ctx *service, struct m0t1fs_sb *csb, struct m0t1fs_inode *inode, struct m0_be_tx_remid *btr)
static const struct m0_fid * file_to_fid(const struct file *file)
static uint64_t max64u(uint64_t a, uint64_t b)
const struct m0_di_ops * fi_di_ops
static struct m0_addb2_frame_header last
int m0t1fs_getxattr(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *buffer, size_t size) ssize_t m0t1fs_getxattr(struct dentry *dentry
static int pargrp_iomap_readrest(struct pargrp_iomap *map)
static struct target_ioreq * target_ioreq_locate(struct nw_xfer_request *xfer, const struct m0_fid *fid)
static struct m0_indexvec_varr * indexvec_create(unsigned long seg_nr, const struct iovec *iov, loff_t pos)
M0_INTERNAL bool m0_ivec_varr_cursor_move(struct m0_ivec_varr_cursor *cur, m0_bcount_t count)
static void m0_atomic64_set(struct m0_atomic64 *a, int64_t num)
static uint64_t pargrp_iomap_fullpages_count(struct pargrp_iomap *map)
static struct data_buf * data_buf_alloc_init(enum page_attr pattr)
int(* iro_file_lock)(struct io_request *req)
M0_INTERNAL struct inode * m0t1fs_file_to_inode(const struct file *file)
static const struct nw_xfer_ops xfer_ops
static int pargrp_iomap_pages_mark_as_failed(struct pargrp_iomap *map, enum m0_pdclust_unit_type type)
#define m0_tl_forall(name, var, head,...)
static int pargrp_iomap_databuf_alloc(struct pargrp_iomap *map, uint32_t row, uint32_t col)
M0_INTERNAL struct m0_reqh_service_ctx * m0_reqh_service_ctx_from_session(struct m0_rpc_session *session)
M0_INTERNAL void io_bob_tlists_init(void)
struct m0_indexvec_varr * vc_ivv
M0_INTERNAL void m0_sm_fini(struct m0_sm *mach)