44 #define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_CLIENT 57 .
bt_name =
"nw_xfer_request_bobtype",
107 #ifndef ENABLE_DATA_INTEGRITY 139 *row = pg_id %
rows_nr(play,
map->pi_ioo->ioo_obj);
140 *col = pg_id /
rows_nr(play,
map->pi_ioo->ioo_obj);
301 const uint64_t *
k1 = (uint64_t *)key1;
302 const uint64_t *
k2 = (uint64_t *)key2;
327 _0C(target_ioreq_bob_check(ti)) &&
342 return xfer !=
NULL &&
343 _0C(nw_xfer_request_bob_check(xfer)) &&
377 target_ioreq_bob_fini(ti);
378 tioreqht_tlink_fini(ti);
509 struct m0_ext goff_span_ext;
510 bool is_goff_in_range;
511 void *dst_attr =
NULL;
576 while (pgstart < toff +
count) {
577 pgend =
min64u(pgstart + page_size,
586 uint32_t row =
map->pi_max_row;
587 uint32_t col =
map->pi_max_col;
592 buf =
map->pi_databufs[row][col];
608 unit_type,
buf->db_flags)) {
609 tseg =
SEG_NR(trunc_ivec);
610 INDEX(trunc_ivec, tseg) = pgstart;
611 COUNT(trunc_ivec, tseg) = pgend - pgstart;
614 "added to target ioreq with "FID_F,
615 tseg,
INDEX(trunc_ivec, tseg),
616 COUNT(trunc_ivec, tseg),
632 pattr[
seg] |=
buf->db_flags;
638 map->pi_grpid, pattr[
seg],
641 "] added to target_ioreq with "FID_F 642 " with flags 0x%x: ",
seg,
650 is_goff_in_range =
m0_ext_is_in(&goff_span_ext, goff) &&
670 &ioo->
ioo_ext, unit_sz, cs_sz);
776 M0_ENTRY(
"ioreq_fop %p net_domain %p delta_size %d",
828 uint32_t seg_idx,
bool rd_in_wr)
830 return (p_attr ==
PA_DATA && rd_in_wr && aux !=
NULL &&
852 uint32_t fop_cksm_nob;
853 uint32_t dispatched_cksm_nob = 0;
868 bool read_in_write =
false;
875 uint32_t ndom_max_segs;
878 M0_ENTRY(
"prepare io fops for target ioreq %p filter 0x%x, tfid "FID_F,
885 if (
rc != 0 && (!M0_IN(
rc, (-ECANCELED, -EINVAL))))
896 read_in_write =
true;
969 bbsegs < ndom_max_segs) {
1001 while (segnext <
SEG_NR(ivec) &&
1002 xfer_len < max_seg_size) {
1007 if (bufnext ==
NULL)
1010 if (
buf + xfer_len == bufnext) {
1021 if (
rc == -EMSGSIZE) {
1046 &rbuf, &delta, maxsize);
1083 !
instance->m0c_config->mc_is_read_verify &&
1098 dispatched_cksm_nob += fop_cksm_nob;
1099 M0_ASSERT(dispatched_cksm_nob <= ti->ti_cksum_copied);
1142 iofops_tlist_add(&ti->
ti_iofops, irfop);
1146 " read bulks = %" PRIu64 ", list_len=%d",
1152 (
int)iofops_tlist_length(&ti->
ti_iofops));
1197 const struct m0_fid *cobfid,
1210 ti, xfer,
FID_P(cobfid));
1244 tioreqht_tlink_init(ti);
1245 target_ioreq_bob_init(ti);
1421 for (row = row_start; row <= row_end; ++row) {
1462 bool do_cobs =
true;
1464 unsigned int op_code;
1488 M0_ENTRY(
"nw_xfer_request %p", xfer);
1581 for (unit = 0; unit <
layout_k(play); ++unit) {
1626 "nxo_tioreq_map() failed: rc=%d",
1676 M0_ENTRY(
"nw_xfer_request %p, rmw %s", xfer,
1677 rmw ? (
char *)
"true" : (
char *)
"false");
1721 "item %p[%u], ri_error %d, ri_state %d",
1745 "[%p] item %p, target fid "FID_F"fop %p, " 1800 uint64_t nr_dispatched = 0;
1896 FID_F"@%p, item %p, fop_nr=%llu, rc=%d, " 1899 (
unsigned long long)
1912 else if (post_error == 0)
1918 if (
rc == 0 && nr_dispatched == 0 && post_error == 0) {
1929 instance->m0c_config->mc_is_read_verify);
1935 "nr_dispatched %llu", ioo,
1938 (
unsigned long long)nr_dispatched);
2041 uint32_t spare_slot;
2042 uint32_t spare_slot_prev;
2046 M0_ENTRY(
"nw_xfer_request=%p", xfer);
2080 ioo,
FID_P(&tfid), dev_state);
2086 &spare_slot, &spare_slot_prev);
2092 spare.
sa_unit = spare_slot_prev;
2107 dev_state = dev_state_prev;
2118 (*tio)->ti_state = dev_state;
2136 M0_ENTRY(
"nw_xfer_request : %p", xfer);
2141 nw_xfer_request_bob_init(xfer);
2161 M0_ENTRY(
"nw_xfer_request : %p", xfer);
2171 nw_xfer_request_bob_fini(xfer);
2177 #undef M0_TRACE_SUBSYSTEM
M0_INTERNAL m0_bcount_t m0_net_domain_get_max_buffer_segment_size(struct m0_net_domain *dom)
static void m0_atomic64_inc(struct m0_atomic64 *a)
static uint64_t tioreqs_hash_func(const struct m0_htable *htable, const void *k)
M0_INTERNAL void m0_ivec_cursor_init(struct m0_ivec_cursor *cur, const struct m0_indexvec *ivec)
M0_INTERNAL int m0_rpc_post(struct m0_rpc_item *item)
static bool target_ioreq_invariant(const struct target_ioreq *ti)
M0_INTERNAL bool m0_pdclust_is_replicated(struct m0_pdclust_layout *play)
#define M0_ALLOC_ARR(arr, nr)
M0_INTERNAL m0_bcount_t m0_ext_length(const struct m0_ext *ext)
M0_INTERNAL int m0_bitmap_init(struct m0_bitmap *map, size_t nr)
static void nw_xfer_req_complete(struct nw_xfer_request *xfer, bool rmw)
M0_INTERNAL m0_bcount_t m0_io_fop_byte_count(struct m0_io_fop *iofop)
#define m0_htable_for(name, var, htable)
M0_INTERNAL struct m0_rpc_session * m0_obj_container_id_to_session(struct m0_pool_version *pv, uint64_t container_id)
M0_INTERNAL int m0_indexvec_alloc(struct m0_indexvec *ivec, uint32_t len)
int const char const void size_t int flags
static int bulk_buffer_add(struct ioreq_fop *irfop, struct m0_net_domain *dom, struct m0_rpc_bulk_buf **rbuf, uint32_t *delta, uint32_t maxsize)
M0_INTERNAL void m0_bitmap_fini(struct m0_bitmap *map)
static void paritybufs_set_dgw_mode(struct pargrp_iomap *iomap, struct m0_op_io *ioo, uint64_t unit)
M0_INTERNAL m0_bcount_t m0_ivec_cursor_step(const struct m0_ivec_cursor *cur)
struct m0_atomic64 nxr_rdbulk_nr
M0_INTERNAL bool m0__obj_is_di_enabled(struct m0_op_io *ioo)
struct m0_bufvec nb_buffer
M0_HT_DEFINE(tioreqht, M0_INTERNAL, struct target_ioreq, uint64_t)
M0_INTERNAL struct m0_pool_version * m0_pool_version_find(struct m0_pools_common *pc, const struct m0_fid *id)
M0_INTERNAL bool m0__is_oostore(struct m0_client *instance)
M0_INTERNAL int m0_rpc_bulk_buf_databuf_add(struct m0_rpc_bulk_buf *rbuf, void *buf, m0_bcount_t count, m0_bindex_t index, struct m0_net_domain *netdom)
struct m0_pool_version * pv
struct data_buf *** pi_paritybufs
M0_INTERNAL void nw_xfer_request_init(struct nw_xfer_request *xfer)
#define M0_LOG(level,...)
struct m0_io_fop irf_iofop
static bool should_spare_be_mapped(struct m0_op_io *ioo, enum m0_pool_nd_state dev_state)
static uint32_t layout_k(const struct m0_pdclust_layout *play)
int(* nxo_tioreq_map)(struct nw_xfer_request *xfer, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt, struct target_ioreq **tio)
static int nw_xfer_tioreq_map(struct nw_xfer_request *xfer, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt, struct target_ioreq **tio)
M0_INTERNAL int m0_sns_repair_spare_map(struct m0_poolmach *pm, const struct m0_fid *fid, struct m0_pdclust_layout *pl, struct m0_pdclust_instance *pi, uint64_t group, uint64_t unit, uint32_t *spare_slot_out, uint32_t *spare_slot_out_prev)
uint64_t(* do_out_shift)(const struct m0_file *file)
static int dgmode_rwvec_alloc_init(struct target_ioreq *ti)
struct m0_rpc_bulk if_rbulk
M0_INTERNAL int ioreq_cc_fop_init(struct target_ioreq *ti)
static int target_cob_fop_prepare(struct target_ioreq *ti)
static uint32_t io_seg_size(void)
enum target_ioreq_type ti_req_type
struct m0_indexvec_varr ti_bufvec
struct m0_net_domain * ntm_dom
static void dgmode_rwvec_dealloc_fini(struct dgmode_rwvec *dg)
M0_INTERNAL void m0_indexvec_free(struct m0_indexvec *ivec)
static struct m0_rpc_session * target_session(struct m0_op_io *ioo, struct m0_fid tfid)
struct m0_indexvec ti_trunc_ivec
M0_BOB_DEFINE(M0_INTERNAL, &nwxfer_bobtype, nw_xfer_request)
struct m0_varr ti_pageattrs
static void * buf_aux_chk_get(struct m0_bufvec *aux, enum page_attr p_attr, uint32_t seg_idx, bool rd_in_wr)
M0_INTERNAL int m0_poolmach_device_state(struct m0_poolmach *pm, uint32_t device_index, enum m0_pool_nd_state *state_out)
M0_INTERNAL m0_bcount_t m0_extent_get_checksum_nob(m0_bindex_t ext_start, m0_bindex_t ext_length, m0_bindex_t unit_sz, m0_bcount_t cs_size)
M0_ADDB2_ADD(M0_AVI_FS_CREATE, new_fid.f_container, new_fid.f_key, mode, rc)
M0_INTERNAL int m0_fid_cmp(const struct m0_fid *fid0, const struct m0_fid *fid1)
static struct m0_rpc_item * item
void target_ioreq_cancel(struct target_ioreq *ti)
M0_INTERNAL uint64_t m0__obj_lid(struct m0_obj *obj)
struct m0_indexvec pi_ivec
M0_INTERNAL void m0_rpc_bulk_buflist_empty(struct m0_rpc_bulk *rbulk)
struct m0_buf crw_di_data_cksum
struct target_ioreq * dr_tioreq
enum m0_pool_nd_state ti_state
M0_INTERNAL int ioreq_fop_init(struct ioreq_fop *fop, struct target_ioreq *ti, enum page_attr pattr)
static struct target_ioreq * target_ioreq_locate(struct nw_xfer_request *xfer, struct m0_fid *fid)
static uint32_t unit_size
int m0_obj_layout_id_to_unit_size(uint64_t layout_id)
void m0_fop_rpc_machine_set(struct m0_fop *fop, struct m0_rpc_machine *mach)
M0_INTERNAL m0_bcount_t m0_rpc_session_get_max_item_payload_size(const struct m0_rpc_session *session)
const struct m0_bob_type ioo_bobtype
struct m0_rpc_machine * c_rpc_machine
static uint32_t rows_nr(struct m0_pdclust_layout *play)
M0_INTERNAL bool m0_ext_is_valid(const struct m0_ext *ext)
struct nw_xfer_request ioo_nwxfer
#define M0_ERR_INFO(rc, fmt,...)
int(* nxo_distribute)(struct nw_xfer_request *xfer)
return M0_ERR(-EOPNOTSUPP)
static int target_ioreq_init(struct target_ioreq *ti, struct nw_xfer_request *xfer, const struct m0_fid *cobfid, uint64_t ta_obj, struct m0_rpc_session *session, uint64_t size)
M0_INTERNAL bool nw_xfer_request_invariant(const struct nw_xfer_request *xfer)
M0_INTERNAL struct m0_poolmach * ioo_to_poolmach(struct m0_op_io *ioo)
M0_INTERNAL struct m0_client * m0__op_instance(const struct m0_op *op)
enum sns_repair_state ioo_sns_state
struct m0_indexvec ioo_ext
void m0_rpc_item_cancel(struct m0_rpc_item *item)
M0_INTERNAL void ioreq_sm_state_set_locked(struct m0_op_io *ioo, int state)
#define m0_tl_teardown(name, head, obj)
int(* tio_cc_fops_prepare)(struct target_ioreq *ti)
static int nw_xfer_tioreq_get(struct nw_xfer_request *xfer, struct m0_fid *fid, uint64_t ta_obj, struct m0_rpc_session *session, uint64_t size, struct target_ioreq **out)
static uint32_t io_di_size(struct m0_op_io *ioo)
struct m0_net_buffer * bb_nbuf
M0_INTERNAL void nw_xfer_request_fini(struct nw_xfer_request *xfer)
enum pargrp_iomap_state pi_state
void(* tio_seg_add)(struct target_ioreq *ti, const struct m0_pdclust_src_addr *src, const struct m0_pdclust_tgt_addr *tgt, m0_bindex_t gob_offset, m0_bcount_t count, struct pargrp_iomap *map)
static uint64_t page_nr(m0_bcount_t size)
uint32_t * ti_cksum_seg_b_nob
M0_INTERNAL size_t m0_io_fop_size_get(struct m0_fop *fop)
struct m0_net_transfer_mc rm_tm
static uint64_t page_id(m0_bindex_t offset)
m0_time_t m0_time_now(void)
struct m0_rpc_item_header2 ri_header
static struct m0_pdclust_instance * pdlayout_instance(const struct m0_layout_instance *li)
const struct nw_xfer_ops * nxr_ops
#define m0_htable_forall(name, var, htable,...)
#define bob_of(ptr, type, field, bt)
struct m0_bufvec ioo_data
static struct m0_bufvec bvec
M0_INTERNAL int ioreq_fop_async_submit(struct m0_io_fop *iofop, struct m0_rpc_session *session)
static struct m0_stob_domain * dom
struct m0_varr dr_pageattrs
static void databufs_set_dgw_mode(struct pargrp_iomap *iomap, struct m0_pdclust_layout *play, struct m0_ext *ext)
M0_INTERNAL bool ioreq_fop_invariant(const struct ioreq_fop *fop)
M0_INTERNAL struct m0_obj_attr * m0_io_attr(struct m0_op_io *ioo)
static int nw_xfer_req_dispatch(struct nw_xfer_request *xfer)
enum m0_pbuf_type ioo_pbuf_type
static void irfop_fini(struct ioreq_fop *irfop)
const struct m0_rpc_item_type * ri_type
static uint64_t layout_unit_size(const struct m0_pdclust_layout *play)
M0_INTERNAL int m0_buf_alloc(struct m0_buf *buf, size_t size)
struct m0_sm_group * sm_grp
m0_bcount_t crw_cksum_size
M0_INTERNAL uint32_t m0_fid_cob_device_id(const struct m0_fid *cob_fid)
M0_INTERNAL void m0_mutex_init(struct m0_mutex *mutex)
const struct m0_bob_type tioreq_bobtype
static void to_op_io_map(const struct m0_op *op, struct m0_op_io *ioo)
struct m0_0vec bb_zerovec
M0_INTERNAL void m0_bitmap_set(struct m0_bitmap *map, size_t idx, bool val)
static m0_bindex_t offset
struct m0_indexvec_varr dr_bufvec
struct m0_htable nxr_tioreqs_hash
M0_INTERNAL void m0_buf_free(struct m0_buf *buf)
struct m0_rpc_session * ti_session
static const struct m0_di_ops di_ops[M0_DI_NR]
M0_INTERNAL bool m0_ivec_cursor_move(struct m0_ivec_cursor *cur, m0_bcount_t count)
static uint64_t min64u(uint64_t a, uint64_t b)
struct m0_op_common oo_oc
static void page_pos_get(struct pargrp_iomap *map, m0_bindex_t index, uint32_t *row, uint32_t *col)
M0_INTERNAL bool m0_ext_is_in(const struct m0_ext *ext, m0_bindex_t index)
static uint64_t data_size(const struct m0_pdclust_layout *play)
static const struct nw_xfer_ops xfer_ops
M0_INTERNAL bool addr_is_network_aligned(void *addr)
M0_INTERNAL struct m0_op * m0__ioo_to_op(struct m0_op_io *ioo)
M0_INTERNAL m0_bcount_t m0_vec_count(const struct m0_vec *vec)
const struct target_ioreq_ops * ti_ops
static uint32_t layout_n(const struct m0_pdclust_layout *play)
static struct m0_pdclust_layout * pdlayout_get(const struct io_request *req)
static int64_t m0_atomic64_get(const struct m0_atomic64 *a)
const struct m0_bob_type nwxfer_bobtype
M0_INTERNAL uint32_t m0_pdclust_size(const struct m0_pdclust_layout *pl)
M0_INTERNAL int m0_rpc_session_validate(struct m0_rpc_session *session)
M0_INTERNAL size_t m0_rpc_bulk_buf_length(struct m0_rpc_bulk *rbulk)
struct m0_pdclust_tgt_addr tgt
static uint8_t fail[DATA_UNIT_COUNT_MAX+PARITY_UNIT_COUNT_MAX]
M0_INTERNAL int64_t m0_ref_read(const struct m0_ref *ref)
struct target_ioreq * irf_tioreq
m0_bcount_t ti_cksum_copied
#define M0_FI_ENABLED(tag)
M0_INTERNAL void ioreq_fop_fini(struct ioreq_fop *fop)
static struct m0_layout_instance * layout_instance(const struct io_request *req)
#define M0_ALLOC_PTR(ptr)
struct cc_req_fop ti_cc_fop
static bool should_unit_be_truncated(bool partial, enum m0_pdclust_unit_type unit_type, enum page_attr flags)
M0_HT_DESCR_DEFINE(tioreqht, "Hash of target_ioreq objects", M0_INTERNAL, struct target_ioreq, ti_link, ti_magic, M0_TIOREQ_MAGIC, M0_TLIST_HEAD_MAGIC, ti_fid.f_container, tioreqs_hash_func, tioreq_key_eq)
struct m0_bufvec dr_auxbufvec
enum nw_xfer_state nxr_state
static void m0_op_io_to_rpc_map(const struct m0_op_io *ioo, const struct m0_rpc_item *item)
static int nw_xfer_io_distribute(struct nw_xfer_request *xfer)
struct m0_rpc_item * m0_fop_to_rpc_item(const struct m0_fop *fop)
static void parity_page_pos_get(struct pargrp_iomap *map, m0_bindex_t index, uint32_t *row, uint32_t *col)
M0_INTERNAL bool m0_bitmap_get(const struct m0_bitmap *map, size_t idx)
M0_INTERNAL enum m0_pdclust_unit_type m0_pdclust_unit_classify(const struct m0_pdclust_layout *pl, int unit)
M0_INTERNAL m0_bindex_t m0_ivec_cursor_index(const struct m0_ivec_cursor *cur)
struct data_buf *** pi_databufs
M0_INTERNAL void m0_mutex_fini(struct m0_mutex *mutex)
struct m0_indexvec dr_ivec
void m0_fop_put_lock(struct m0_fop *fop)
struct m0_atomic64 nxr_iofop_nr
static struct m0_fop * fop
M0_INTERNAL int32_t m0_net_domain_get_max_buffer_segments(struct m0_net_domain *dom)
static struct m0 instance
M0_INTERNAL void m0_bitmap_reset(struct m0_bitmap *map)
static struct m0_be_seg * seg
static uint32_t ioreq_sm_state(const struct io_request *req)
static int target_ioreq_iofops_prepare(struct target_ioreq *ti, enum page_attr filter)
M0_INTERNAL void m0_ext_intersection(const struct m0_ext *e0, const struct m0_ext *e1, struct m0_ext *result)
M0_INTERNAL struct m0_fid target_fid(struct m0_op_io *ioo, struct m0_pdclust_tgt_addr *tgt)
struct nw_xfer_request * ti_nwxfer
M0_INTERNAL uint64_t m0__page_size(const struct m0_op_io *ioo)
M0_INTERNAL bool m0_is_read_fop(const struct m0_fop *fop)
static bool tioreq_key_eq(const void *key1, const void *key2)
M0_INTERNAL struct m0_fop_cob_rw * io_rw_get(struct m0_fop *fop)
static int32_t min32(int32_t a, int32_t b)
M0_INTERNAL bool m0_is_write_fop(const struct m0_fop *fop)
M0_INTERNAL bool m0_fid_is_valid(const struct m0_fid *fid)
void target_ioreq_fini(struct target_ioreq *ti)
M0_INTERNAL void m0_fd_fwd_map(struct m0_pdclust_instance *pi, const struct m0_pdclust_src_addr *src, struct m0_pdclust_tgt_addr *tgt)
struct m0_indexvec ti_goff_ivec
M0_INTERNAL int m0_io_fop_prepare(struct m0_fop *fop)
struct m0_bufvec ti_auxbufvec
struct m0_rpc_machine * ri_rmachine
static void m0_atomic64_add(struct m0_atomic64 *a, int64_t num)
M0_INTERNAL int m0_rpc_bulk_buf_add(struct m0_rpc_bulk *rbulk, uint32_t segs_nr, m0_bcount_t length, struct m0_net_domain *netdom, struct m0_net_buffer *nb, struct m0_rpc_bulk_buf **out)
M0_INTERNAL uint64_t m0_sm_id_get(const struct m0_sm *sm)
#define m0_tl_for(name, head, obj)
static const struct target_ioreq_ops tioreq_ops
struct m0_rpc_item f_item
struct m0_bufvec ioo_attr
struct m0_pdclust_src_addr src
struct dgmode_rwvec * ti_dgvec
static uint32_t io_desc_size(struct m0_net_domain *ndom)
struct m0_indexvec ti_ivec
#define offsetof(typ, memb)
M0_INTERNAL bool m0_sm_group_is_locked(const struct m0_sm_group *grp)
M0_INTERNAL void m0_poolmach_gob2cob(struct m0_poolmach *pm, const struct m0_fid *gfid, uint32_t idx, struct m0_fid *cob_fid)
struct m0_rpc_conn * s_conn
static uint64_t target_offset(uint64_t frame, struct m0_pdclust_layout *play, m0_bindex_t gob_offset)
int(* tio_iofops_prepare)(struct target_ioreq *ti, enum page_attr filter)
struct pargrp_iomap ** ioo_iomaps
const struct m0_di_ops * fi_di_ops
M0_INTERNAL void m0_file_init(struct m0_file *file, const struct m0_fid *fid, struct m0_rm_domain *dom, enum m0_di_types di_type)
static void m0_atomic64_set(struct m0_atomic64 *a, int64_t num)
M0_INTERNAL void * m0_extent_vec_get_checksum_addr(void *cksum_buf_vec, m0_bindex_t off, void *ivec, m0_bindex_t unit_sz, m0_bcount_t cs_sz)
#define m0_tl_forall(name, var, head,...)
static void target_ioreq_seg_add(struct target_ioreq *ti, const struct m0_pdclust_src_addr *src, const struct m0_pdclust_tgt_addr *tgt, m0_bindex_t gob_offset, m0_bcount_t count, struct pargrp_iomap *map)