32 #define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_SNS 35 #define BUF_ALLOC_ERR_INFO(ret, str, len) ({ \ 36 M0_ERR_INFO(ret, "Failed to allocate memory for " \ 37 str " buffer of length = %u", \ 41 #define VALUE_ASSERT_INFO(cond, var) ({ \ 42 M0_ASSERT_INFO(cond, "Invalid "#var" value. " \ 43 #var" = %u.", (uint32_t)var); \ 46 #define VALUE_MISMATCH_ASSERT_INFO(lhs, rhs) ({ \ 47 M0_ASSERT_INFO(lhs == rhs, #lhs" Value mismatch. " \ 48 #lhs" = %u, "#rhs"=%u", \ 49 (uint32_t)lhs, (uint32_t)rhs); \ 52 #define BLOCK_SIZE_ASSERT_INFO(blksz, index, block) ({ \ 53 VALUE_MISMATCH_ASSERT_INFO(blksz, block[index].b_nob); \ 56 #define MAT_INIT_ERR_INFO(ret, str, row, col) ({ \ 57 M0_ERR_INFO(ret, "Failed to initialize %ux%u " \ 58 str " matrix.", (uint32_t)row, \ 62 #define MATVEC_INIT_ERR_INFO(ret, str, size) ({ \ 63 M0_ERR_INFO(ret, "Failed to initialize " str " matrix " \ 64 "vector of size=%u", (uint32_t)size); \ 87 const uint32_t failure_index);
138 const uint32_t failure_index);
154 uint32_t failed_index);
225 uint32_t vec_idx, uint8_t *g_tbls,
226 uint32_t data_nr, uint32_t dest_nr);
238 uint32_t total_count, uint32_t parity_count);
259 uint32_t data_count, uint8_t *
fail,
311 const uint32_t fidx) = {
334 uint32_t data_count, uint32_t parity_count)
338 M0_ENTRY(
"data_count=%u parity_count=%u", data_count, parity_count);
348 if (parity_count == 1)
409 uint32_t data_ind_changed)
420 for (ei = 0; ei <
src[0].b_nob; ++ei)
430 M0_ENTRY(
"math=%p, local_nr=%u, ir=%p", math, local_nr, ir);
445 return M0_ERR_INFO(ret,
"Failed to initialize si_blocks");
454 uint32_t failed_index,
459 M0_ENTRY(
"recov_addr=%p, failed_index=%u, ir=%p",
460 recov_addr, failed_index, ir);
506 uint32_t failed_index,
515 M0_ENTRY(
"ir=%p, bufvec=%p, bitmap=%p, failed_index=%u, block_type=%u",
516 ir, bufvec, bitmap, failed_index, block_type);
537 switch (block_type) {
543 alive_block = &blocks[block_idx];
548 return M0_ERR_INFO(ret,
"Incremental recovery failed.");
555 &blocks[failed_index]))
583 for (
x = 0;
x < unit_count; ++
x)
603 uint32_t block_size =
data[0].b_nob;
611 for (ei = 0; ei < block_size; ++ei) {
637 for (ei = 0; ei <
new[
index].b_nob; ++ei) {
657 uint32_t block_size =
data[0].b_nob;
666 M0_PRE(fail_count <= math->pmi_parity_count);
672 for (ei = 0; ei < block_size; ++ei) {
677 data[ui].b_addr)[ei];
684 ((uint8_t*)
data[fail_index].
b_addr)[ei] = pe ^
695 const uint32_t failure_index)
700 uint32_t block_size =
data[0].b_nob;
711 for (ei = 0; ei < block_size; ++ei) {
714 if (ui != failure_index)
716 data[ui].b_addr)[ei];
718 if (ui != failure_index)
719 ((uint8_t*)
data[failure_index].
b_addr)[ei] = pe ^
730 const uint32_t failure_index)
737 uint32_t blocks_count;
747 for (
i = 0;
i < blocks_count; ++
i) {
753 return M0_ERR_INFO(ret,
"Failed to initialize bitmap " 754 "for si_blocks with index = %u",
i);
776 M0_ENTRY(
"y=%p, x=%p, alpha=%u", y,
x, (uint32_t)alpha);
797 y_addr[
i] =
gadd(y_addr[
i], x_addr[
i]);
817 uint32_t last_usable_bid;
825 for (
i = 0;
i <= last_usable_bid; ++
i) {
856 uint32_t total_count;
936 block_size =
data[0].b_nob;
963 struct m0_buf diff_data_buf;
964 uint8_t *diff_data_arr =
NULL;
966 uint32_t alignment =
sizeof(uint_fast32_t);
970 M0_ENTRY(
"math=%p, old=%p, new=%p, parity=%p, index=%u",
980 return M0_ERR_INFO(-EINVAL,
"Data block size mismatch. " 981 "Index=%u, Old data block size=%u, " 982 "New data block size=%u",
index,
986 block_size =
new[
index].b_nob;
993 "block_size=%u is not %u-bytes aligned",
994 block_size, alignment);
997 if (diff_data_arr ==
NULL)
1001 m0_buf_init(&diff_data_buf, diff_data_arr, block_size);
1004 for (
i = 0;
i < (block_size / alignment);
i++)
1005 ((uint_fast32_t *)diff_data_arr)[
i] =
1007 ((uint_fast32_t *)
new[
index].b_addr)[
i];
1025 uint32_t fail_count;
1026 uint32_t total_count;
1027 uint32_t block_size;
1033 M0_ENTRY(
"math=%p, data=%p, parity=%p, fails=%p",
1042 block_size =
data[0].b_nob;
1049 M0_ASSERT(fail_count <= math->pmi_parity_count);
1072 return M0_ERR_INFO(ret,
"failed to generate recovery " 1073 "coefficient tables");
1085 M0_ENTRY(
"math=%p, ir=%p", math, ir);
1100 M0_ENTRY(
"ir=%p, failed_index=%u", ir, failed_index);
1117 uint32_t total_blocks_nr;
1127 for (
i = 0, alive_nr = 0;
i < total_blocks_nr;
i++)
1135 return M0_ERR_INFO(ret,
"failed to generate decode matrix");
1156 M0_ENTRY(
"ir=%p, alive_block=%p", ir, alive_block);
1167 alive_bufvec = alive_block->
sib_addr;
1171 if (out_bufs ==
NULL)
1186 ret =
M0_ERR_INFO(ret,
"Failed to copy data from " 1187 "si_blocks[%u].sib_addr to " 1202 ret =
M0_ERR_INFO(-EINVAL,
"Failed to find alive block " 1203 "index %d in alive index array",
1218 ret =
M0_ERR_INFO(ret,
"Failed to copy data from " 1219 "alive_bufvec to source buffer.");
1228 ret =
M0_ERR_INFO(ret,
"Failed to recover the data.");
1237 ret =
M0_ERR_INFO(ret,
"Failed to copy data from " 1239 "si_blocks[%u].sib_addr.",
i,
1255 uint32_t vec_idx, uint8_t *g_tbls,
1256 uint32_t data_nr, uint32_t dest_nr)
1259 uint32_t block_size;
1260 uint8_t **dest_frags;
1262 M0_ENTRY(
"dest_bufs=%p, src_buf=%p, vec_idx=%u, " 1263 "g_tbls=%p, data_nr=%u, dest_nr=%u",
1264 dest_bufs, src_buf, vec_idx, g_tbls, data_nr, dest_nr);
1271 if (dest_frags ==
NULL)
1275 block_size = (uint32_t)src_buf->
b_nob;
1277 for (
i = 0;
i < dest_nr; ++
i) {
1279 dest_frags[
i] = (uint8_t *)dest_bufs[
i].
b_addr;
1282 ec_encode_data_update(block_size, data_nr, dest_nr, vec_idx,
1283 g_tbls, (uint8_t *)src_buf->
b_addr, dest_frags);
1290 uint32_t total_count, uint32_t parity_count)
1303 for (
i = 0, alive_nr = 0;
i < total_count;
i++) {
1315 uint32_t data_count, uint8_t *
fail,
1332 for (
i = 0, j = 0, k = 0;
i < total_count;
i++) {
1340 else if (k < data_count)
1351 uint32_t total_count;
1359 uint8_t *decode_mat =
NULL;
1360 uint8_t *temp_mat =
NULL;
1361 uint8_t *invert_mat =
NULL;
1363 M0_ENTRY(
"data_count=%u, parity_count=%u, rs=%p",
1364 data_count, parity_count, rs);
1368 total_count = data_count + parity_count;
1369 mat_size = total_count * data_count;
1372 if (decode_mat ==
NULL) {
1379 if (temp_mat ==
NULL) {
1386 if (invert_mat ==
NULL) {
1394 for (
i = 0;
i < data_count;
i++) {
1396 for (j = 0; j < data_count; j++)
1397 temp_mat[data_count *
i + j] =
1402 ret = gf_invert_matrix(temp_mat, invert_mat, data_count);
1404 ret =
M0_ERR_INFO(ret,
"failed to construct an %u x %u " 1405 "inverse of the input matrix",
1406 data_count, data_count);
1414 if (idx < data_count) {
1415 for (
i = 0;
i < data_count;
i++)
1416 decode_mat[data_count *
r +
i] =
1417 invert_mat[data_count * idx +
i];
1421 for (
i = 0;
i < data_count;
i++) {
1423 for (j = 0; j < data_count; j++)
1424 s ^= gf_mul(invert_mat[j * data_count +
i],
1427 decode_mat[data_count *
r +
i] =
s;
1519 #undef M0_TRACE_SUBSYSTEM static m0_bcount_t seg_size
enum m0_sns_ir_block_status sib_status
M0_INTERNAL int m0_bufvec_to_buf_copy(struct m0_buf *buf, const struct m0_bufvec *bvec)
struct m0_reed_solomon pmi_rs
static m0_parity_elem_t m0_parity_add(m0_parity_elem_t x, m0_parity_elem_t y)
M0_INTERNAL void m0_parity_math_fini(struct m0_parity_math *math)
#define M0_ALLOC_ARR(arr, nr)
static int gadd(int x, int y)
M0_INTERNAL int m0_bitmap_init(struct m0_bitmap *map, size_t nr)
static int(* diff[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
static bool is_usable(const struct m0_sns_ir *ir, const struct m0_bitmap *in_bmap, struct m0_sns_ir_block *failed_block)
static void dependency_bitmap_prepare(struct m0_sns_ir_block *f_block, struct m0_sns_ir *ir)
static int xor_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
M0_INTERNAL void m0_bitmap_fini(struct m0_bitmap *map)
enum m0_parity_cal_algo pmi_parity_algo
M0_INTERNAL int m0_buf_to_bufvec_copy(struct m0_bufvec *bvec, const struct m0_buf *buf)
static uint32_t ir_blocks_count(const struct m0_sns_ir *ir)
#define VALUE_ASSERT_INFO(cond, var)
static void(* calculate[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
M0_INTERNAL void m0_parity_math_buffer_xor(struct m0_buf *dest, const struct m0_buf *src)
M0_INTERNAL void m0_buf_init(struct m0_buf *buf, void *data, uint32_t nob)
uint8_t * rs_encode_matrix
static int reed_solomon_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
static int gmul(int x, int y)
struct m0_bitmap sib_bitmap
M0_INTERNAL void * m0_bufvec_cursor_addr(struct m0_bufvec_cursor *cur)
M0_INTERNAL size_t m0_bitmap_set_nr(const struct m0_bitmap *map)
static void fails_sort(struct m0_reed_solomon *rs, uint8_t *fail, uint32_t total_count, uint32_t parity_count)
M0_INTERNAL int m0_parity_math_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
M0_INTERNAL void m0_sns_ir_fini(struct m0_sns_ir *ir)
M0_INTERNAL int m0_parity_math_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
static int ir_recover(struct m0_sns_ir *ir, struct m0_sns_ir_block *alive_block)
struct m0_sns_ir_block * si_blocks
M0_INTERNAL int m0_sns_ir_failure_register(struct m0_bufvec *recov_addr, uint32_t failed_index, struct m0_sns_ir *ir)
M0_INTERNAL int m0_sns_ir_init(const struct m0_parity_math *math, uint32_t local_nr, struct m0_sns_ir *ir)
M0_INTERNAL void m0_parity_math_calculate(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity)
static int reed_solomon_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
static void fail_idx_reed_solomon_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t failure_index)
M0_INTERNAL bool m0_bufvec_cursor_move(struct m0_bufvec_cursor *cur, m0_bcount_t count)
M0_INTERNAL void m0_parity_math_refine(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, uint32_t data_ind_changed)
static uint32_t last_usable_block_id(const struct m0_sns_ir *ir, uint32_t block_idx)
struct m0_bufvec * sib_addr
static void reed_solomon_encode(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
#define M0_ERR_INFO(rc, fmt,...)
return M0_ERR(-EOPNOTSUPP)
static void xor_calculate(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
static int reed_solomon_init(struct m0_parity_math *math)
M0_INTERNAL m0_bcount_t m0_bufvec_cursor_step(const struct m0_bufvec_cursor *cur)
static bool m0_is_aligned(uint64_t val, uint64_t alignment)
M0_INTERNAL int m0_sns_ir_recover(struct m0_sns_ir *ir, struct m0_bufvec *bufvec, const struct m0_bitmap *bitmap, uint32_t failed_index, enum m0_sns_ir_block_type block_type)
M0_INTERNAL void m0_bufvec_cursor_init(struct m0_bufvec_cursor *cur, const struct m0_bufvec *bvec)
M0_INTERNAL int m0_sns_ir_mat_compute(struct m0_sns_ir *ir)
M0_INTERNAL int m0_parity_math_init(struct m0_parity_math *math, uint32_t data_count, uint32_t parity_count)
M0_INTERNAL int m0_buf_alloc(struct m0_buf *buf, size_t size)
struct m0_reed_solomon si_rs
M0_INTERNAL void m0_bitmap_set(struct m0_bitmap *map, size_t idx, bool val)
static bool is_valid_block_idx(const struct m0_sns_ir *ir, uint32_t block_idx)
M0_INTERNAL void m0_buf_free(struct m0_buf *buf)
M0_INTERNAL int m0_bitmap_ffs(const struct m0_bitmap *map)
#define BUF_ALLOC_ERR_INFO(ret, str, len)
M0_INTERNAL m0_bcount_t m0_vec_count(const struct m0_vec *vec)
#define m0_forall(var, nr,...)
static uint8_t fail[DATA_UNIT_COUNT_MAX+PARITY_UNIT_COUNT_MAX]
static uint32_t fails_count(uint8_t *fail, uint32_t unit_count)
static int(* recover[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
static void gfaxpy(struct m0_bufvec *y, struct m0_bufvec *x, m0_parity_elem_t alpha)
M0_INTERNAL bool m0_bitmap_get(const struct m0_bitmap *map, size_t idx)
static void ir_rs_init(const struct m0_parity_math *math, struct m0_sns_ir *ir)
static void(* fidx_recover[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t fidx)
static int isal_gen_recov_coeff_tbl(uint32_t data_count, uint32_t parity_count, struct m0_reed_solomon *rs)
#define VALUE_MISMATCH_ASSERT_INFO(lhs, rhs)
#define M0_PARITY_GALOIS_W
static void fail_idx_xor_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t failure_index)
#define M0_ASSERT_INFO(cond, fmt,...)
static int isal_encode_data_update(struct m0_buf *dest_bufs, struct m0_buf *src_buf, uint32_t vec_idx, uint8_t *g_tbls, uint32_t data_nr, uint32_t dest_nr)
static void reed_solomon_fini(struct m0_parity_math *math)
static m0_parity_elem_t m0_parity_mul(m0_parity_elem_t x, m0_parity_elem_t y)
static struct m0_addb2_source * s
M0_INTERNAL void m0_parity_math_fail_index_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t fidx)
static void buf_sort(struct m0_reed_solomon *rs, uint32_t total_count, uint32_t data_count, uint8_t *fail, struct m0_buf *data, struct m0_buf *parity)
struct m0_pdclust_src_addr src
static int ir_si_blocks_init(struct m0_sns_ir *ir)
static void ir_failure_register(struct m0_sns_ir *ir, uint32_t failed_index)
static bool parity_math_invariant(const struct m0_parity_math *math)
uint32_t pmi_parity_count
#define BLOCK_SIZE_ASSERT_INFO(blksz, index, block)
static uint8_t parity[DATA_UNIT_COUNT_MAX][UNIT_BUFF_SIZE_MAX]
static int xor_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
static int ir_mat_compute(struct m0_sns_ir *ir)