Motr  M0
parity_math.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013-2021 Seagate Technology LLC and/or its Affiliates
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *
16  * For any questions about this software or licensing,
17  * please email opensource@seagate.com or cortx-questions@seagate.com.
18  *
19  */
20 
21 
22 #include "lib/arith.h"
23 #include "lib/assert.h"
24 #include "lib/errno.h"
25 #include "lib/memory.h"
26 #include "lib/misc.h" /* SET0() */
27 #include "lib/types.h"
28 
29 #include "sns/parity_ops.h"
30 #include "sns/parity_math.h"
31 
32 #define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_SNS
33 #include "lib/trace.h"
34 
35 #define BUF_ALLOC_ERR_INFO(ret, str, len) ({ \
36  M0_ERR_INFO(ret, "Failed to allocate memory for " \
37  str " buffer of length = %u", \
38  (uint32_t)len); \
39 })
40 
41 #define VALUE_ASSERT_INFO(cond, var) ({ \
42  M0_ASSERT_INFO(cond, "Invalid "#var" value. " \
43  #var" = %u.", (uint32_t)var); \
44 })
45 
46 #define VALUE_MISMATCH_ASSERT_INFO(lhs, rhs) ({ \
47  M0_ASSERT_INFO(lhs == rhs, #lhs" Value mismatch. " \
48  #lhs" = %u, "#rhs"=%u", \
49  (uint32_t)lhs, (uint32_t)rhs); \
50 })
51 
52 #define BLOCK_SIZE_ASSERT_INFO(blksz, index, block) ({ \
53  VALUE_MISMATCH_ASSERT_INFO(blksz, block[index].b_nob); \
54 })
55 
56 #define MAT_INIT_ERR_INFO(ret, str, row, col) ({ \
57  M0_ERR_INFO(ret, "Failed to initialize %ux%u " \
58  str " matrix.", (uint32_t)row, \
59  (uint32_t)col); \
60 })
61 
62 #define MATVEC_INIT_ERR_INFO(ret, str, size) ({ \
63  M0_ERR_INFO(ret, "Failed to initialize " str " matrix " \
64  "vector of size=%u", (uint32_t)size); \
65 })
66 
67 /* Forward declarations. */
68 static void xor_calculate(struct m0_parity_math *math,
69  const struct m0_buf *data,
70  struct m0_buf *parity);
71 
72 static int xor_diff(struct m0_parity_math *math,
73  struct m0_buf *old,
74  struct m0_buf *new,
75  struct m0_buf *parity,
76  uint32_t index);
77 
78 static int xor_recover(struct m0_parity_math *math,
79  struct m0_buf *data,
80  struct m0_buf *parity,
81  struct m0_buf *fails,
82  enum m0_parity_linsys_algo algo);
83 
84 static void fail_idx_xor_recover(struct m0_parity_math *math,
85  struct m0_buf *data,
86  struct m0_buf *parity,
87  const uint32_t failure_index);
88 
89 /* Below are some functions which have separate implementations for user
90  * space and kernel space. Intel ISA library is used in user space only. Hence
91  * implementation of these functions in kernel space are empty for now and will
92  * be removed once kernel space compilation is removed.
93  */
94 
98 static int reed_solomon_init(struct m0_parity_math *math);
99 
103 static void reed_solomon_fini(struct m0_parity_math *math);
104 
108 static void reed_solomon_encode(struct m0_parity_math *math,
109  const struct m0_buf *data,
110  struct m0_buf *parity);
111 
115 static int reed_solomon_diff(struct m0_parity_math *math,
116  struct m0_buf *old,
117  struct m0_buf *new,
118  struct m0_buf *parity,
119  uint32_t index);
120 
125 static int reed_solomon_recover(struct m0_parity_math *math,
126  struct m0_buf *data,
127  struct m0_buf *parity,
128  struct m0_buf *fails,
129  enum m0_parity_linsys_algo algo);
130 
135 static void fail_idx_reed_solomon_recover(struct m0_parity_math *math,
136  struct m0_buf *data,
137  struct m0_buf *parity,
138  const uint32_t failure_index);
139 
146 static void ir_rs_init(const struct m0_parity_math *math, struct m0_sns_ir *ir);
147 
153 static void ir_failure_register(struct m0_sns_ir *ir,
154  uint32_t failed_index);
155 
164 static int ir_mat_compute(struct m0_sns_ir *ir);
165 
172 static int ir_recover(struct m0_sns_ir *ir, struct m0_sns_ir_block *alive_block);
173 
177 static uint32_t last_usable_block_id(const struct m0_sns_ir *ir,
178  uint32_t block_idx);
179 
180 static bool parity_math_invariant(const struct m0_parity_math *math);
181 
182 /* Counts number of failed blocks. */
183 static uint32_t fails_count(uint8_t *fail, uint32_t unit_count);
184 
185 static inline uint32_t ir_blocks_count(const struct m0_sns_ir *ir);
186 
191 static int ir_si_blocks_init(struct m0_sns_ir *ir);
192 
197 static void gfaxpy(struct m0_bufvec *y, struct m0_bufvec *x,
198  m0_parity_elem_t alpha);
199 
200 static bool is_usable(const struct m0_sns_ir *ir,
201  const struct m0_bitmap *in_bmap,
202  struct m0_sns_ir_block *failed_block);
203 
204 static inline bool is_valid_block_idx(const struct m0_sns_ir *ir,
205  uint32_t block_idx);
206 
207 #ifndef __KERNEL__
208 
224 static int isal_encode_data_update(struct m0_buf *dest_bufs, struct m0_buf *src_buf,
225  uint32_t vec_idx, uint8_t *g_tbls,
226  uint32_t data_nr, uint32_t dest_nr);
227 
237 static void fails_sort(struct m0_reed_solomon *rs, uint8_t *fail,
238  uint32_t total_count, uint32_t parity_count);
239 
258 static void buf_sort(struct m0_reed_solomon *rs, uint32_t total_count,
259  uint32_t data_count, uint8_t *fail,
260  struct m0_buf *data, struct m0_buf *parity);
261 
270 static int isal_gen_recov_coeff_tbl(uint32_t data_count, uint32_t parity_count,
271  struct m0_reed_solomon *rs);
272 
279 static void dependency_bitmap_prepare(struct m0_sns_ir_block *f_block,
280  struct m0_sns_ir *ir);
281 #endif /* __KERNEL__ */
282 
283 static void (*calculate[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math,
284  const struct m0_buf *data,
285  struct m0_buf *parity) = {
288 };
289 
291  struct m0_buf *old,
292  struct m0_buf *new,
293  struct m0_buf *parity,
294  uint32_t index) = {
297 };
298 
300  struct m0_buf *data,
301  struct m0_buf *parity,
302  struct m0_buf *fails,
303  enum m0_parity_linsys_algo algo) = {
306 };
307 
308 static void (*fidx_recover[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math,
309  struct m0_buf *data,
310  struct m0_buf *parity,
311  const uint32_t fidx) = {
314 };
315 
316 enum {
321 };
322 
323 /* Parity Math Functions. */
324 
325 M0_INTERNAL void m0_parity_math_fini(struct m0_parity_math *math)
326 {
327  M0_ENTRY();
329  reed_solomon_fini(math);
330  M0_LEAVE();
331 }
332 
333 M0_INTERNAL int m0_parity_math_init(struct m0_parity_math *math,
334  uint32_t data_count, uint32_t parity_count)
335 {
336  int ret = 0;
337 
338  M0_ENTRY("data_count=%u parity_count=%u", data_count, parity_count);
339 
340  M0_PRE(math != NULL);
341 
342  M0_SET0(math);
343  math->pmi_data_count = data_count;
344  math->pmi_parity_count = parity_count;
345 
347 
348  if (parity_count == 1)
350  else {
352  ret = reed_solomon_init(math);
353  if (ret != 0) {
354  m0_parity_math_fini(math);
355  return M0_ERR(ret);
356  }
357  }
358 
359  return M0_RC(ret);
360 }
361 
362 M0_INTERNAL void m0_parity_math_calculate(struct m0_parity_math *math,
363  struct m0_buf *data,
364  struct m0_buf *parity)
365 {
366  M0_ENTRY();
367  (*calculate[math->pmi_parity_algo])(math, data, parity);
368  M0_LEAVE();
369 }
370 
371 M0_INTERNAL int m0_parity_math_diff(struct m0_parity_math *math,
372  struct m0_buf *old,
373  struct m0_buf *new,
374  struct m0_buf *parity, uint32_t index)
375 {
376  int rc;
377 
378  M0_ENTRY();
379  rc = (*diff[math->pmi_parity_algo])(math, old, new, parity, index);
380  return rc == 0 ? M0_RC(rc) : M0_ERR(rc);
381 }
382 
383 M0_INTERNAL int m0_parity_math_recover(struct m0_parity_math *math,
384  struct m0_buf *data,
385  struct m0_buf *parity,
386  struct m0_buf *fails,
387  enum m0_parity_linsys_algo algo)
388 {
389  int rc;
390 
391  M0_ENTRY();
392  rc = (*recover[math->pmi_parity_algo])(math, data, parity, fails, algo);
393  return rc == 0 ? M0_RC(rc) : M0_ERR(rc);
394 }
395 
396 M0_INTERNAL void m0_parity_math_fail_index_recover(struct m0_parity_math *math,
397  struct m0_buf *data,
398  struct m0_buf *parity,
399  const uint32_t fidx)
400 {
401  M0_ENTRY();
402  (*fidx_recover[math->pmi_parity_algo])(math, data, parity, fidx);
403  M0_LEAVE();
404 }
405 
406 M0_INTERNAL void m0_parity_math_refine(struct m0_parity_math *math,
407  struct m0_buf *data,
408  struct m0_buf *parity,
409  uint32_t data_ind_changed)
410 {
411  /* for simplicity: */
413 }
414 
415 M0_INTERNAL void m0_parity_math_buffer_xor(struct m0_buf *dest,
416  const struct m0_buf *src)
417 {
418  uint32_t ei; /* block element index. */
419 
420  for (ei = 0; ei < src[0].b_nob; ++ei)
421  ((uint8_t*)dest[0].b_addr)[ei] ^=
422  (m0_parity_elem_t)((uint8_t*)src[0].b_addr)[ei];
423 }
424 
425 M0_INTERNAL int m0_sns_ir_init(const struct m0_parity_math *math,
426  uint32_t local_nr, struct m0_sns_ir *ir)
427 {
428  int ret;
429 
430  M0_ENTRY("math=%p, local_nr=%u, ir=%p", math, local_nr, ir);
431 
432  M0_PRE(math != NULL);
433  M0_PRE(ir != NULL);
434 
435  M0_SET0(ir);
436 
437  ir->si_data_nr = math->pmi_data_count;
438  ir->si_parity_nr = math->pmi_parity_count;
439  ir->si_local_nr = local_nr;
440  ir->si_alive_nr = ir_blocks_count(ir);
441 
442  ret = ir_si_blocks_init(ir);
443  if (ret != 0) {
444  m0_sns_ir_fini(ir);
445  return M0_ERR_INFO(ret, "Failed to initialize si_blocks");
446  }
447 
448  ir_rs_init(math, ir);
449 
450  return M0_RC(ret);
451 }
452 
453 M0_INTERNAL int m0_sns_ir_failure_register(struct m0_bufvec *recov_addr,
454  uint32_t failed_index,
455  struct m0_sns_ir *ir)
456 {
457  struct m0_sns_ir_block *block;
458 
459  M0_ENTRY("recov_addr=%p, failed_index=%u, ir=%p",
460  recov_addr, failed_index, ir);
461 
462  M0_PRE(ir != NULL);
463  M0_PRE(ir->si_blocks != NULL);
464  M0_PRE(recov_addr != NULL);
465  M0_PRE(is_valid_block_idx(ir, failed_index));
466 
467  block = &ir->si_blocks[failed_index];
469 
470  block->sib_addr = recov_addr;
472 
473  M0_CNT_DEC(ir->si_alive_nr);
474  if (ir->si_alive_nr < ir->si_data_nr)
475  return M0_ERR(-ERANGE);
476 
477  ir_failure_register(ir, failed_index);
478 
479  return M0_RC(0);
480 }
481 
482 M0_INTERNAL int m0_sns_ir_mat_compute(struct m0_sns_ir *ir)
483 {
484  M0_PRE(ir != NULL);
485  return M0_RC(ir_mat_compute(ir));
486 }
487 
488 M0_INTERNAL void m0_sns_ir_fini(struct m0_sns_ir *ir)
489 {
490  uint32_t i;
491 
492  M0_ENTRY("ir=%p", ir);
493  M0_PRE(ir != NULL);
494 
495  for (i = 0; i < ir_blocks_count(ir); ++i)
496  if (ir->si_blocks[i].sib_bitmap.b_words != NULL)
498 
499  m0_free(ir->si_blocks);
500  M0_LEAVE();
501 }
502 
503 M0_INTERNAL int m0_sns_ir_recover(struct m0_sns_ir *ir,
504  struct m0_bufvec *bufvec,
505  const struct m0_bitmap *bitmap,
506  uint32_t failed_index,
507  enum m0_sns_ir_block_type block_type)
508 {
509  int block_idx = 0;
510  size_t b_set_nr;
511  struct m0_sns_ir_block *blocks;
512  struct m0_sns_ir_block *alive_block;
513  int ret;
514 
515  M0_ENTRY("ir=%p, bufvec=%p, bitmap=%p, failed_index=%u, block_type=%u",
516  ir, bufvec, bitmap, failed_index, block_type);
517 
518  M0_PRE(ir != NULL && bufvec != NULL && bitmap != NULL);
519 
520  b_set_nr = m0_bitmap_set_nr(bitmap);
521  M0_PRE(b_set_nr > 0);
522  M0_PRE(ergo(b_set_nr > 1, block_type == M0_SI_BLOCK_REMOTE));
523  M0_PRE(ergo(block_type == M0_SI_BLOCK_REMOTE,
524  is_valid_block_idx(ir, failed_index)));
525  M0_PRE(ergo(block_type == M0_SI_BLOCK_REMOTE,
526  ir->si_blocks[failed_index].sib_status ==
528  M0_PRE(ergo(block_type == M0_SI_BLOCK_LOCAL, ir->si_local_nr != 0));
529 
530  if (b_set_nr == 1)
531  block_idx = m0_bitmap_ffs(bitmap);
532 
533  VALUE_ASSERT_INFO(is_valid_block_idx(ir, block_idx), block_idx);
534 
535  blocks = ir->si_blocks;
536 
537  switch (block_type) {
538  /* Input block is assumed to be an untransformed block, and is used for
539  * recovering all failed blocks */
540  case M0_SI_BLOCK_LOCAL:
541 
542  M0_CNT_DEC(ir->si_local_nr);
543  alive_block = &blocks[block_idx];
544  alive_block->sib_addr = bufvec;
545 
546  ret = ir_recover(ir, alive_block);
547  if (ret != 0)
548  return M0_ERR_INFO(ret, "Incremental recovery failed.");
549  break;
550 
551  /* Input block is assumed to be a transformed block, and is
552  * cumulatively added to a block with index failed_index. */
553  case M0_SI_BLOCK_REMOTE:
554  if (!is_usable(ir, (struct m0_bitmap*) bitmap,
555  &blocks[failed_index]))
556  break;
557  gfaxpy(blocks[failed_index].sib_addr, bufvec,
558  1);
559  break;
560  }
561 
562  return M0_RC(0);
563 }
564 
565 /* Parity Math Helper Functions */
566 
567 /* Functions m0_parity_* are too much eclectic. Just more simple names. */
568 static int gadd(int x, int y)
569 {
570  return m0_parity_add(x, y);
571 }
572 
573 static int gmul(int x, int y)
574 {
575  return m0_parity_mul(x, y);
576 }
577 
578 static uint32_t fails_count(uint8_t *fail, uint32_t unit_count)
579 {
580  uint32_t x;
581  uint32_t count = 0;
582 
583  for (x = 0; x < unit_count; ++x)
584  count += !!fail[x];
585 
586  return count;
587 }
588 
589 static bool parity_math_invariant(const struct m0_parity_math *math)
590 {
591  return _0C(math != NULL) && _0C(math->pmi_data_count >= 1) &&
592  _0C(math->pmi_parity_count >= 1) &&
593  _0C(math->pmi_data_count >= math->pmi_parity_count) &&
595 }
596 
597 static void xor_calculate(struct m0_parity_math *math,
598  const struct m0_buf *data,
599  struct m0_buf *parity)
600 {
601  uint32_t ei; /* block element index. */
602  uint32_t ui; /* unit index. */
603  uint32_t block_size = data[0].b_nob;
604  m0_parity_elem_t pe;
605 
606  M0_ENTRY();
607  M0_PRE(block_size == parity[0].b_nob);
608  for (ui = 1; ui < math->pmi_data_count; ++ui)
609  M0_PRE(block_size == data[ui].b_nob);
610 
611  for (ei = 0; ei < block_size; ++ei) {
612  pe = 0;
613  for (ui = 0; ui < math->pmi_data_count; ++ui)
614  pe ^= (m0_parity_elem_t)((uint8_t*)data[ui].b_addr)[ei];
615 
616  ((uint8_t*)parity[0].b_addr)[ei] = pe;
617  }
618  M0_LEAVE();
619 }
620 
621 static int xor_diff(struct m0_parity_math *math,
622  struct m0_buf *old,
623  struct m0_buf *new,
624  struct m0_buf *parity,
625  uint32_t index)
626 {
627  uint32_t ei;
628 
629  M0_PRE(math != NULL);
630  M0_PRE(old != NULL);
631  M0_PRE(new != NULL);
632  M0_PRE(parity != NULL);
633  M0_PRE(index < math->pmi_data_count);
634  M0_PRE(old[index].b_nob == new[index].b_nob);
635  M0_PRE(new[index].b_nob == parity[0].b_nob);
636 
637  for (ei = 0; ei < new[index].b_nob; ++ei) {
638  ((uint8_t*)parity[0].b_addr)[ei] ^=
639  ((uint8_t *)old[index].b_addr)[ei] ^
640  ((uint8_t *)new[index].b_addr)[ei];
641  }
642 
643  return M0_RC(0);
644 }
645 
646 static int xor_recover(struct m0_parity_math *math,
647  struct m0_buf *data,
648  struct m0_buf *parity,
649  struct m0_buf *fails,
650  enum m0_parity_linsys_algo algo)
651 {
652  uint32_t ei; /* block element index. */
653  uint32_t ui; /* unit index. */
654  uint8_t *fail;
655  uint32_t fail_count;
656  uint32_t unit_count;
657  uint32_t block_size = data[0].b_nob;
658  m0_parity_elem_t pe;
659  int fail_index = BAD_FAIL_INDEX;
660 
661  unit_count = math->pmi_data_count + math->pmi_parity_count;
662  fail = (uint8_t*) fails->b_addr;
663  fail_count = fails_count(fail, unit_count);
664 
665  M0_PRE(fail_count == 1);
666  M0_PRE(fail_count <= math->pmi_parity_count);
667  M0_PRE(block_size == parity[0].b_nob);
668 
669  for (ui = 1; ui < math->pmi_data_count; ++ui)
670  M0_PRE(block_size == data[ui].b_nob);
671 
672  for (ei = 0; ei < block_size; ++ei) {
673  pe = 0;
674  for (ui = 0; ui < math->pmi_data_count; ++ui) {
675  if (fail[ui] != 1)
676  pe ^= (m0_parity_elem_t)((uint8_t*)
677  data[ui].b_addr)[ei];
678  else
679  fail_index = ui;
680  }
681  /* Now ui points to parity block, test if it was failed. */
682  if (fail[ui] != 1) {
683  M0_ASSERT(fail_index != BAD_FAIL_INDEX);
684  ((uint8_t*)data[fail_index].b_addr)[ei] = pe ^
685  ((uint8_t*)parity[0].b_addr)[ei];
686  } else /* Parity was lost, so recover it. */
687  ((uint8_t*)parity[0].b_addr)[ei] = pe;
688  }
689  return M0_RC(0);
690 }
691 
692 static void fail_idx_xor_recover(struct m0_parity_math *math,
693  struct m0_buf *data,
694  struct m0_buf *parity,
695  const uint32_t failure_index)
696 {
697  uint32_t ei; /* block element index. */
698  uint32_t ui; /* unit index. */
699  uint32_t unit_count;
700  uint32_t block_size = data[0].b_nob;
701  m0_parity_elem_t pe;
702 
703  M0_PRE(block_size == parity[0].b_nob);
704 
705  unit_count = math->pmi_data_count + math->pmi_parity_count;
706  M0_ASSERT(failure_index < unit_count);
707 
708  for (ui = 1; ui < math->pmi_data_count; ++ui)
709  M0_ASSERT(block_size == data[ui].b_nob);
710 
711  for (ei = 0; ei < block_size; ++ei) {
712  pe = 0;
713  for (ui = 0; ui < math->pmi_data_count; ++ui)
714  if (ui != failure_index)
715  pe ^= (m0_parity_elem_t)((uint8_t*)
716  data[ui].b_addr)[ei];
717 
718  if (ui != failure_index)
719  ((uint8_t*)data[failure_index].b_addr)[ei] = pe ^
720  ((uint8_t*)parity[0].b_addr)[ei];
721  else /* Parity was lost, so recover it. */
722  ((uint8_t*)parity[0].b_addr)[ei] = pe;
723  }
724 }
725 
728  struct m0_buf *data,
729  struct m0_buf *parity,
730  const uint32_t failure_index)
731 {
732 }
733 
734 static int ir_si_blocks_init(struct m0_sns_ir *ir)
735 {
736  uint32_t i;
737  uint32_t blocks_count;
738  int ret = 0;
739 
740  M0_PRE(ir != NULL);
741 
742  blocks_count = ir_blocks_count(ir);
743  M0_ALLOC_ARR(ir->si_blocks, blocks_count);
744  if (ir->si_blocks == NULL)
745  return BUF_ALLOC_ERR_INFO(-ENOMEM, "ir blocks", blocks_count);
746 
747  for (i = 0; i < blocks_count; ++i) {
748  ir->si_blocks[i].sib_idx = i;
750  ret = m0_bitmap_init(&ir->si_blocks[i].sib_bitmap,
751  blocks_count);
752  if (ret != 0)
753  return M0_ERR_INFO(ret, "Failed to initialize bitmap "
754  "for si_blocks with index = %u", i);
755  }
756 
757  return ret;
758 }
759 
760 static inline uint32_t ir_blocks_count(const struct m0_sns_ir *ir)
761 {
762  return ir->si_data_nr + ir->si_parity_nr;
763 }
764 
765 static void gfaxpy(struct m0_bufvec *y, struct m0_bufvec *x,
766  m0_parity_elem_t alpha)
767 {
768  uint32_t i;
769  uint32_t seg_size;
770  uint8_t *y_addr;
771  uint8_t *x_addr;
772  m0_bcount_t step;
773  struct m0_bufvec_cursor x_cursor;
774  struct m0_bufvec_cursor y_cursor;
775 
776  M0_ENTRY("y=%p, x=%p, alpha=%u", y, x, (uint32_t)alpha);
777 
778  M0_PRE(y != NULL && x != NULL);
779  M0_PRE(y->ov_vec.v_nr != 0);
780  M0_PRE(y->ov_vec.v_nr == x->ov_vec.v_nr);
781  M0_PRE(y->ov_vec.v_count[0] != 0);
782  M0_PRE(y->ov_vec.v_count[0] == x->ov_vec.v_count[0]);
783  seg_size = y->ov_vec.v_count[0];
785 
786  m0_bufvec_cursor_init(&y_cursor, y);
787  m0_bufvec_cursor_init(&x_cursor, x);
788  do {
789  x_addr = m0_bufvec_cursor_addr(&x_cursor);
790  y_addr = m0_bufvec_cursor_addr(&y_cursor);
791 
792  switch (alpha) {
793  /* This is a special case of the 'default' case that follows.
794  * Here we avoid unnecessary call to gmul.*/
795  case 1:
796  for (i = 0; i <seg_size; ++i)
797  y_addr[i] = gadd(y_addr[i], x_addr[i]);
798  break;
799  default:
800  for (i = 0; i < seg_size; ++i)
801  y_addr[i] = gadd(y_addr[i], gmul(x_addr[i],
802  alpha));
803  break;
804  }
805  step = m0_bufvec_cursor_step(&y_cursor);
806  } while (!m0_bufvec_cursor_move(&x_cursor, step) &&
807  !m0_bufvec_cursor_move(&y_cursor, step));
808 
809  M0_LEAVE();
810 }
811 
812 static bool is_usable(const struct m0_sns_ir *ir,
813  const struct m0_bitmap *in_bmap,
814  struct m0_sns_ir_block *failed_block)
815 {
816  size_t i;
817  uint32_t last_usable_bid;
818 
819  M0_PRE(in_bmap != NULL && failed_block != NULL && ir != NULL);
820  M0_PRE(in_bmap->b_nr == failed_block->sib_bitmap.b_nr);
821 
822  last_usable_bid = last_usable_block_id(ir, failed_block->sib_idx);
823  if (last_usable_bid == ir_blocks_count(ir))
824  return false;
825  for (i = 0; i <= last_usable_bid; ++i) {
826  if (m0_bitmap_get(in_bmap, i) &&
827  !m0_bitmap_get(&failed_block->sib_bitmap, i))
828  return false;
829  }
830  return true;
831 }
832 
833 static inline bool is_valid_block_idx(const struct m0_sns_ir *ir,
834  uint32_t block_idx)
835 {
836  return block_idx < ir_blocks_count(ir);
837 }
838 
839 #ifndef __KERNEL__
840 static void reed_solomon_fini(struct m0_parity_math *math)
841 {
842  M0_ENTRY();
846  m0_free(math->pmi_rs.rs_alive_idx);
848  m0_free(math->pmi_rs.rs_bufs_in);
849  m0_free(math->pmi_rs.rs_bufs_out);
850  M0_LEAVE();
851 }
852 
853 static int reed_solomon_init(struct m0_parity_math *math)
854 {
855  struct m0_reed_solomon *rs;
856  uint32_t total_count;
857  uint32_t tbl_len;
858  int ret = 0;
859 
860  M0_ENTRY("math=%p", math);
861 
863 
864  rs = &math->pmi_rs;
865  total_count = math->pmi_data_count + math->pmi_parity_count;
866  /* The encode and decode tables are the expanded tables needed for
867  * fast encode or decode for erasure codes on blocks of data. 32bytes
868  * are generated for each input coefficient. Hence total table length
869  * required is 32 * data_count * parity_count.
870  */
871  tbl_len = math->pmi_data_count * math->pmi_parity_count * MIN_TABLE_LEN;
872 
873  M0_ALLOC_ARR(rs->rs_encode_matrix, (total_count * math->pmi_data_count));
874  if (rs->rs_encode_matrix == NULL)
875  return BUF_ALLOC_ERR_INFO(-ENOMEM, "encode matrix",
876  (total_count * math->pmi_data_count));
877 
878  M0_ALLOC_ARR(rs->rs_encode_tbls, tbl_len);
879  if (rs->rs_encode_tbls == NULL)
880  return BUF_ALLOC_ERR_INFO(-ENOMEM, "encode coefficient tables",
881  tbl_len);
882 
883  M0_ALLOC_ARR(rs->rs_decode_tbls, tbl_len);
884  if (rs->rs_decode_tbls == NULL)
885  return BUF_ALLOC_ERR_INFO(-ENOMEM, "decode coefficient tables",
886  tbl_len);
887 
888  /* Generate a matrix of coefficients to be used for encoding. */
889  gf_gen_rs_matrix(rs->rs_encode_matrix, total_count,
890  math->pmi_data_count);
891 
892  /* Initialize tables for fast Erasure Code encode. */
893  ec_init_tables(math->pmi_data_count, math->pmi_parity_count,
894  &rs->rs_encode_matrix[math->pmi_data_count *
895  math->pmi_data_count],
896  rs->rs_encode_tbls);
897 
899  if (rs->rs_failed_idx == NULL)
900  return BUF_ALLOC_ERR_INFO(-ENOMEM, "failed index",
901  math->pmi_parity_count);
902 
903  M0_ALLOC_ARR(rs->rs_alive_idx, total_count);
904  if (rs->rs_alive_idx == NULL)
905  return BUF_ALLOC_ERR_INFO(-ENOMEM, "alive index",
906  total_count);
907 
909  if (rs->rs_bufs_in == NULL)
910  return BUF_ALLOC_ERR_INFO(-ENOMEM, "input buffers array",
911  math->pmi_data_count);
912 
914  if (rs->rs_bufs_out == NULL)
915  return BUF_ALLOC_ERR_INFO(-ENOMEM, "output buffers array",
916  math->pmi_parity_count);
917 
918  return M0_RC(ret);
919 }
920 
921 static void reed_solomon_encode(struct m0_parity_math *math,
922  const struct m0_buf *data,
923  struct m0_buf *parity)
924 {
925  uint32_t i;
926  uint32_t block_size;
927  struct m0_reed_solomon *rs;
928 
929  M0_ENTRY("math=%p, data=%p, parity=%p", math, data, parity);
930 
932  M0_PRE(data != NULL);
933  M0_PRE(parity != NULL);
934 
935  rs = &math->pmi_rs;
936  block_size = data[0].b_nob;
937 
938  rs->rs_bufs_in[0] = (uint8_t *)data[0].b_addr;
939  for (i = 1; i < math->pmi_data_count; ++i) {
940  BLOCK_SIZE_ASSERT_INFO(block_size, i, data);
941  rs->rs_bufs_in[i] = (uint8_t *)data[i].b_addr;
942  }
943 
944  for (i = 0; i < math->pmi_parity_count; ++i) {
945  BLOCK_SIZE_ASSERT_INFO(block_size, i, parity);
946  rs->rs_bufs_out[i] = (uint8_t *)parity[i].b_addr;
947  }
948 
949  /* Generate erasure codes on given blocks of data. */
950  ec_encode_data(block_size, math->pmi_data_count,
951  math->pmi_parity_count, rs->rs_encode_tbls,
952  rs->rs_bufs_in, rs->rs_bufs_out);
953 
954  M0_LEAVE();
955 }
956 
957 static int reed_solomon_diff(struct m0_parity_math *math,
958  struct m0_buf *old,
959  struct m0_buf *new,
960  struct m0_buf *parity,
961  uint32_t index)
962 {
963  struct m0_buf diff_data_buf;
964  uint8_t *diff_data_arr = NULL;
965  uint32_t block_size;
966  uint32_t alignment = sizeof(uint_fast32_t);
967  uint32_t i;
968  int ret = 0;
969 
970  M0_ENTRY("math=%p, old=%p, new=%p, parity=%p, index=%u",
971  math, old, new, parity, index);
972 
974  M0_PRE(old != NULL);
975  M0_PRE(new != NULL);
976  M0_PRE(parity != NULL);
977  M0_PRE(index < math->pmi_data_count);
978 
979  if (old[index].b_nob != new[index].b_nob)
980  return M0_ERR_INFO(-EINVAL, "Data block size mismatch. "
981  "Index=%u, Old data block size=%u, "
982  "New data block size=%u", index,
983  (uint32_t)old[index].b_nob,
984  (uint32_t)new[index].b_nob);
985 
986  block_size = new[index].b_nob;
987 
988  /* It is assumed that the buffer size will always be a multiple of
989  * 8-bytes (especially since block size currently is 4K) and this assert
990  * is added with the hope that any deviation from this assumption is
991  * caught during development instead of on the field. */
992  M0_ASSERT_INFO(m0_is_aligned(block_size, alignment),
993  "block_size=%u is not %u-bytes aligned",
994  block_size, alignment);
995 
996  M0_ALLOC_ARR(diff_data_arr, block_size);
997  if (diff_data_arr == NULL)
998  return BUF_ALLOC_ERR_INFO(-ENOMEM, "differential data block",
999  block_size);
1000 
1001  m0_buf_init(&diff_data_buf, diff_data_arr, block_size);
1002 
1003  /* Calculate the differential data using old and new data blocks. */
1004  for (i = 0; i < (block_size / alignment); i++)
1005  ((uint_fast32_t *)diff_data_arr)[i] =
1006  ((uint_fast32_t *)old[index].b_addr)[i] ^
1007  ((uint_fast32_t *)new[index].b_addr)[i];
1008 
1009  /* Update differential parity using differential data. */
1010  ret = isal_encode_data_update(parity, &diff_data_buf, index,
1011  math->pmi_rs.rs_encode_tbls,
1012  math->pmi_data_count,
1013  math->pmi_parity_count);
1014 
1015  m0_free(diff_data_arr);
1016  return M0_RC(ret);
1017 }
1018 
1019 static int reed_solomon_recover(struct m0_parity_math *math,
1020  struct m0_buf *data,
1021  struct m0_buf *parity,
1022  struct m0_buf *fails,
1023  enum m0_parity_linsys_algo algo)
1024 {
1025  uint32_t fail_count;
1026  uint32_t total_count;
1027  uint32_t block_size;
1028  uint32_t i;
1029  uint8_t *fail = NULL;
1030  int ret;
1031  struct m0_reed_solomon *rs;
1032 
1033  M0_ENTRY("math=%p, data=%p, parity=%p, fails=%p",
1034  math, data, parity, fails);
1035 
1037  M0_PRE(data != NULL);
1038  M0_PRE(parity != NULL);
1039  M0_PRE(fails != NULL);
1040 
1041  total_count = math->pmi_data_count + math->pmi_parity_count;
1042  block_size = data[0].b_nob;
1043  rs = &math->pmi_rs;
1044 
1045  fail = (uint8_t*) fails->b_addr;
1046  fail_count = fails_count(fail, total_count);
1047 
1048  M0_ASSERT(fail_count > 0);
1049  M0_ASSERT(fail_count <= math->pmi_parity_count);
1050 
1051  /* Validate block size for data buffers. */
1052  for (i = 1; i < math->pmi_data_count; ++i)
1053  BLOCK_SIZE_ASSERT_INFO(block_size, i, data);
1054 
1055  /* Validate block size for parity buffers. */
1056  for (i = 0; i < math->pmi_parity_count; ++i)
1057  BLOCK_SIZE_ASSERT_INFO(block_size, i, parity);
1058 
1059  /* Sort buffers which are to be recovered. */
1060  buf_sort(rs, total_count, math->pmi_data_count,
1061  fail, data, parity);
1062 
1063  /* Sort failed buffer indices. */
1064  fails_sort(rs, fail, total_count, math->pmi_parity_count);
1065 
1066  VALUE_MISMATCH_ASSERT_INFO(fail_count, rs->rs_failed_nr);
1067 
1068  /* Get encoding coefficient tables. */
1070  math->pmi_parity_count, rs);
1071  if (ret != 0)
1072  return M0_ERR_INFO(ret, "failed to generate recovery "
1073  "coefficient tables");
1074 
1075  /* Recover failed data. */
1076  ec_encode_data(block_size, math->pmi_data_count,
1077  fail_count, rs->rs_decode_tbls,
1078  rs->rs_bufs_in, rs->rs_bufs_out);
1079 
1080  return M0_RC(ret);
1081 }
1082 
1083 static void ir_rs_init(const struct m0_parity_math *math, struct m0_sns_ir *ir)
1084 {
1085  M0_ENTRY("math=%p, ir=%p", math, ir);
1086 
1087  M0_PRE(math != NULL);
1088  M0_PRE(ir != NULL);
1089 
1093  ir->si_rs.rs_alive_idx = math->pmi_rs.rs_alive_idx;
1094 
1095  M0_LEAVE();
1096 }
1097 
1098 static void ir_failure_register(struct m0_sns_ir *ir, uint32_t failed_index)
1099 {
1100  M0_ENTRY("ir=%p, failed_index=%u", ir, failed_index);
1101 
1102  M0_PRE(ir != NULL);
1103  M0_PRE(ir->si_rs.rs_failed_idx != NULL);
1105 
1106  ir->si_rs.rs_failed_idx[ir->si_rs.rs_failed_nr++] = failed_index;
1107 
1108  M0_LEAVE();
1109 }
1110 
1111 static int ir_mat_compute(struct m0_sns_ir *ir)
1112 {
1113  struct m0_sns_ir_block *blocks;
1114  struct m0_reed_solomon *rs;
1115  uint32_t i;
1116  uint32_t alive_nr;
1117  uint32_t total_blocks_nr;
1118  int ret;
1119 
1120  M0_ENTRY("ir=%p", ir);
1121  M0_PRE(ir != NULL);
1122 
1123  blocks = ir->si_blocks;
1124  rs = &ir->si_rs;
1125  total_blocks_nr = ir_blocks_count(ir);
1126 
1127  for (i = 0, alive_nr = 0; i < total_blocks_nr; i++)
1128  if (blocks[i].sib_status == M0_SI_BLOCK_ALIVE)
1129  rs->rs_alive_idx[alive_nr++] = blocks[i].sib_idx;
1130 
1131  VALUE_MISMATCH_ASSERT_INFO(alive_nr, ir->si_alive_nr);
1132 
1133  ret = isal_gen_recov_coeff_tbl(ir->si_data_nr, ir->si_parity_nr, rs);
1134  if (ret != 0)
1135  return M0_ERR_INFO(ret, "failed to generate decode matrix");
1136 
1137  for (i = 0; i < rs->rs_failed_nr; i++)
1138  dependency_bitmap_prepare(&blocks[rs->rs_failed_idx[i]], ir);
1139 
1140  return M0_RC(ret);
1141 }
1142 
1143 static int ir_recover(struct m0_sns_ir *ir, struct m0_sns_ir_block *alive_block)
1144 {
1145  struct m0_reed_solomon *rs;
1146  struct m0_bitmap *failed_bitmap;
1147  struct m0_bufvec *alive_bufvec;
1148  struct m0_bufvec *failed_bufvec;
1149  struct m0_buf in_buf = M0_BUF_INIT0;
1150  struct m0_buf *out_bufs;
1151  uint8_t curr_idx = UINT8_MAX;
1152  uint32_t i;
1153  uint32_t length;
1154  int ret = 0;
1155 
1156  M0_ENTRY("ir=%p, alive_block=%p", ir, alive_block);
1157 
1158  rs = &ir->si_rs;
1159 
1160  /* Check if given alive block is dependecy of any failed block. */
1161  for (i = 0; i < rs->rs_failed_nr; i++) {
1162  failed_bitmap = &ir->si_blocks[rs->rs_failed_idx[i]].sib_bitmap;
1163  if (m0_bitmap_get(failed_bitmap, alive_block->sib_idx) == 0)
1164  return M0_RC(ret);
1165  }
1166 
1167  alive_bufvec = alive_block->sib_addr;
1168  length = (uint32_t)m0_vec_count(&alive_bufvec->ov_vec);
1169 
1170  M0_ALLOC_ARR(out_bufs, rs->rs_failed_nr);
1171  if (out_bufs == NULL)
1172  return BUF_ALLOC_ERR_INFO(-ENOMEM, "output bufs",
1173  rs->rs_failed_nr);
1174 
1175  for (i = 0; i < rs->rs_failed_nr; i++) {
1176  ret = m0_buf_alloc(&out_bufs[i], length);
1177  if (ret != 0) {
1178  ret = BUF_ALLOC_ERR_INFO(ret, "OUT", length);
1179  goto exit;
1180  }
1181 
1182  /* Get data from failed vectors in target blocks. */
1183  failed_bufvec = ir->si_blocks[rs->rs_failed_idx[i]].sib_addr;
1184  ret = m0_bufvec_to_buf_copy(&out_bufs[i], failed_bufvec);
1185  if (ret != 0) {
1186  ret = M0_ERR_INFO(ret, "Failed to copy data from "
1187  "si_blocks[%u].sib_addr to "
1188  "out_bufs[%u].",
1189  rs->rs_failed_idx[i], i);
1190  goto exit;
1191  }
1192  }
1193 
1194  for (i = 0; i < ir->si_alive_nr; i++) {
1195  if(rs->rs_alive_idx[i] == alive_block->sib_idx) {
1196  curr_idx = i;
1197  break;
1198  }
1199  }
1200 
1201  if (curr_idx == UINT8_MAX) {
1202  ret = M0_ERR_INFO(-EINVAL, "Failed to find alive block "
1203  "index %d in alive index array",
1204  alive_block->sib_idx);
1205  goto exit;
1206  }
1207 
1208  /* Allocate buffer for input block. */
1209  ret = m0_buf_alloc(&in_buf, length);
1210  if (ret != 0) {
1211  ret = BUF_ALLOC_ERR_INFO(ret, "IN", length);
1212  goto exit;
1213  }
1214 
1215  /* Get data from current alive vector in input buffer. */
1216  ret = m0_bufvec_to_buf_copy(&in_buf, alive_bufvec);
1217  if (ret != 0) {
1218  ret = M0_ERR_INFO(ret, "Failed to copy data from "
1219  "alive_bufvec to source buffer.");
1220  goto exit;
1221  }
1222 
1223  /* Recover the data using input buffer and its index. */
1224  ret = isal_encode_data_update(out_bufs, &in_buf, curr_idx,
1225  rs->rs_decode_tbls, ir->si_data_nr,
1226  rs->rs_failed_nr);
1227  if (ret != 0) {
1228  ret = M0_ERR_INFO(ret, "Failed to recover the data.");
1229  goto exit;
1230  }
1231 
1232  /* Copy recovered data back to failed vectors. */
1233  for (i = 0; i < rs->rs_failed_nr; i++) {
1234  failed_bufvec = ir->si_blocks[rs->rs_failed_idx[i]].sib_addr;
1235  ret = m0_buf_to_bufvec_copy(failed_bufvec, &out_bufs[i]);
1236  if (ret != 0){
1237  ret = M0_ERR_INFO(ret, "Failed to copy data from "
1238  "out_bufs[%u] to "
1239  "si_blocks[%u].sib_addr.", i,
1240  rs->rs_failed_idx[i]);
1241  goto exit;
1242  }
1243  }
1244 
1245 exit:
1246  m0_buf_free(&in_buf);
1247  for (i = 0; i < rs->rs_failed_nr; i++)
1248  m0_buf_free(&out_bufs[i]);
1249  m0_free(out_bufs);
1250 
1251  return M0_RC(ret);
1252 }
1253 
1254 static int isal_encode_data_update(struct m0_buf *dest_bufs, struct m0_buf *src_buf,
1255  uint32_t vec_idx, uint8_t *g_tbls,
1256  uint32_t data_nr, uint32_t dest_nr)
1257 {
1258  uint32_t i;
1259  uint32_t block_size;
1260  uint8_t **dest_frags;
1261 
1262  M0_ENTRY("dest_bufs=%p, src_buf=%p, vec_idx=%u, "
1263  "g_tbls=%p, data_nr=%u, dest_nr=%u",
1264  dest_bufs, src_buf, vec_idx, g_tbls, data_nr, dest_nr);
1265 
1266  M0_PRE(dest_bufs != NULL);
1267  M0_PRE(src_buf != NULL);
1268  M0_PRE(g_tbls != NULL);
1269 
1270  M0_ALLOC_ARR(dest_frags, dest_nr);
1271  if (dest_frags == NULL)
1272  return BUF_ALLOC_ERR_INFO(-ENOMEM, "destination fragments",
1273  dest_nr);
1274 
1275  block_size = (uint32_t)src_buf->b_nob;
1276 
1277  for (i = 0; i < dest_nr; ++i) {
1278  BLOCK_SIZE_ASSERT_INFO(block_size, i, dest_bufs);
1279  dest_frags[i] = (uint8_t *)dest_bufs[i].b_addr;
1280  }
1281 
1282  ec_encode_data_update(block_size, data_nr, dest_nr, vec_idx,
1283  g_tbls, (uint8_t *)src_buf->b_addr, dest_frags);
1284 
1285  m0_free(dest_frags);
1286  return M0_RC(0);
1287 }
1288 
1289 static void fails_sort(struct m0_reed_solomon *rs, uint8_t *fail,
1290  uint32_t total_count, uint32_t parity_count)
1291 {
1292  uint32_t i;
1293  uint32_t alive_nr;
1294 
1295  M0_ENTRY();
1296 
1297  M0_PRE(fail != NULL);
1298  M0_PRE(rs != NULL);
1299  M0_PRE(rs->rs_failed_idx != NULL);
1300  M0_PRE(rs->rs_alive_idx != NULL);
1301 
1302  rs->rs_failed_nr = 0;
1303  for (i = 0, alive_nr = 0; i < total_count; i++) {
1304  if (fail[i] != 0) {
1305  VALUE_ASSERT_INFO(rs->rs_failed_nr < parity_count,
1306  rs->rs_failed_nr);
1307  rs->rs_failed_idx[rs->rs_failed_nr++] = i;
1308  } else
1309  rs->rs_alive_idx[alive_nr++] = i;
1310  }
1311  M0_LEAVE();
1312 }
1313 
1314 static void buf_sort(struct m0_reed_solomon *rs, uint32_t total_count,
1315  uint32_t data_count, uint8_t *fail,
1316  struct m0_buf *data, struct m0_buf *parity)
1317 {
1318  uint32_t i;
1319  uint32_t j;
1320  uint32_t k;
1321  uint8_t *addr;
1322 
1323  M0_ENTRY();
1324 
1325  M0_PRE(rs != NULL);
1326  M0_PRE(fail != NULL);
1327  M0_PRE(data != NULL);
1328  M0_PRE(parity != NULL);
1329  M0_PRE(rs->rs_bufs_in != NULL);
1330  M0_PRE(rs->rs_bufs_out != NULL);
1331 
1332  for (i = 0, j = 0, k = 0; i < total_count; i++) {
1333  if (i < data_count)
1334  addr = (uint8_t *)data[i].b_addr;
1335  else
1336  addr = (uint8_t *)parity[i - data_count].b_addr;
1337 
1338  if (fail[i] != 0)
1339  rs->rs_bufs_out[j++] = addr;
1340  else if (k < data_count)
1341  rs->rs_bufs_in[k++] = addr;
1342  else
1343  continue;
1344  }
1345  M0_LEAVE();
1346 }
1347 
1348 static int isal_gen_recov_coeff_tbl(uint32_t data_count, uint32_t parity_count,
1349  struct m0_reed_solomon *rs)
1350 {
1351  uint32_t total_count;
1352  uint32_t mat_size;
1353  uint32_t i;
1354  uint32_t j;
1355  uint32_t r;
1356  uint8_t s;
1357  uint8_t idx;
1358  int ret;
1359  uint8_t *decode_mat = NULL;
1360  uint8_t *temp_mat = NULL;
1361  uint8_t *invert_mat = NULL;
1362 
1363  M0_ENTRY("data_count=%u, parity_count=%u, rs=%p",
1364  data_count, parity_count, rs);
1365 
1366  M0_PRE(rs != NULL);
1367 
1368  total_count = data_count + parity_count;
1369  mat_size = total_count * data_count;
1370 
1371  M0_ALLOC_ARR(decode_mat, mat_size);
1372  if (decode_mat == NULL) {
1373  ret = BUF_ALLOC_ERR_INFO(-ENOMEM, "decode matrix",
1374  mat_size);
1375  goto exit;
1376  }
1377 
1378  M0_ALLOC_ARR(temp_mat, mat_size);
1379  if (temp_mat == NULL) {
1380  ret = BUF_ALLOC_ERR_INFO(-ENOMEM, "temp matrix",
1381  mat_size);
1382  goto exit;
1383  }
1384 
1385  M0_ALLOC_ARR(invert_mat, mat_size);
1386  if (invert_mat == NULL) {
1387  ret = BUF_ALLOC_ERR_INFO(-ENOMEM, "invert matrix",
1388  mat_size);
1389  goto exit;
1390  }
1391 
1392  /* Construct temp_mat (matrix that encoded remaining frags)
1393  * by removing erased rows. */
1394  for (i = 0; i < data_count; i++) {
1395  idx = rs->rs_alive_idx[i];
1396  for (j = 0; j < data_count; j++)
1397  temp_mat[data_count * i + j] =
1398  rs->rs_encode_matrix[data_count * idx + j];
1399  }
1400 
1401  /* Invert matrix to get recovery matrix. */
1402  ret = gf_invert_matrix(temp_mat, invert_mat, data_count);
1403  if (ret != 0) {
1404  ret = M0_ERR_INFO(ret, "failed to construct an %u x %u "
1405  "inverse of the input matrix",
1406  data_count, data_count);
1407  goto exit;
1408  }
1409 
1410  /* Create decode matrix. */
1411  for (r = 0; r < rs->rs_failed_nr; r++) {
1412  idx = rs->rs_failed_idx[r];
1413  /* Get decode matrix with only wanted recovery rows */
1414  if (idx < data_count) { /* A src err */
1415  for (i = 0; i < data_count; i++)
1416  decode_mat[data_count * r + i] =
1417  invert_mat[data_count * idx + i];
1418  } else { /* A parity err */
1419  /* For non-src (parity) erasures need to multiply
1420  * encode matrix * invert */
1421  for (i = 0; i < data_count; i++) {
1422  s = 0;
1423  for (j = 0; j < data_count; j++)
1424  s ^= gf_mul(invert_mat[j * data_count + i],
1425  rs->rs_encode_matrix[j +
1426  data_count * idx]);
1427  decode_mat[data_count * r + i] = s;
1428  }
1429  }
1430  }
1431 
1432  ec_init_tables(data_count, rs->rs_failed_nr,
1433  decode_mat, rs->rs_decode_tbls);
1434 
1435 exit:
1436  m0_free(decode_mat);
1437  m0_free(temp_mat);
1438  m0_free(invert_mat);
1439 
1440  return M0_RC(ret);
1441 }
1442 
1443 static void dependency_bitmap_prepare(struct m0_sns_ir_block *f_block,
1444  struct m0_sns_ir *ir)
1445 {
1446  uint32_t i;
1447 
1448  M0_PRE(f_block != NULL && ir != NULL);
1449  M0_PRE(f_block->sib_status == M0_SI_BLOCK_FAILED);
1450 
1451  for (i = 0; i < ir->si_data_nr; ++i)
1452  m0_bitmap_set(&f_block->sib_bitmap, ir->si_rs.rs_alive_idx[i], true);
1453 }
1454 
1455 static uint32_t last_usable_block_id(const struct m0_sns_ir *ir,
1456  uint32_t block_idx)
1457 {
1458  return ir->si_rs.rs_alive_idx[ir->si_data_nr - 1];
1459 }
1460 #else
1461 static void reed_solomon_fini(struct m0_parity_math *math)
1462 {
1463 }
1464 
1465 static int reed_solomon_init(struct m0_parity_math *math)
1466 {
1467  return 0;
1468 }
1469 
1470 static void reed_solomon_encode(struct m0_parity_math *math,
1471  const struct m0_buf *data,
1472  struct m0_buf *parity)
1473 {
1474 }
1475 
1476 static int reed_solomon_diff(struct m0_parity_math *math,
1477  struct m0_buf *old,
1478  struct m0_buf *new,
1479  struct m0_buf *parity,
1480  uint32_t index)
1481 {
1482  return 0;
1483 }
1484 
1485 static int reed_solomon_recover(struct m0_parity_math *math,
1486  struct m0_buf *data,
1487  struct m0_buf *parity,
1488  struct m0_buf *fails,
1489  enum m0_parity_linsys_algo algo)
1490 {
1491  return 0;
1492 }
1493 
1494 static void ir_rs_init(const struct m0_parity_math *math, struct m0_sns_ir *ir)
1495 {
1496 }
1497 
1498 static void ir_failure_register(struct m0_sns_ir *ir, uint32_t failed_index)
1499 {
1500 }
1501 
1502 static int ir_mat_compute(struct m0_sns_ir *ir)
1503 {
1504  return 0;
1505 }
1506 
1507 static int ir_recover(struct m0_sns_ir *ir, struct m0_sns_ir_block *alive_block)
1508 {
1509  return 0;
1510 }
1511 
1512 static uint32_t last_usable_block_id(const struct m0_sns_ir *ir,
1513  uint32_t block_idx)
1514 {
1515  return 0;
1516 }
1517 #endif /* __KERNEL__ */
1518 
1519 #undef M0_TRACE_SUBSYSTEM
1520 
1521 /*
1522  * Local variables:
1523  * c-indentation-style: "K&R"
1524  * c-basic-offset: 8
1525  * tab-width: 8
1526  * fill-column: 80
1527  * scroll-step: 1
1528  * End:
1529  */
static m0_bcount_t seg_size
Definition: net.c:118
enum m0_sns_ir_block_status sib_status
Definition: parity_math.h:88
M0_INTERNAL int m0_bufvec_to_buf_copy(struct m0_buf *buf, const struct m0_bufvec *bvec)
Definition: vec.c:1301
struct m0_reed_solomon pmi_rs
Definition: parity_math.h:129
static m0_parity_elem_t m0_parity_add(m0_parity_elem_t x, m0_parity_elem_t y)
Definition: parity_ops.h:41
M0_INTERNAL void m0_parity_math_fini(struct m0_parity_math *math)
Definition: parity_math.c:325
#define M0_PRE(cond)
#define M0_ALLOC_ARR(arr, nr)
Definition: memory.h:84
static int gadd(int x, int y)
Definition: parity_math.c:568
uint8_t ** rs_bufs_in
Definition: parity_math.h:114
M0_INTERNAL int m0_bitmap_init(struct m0_bitmap *map, size_t nr)
Definition: bitmap.c:86
uint32_t sib_idx
Definition: parity_math.h:78
uint32_t pmi_data_count
Definition: parity_math.h:127
static int(* diff[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
Definition: parity_math.c:290
static bool is_usable(const struct m0_sns_ir *ir, const struct m0_bitmap *in_bmap, struct m0_sns_ir_block *failed_block)
Definition: parity_math.c:812
static void dependency_bitmap_prepare(struct m0_sns_ir_block *f_block, struct m0_sns_ir *ir)
Definition: parity_math.c:1443
static int xor_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
Definition: parity_math.c:621
#define NULL
Definition: misc.h:38
M0_INTERNAL void m0_bitmap_fini(struct m0_bitmap *map)
Definition: bitmap.c:97
enum m0_parity_cal_algo pmi_parity_algo
Definition: parity_math.h:125
M0_INTERNAL int m0_buf_to_bufvec_copy(struct m0_bufvec *bvec, const struct m0_buf *buf)
Definition: vec.c:1315
static uint32_t ir_blocks_count(const struct m0_sns_ir *ir)
Definition: parity_math.c:760
#define ergo(a, b)
Definition: misc.h:293
#define VALUE_ASSERT_INFO(cond, var)
Definition: parity_math.c:41
static void(* calculate[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
Definition: parity_math.c:283
static bool x
Definition: sm.c:168
void * b_addr
Definition: buf.h:39
M0_LEAVE()
M0_INTERNAL void m0_parity_math_buffer_xor(struct m0_buf *dest, const struct m0_buf *src)
Definition: parity_math.c:415
uint32_t si_alive_nr
Definition: parity_math.h:136
struct m0_vec ov_vec
Definition: vec.h:147
M0_INTERNAL void m0_buf_init(struct m0_buf *buf, void *data, uint32_t nob)
Definition: buf.c:37
struct m0_bufvec data
Definition: di.c:40
uint8_t * rs_encode_matrix
Definition: parity_math.h:97
static int reed_solomon_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
Definition: parity_math.c:957
static int gmul(int x, int y)
Definition: parity_math.c:573
struct m0_bitmap sib_bitmap
Definition: parity_math.h:86
M0_INTERNAL void * m0_bufvec_cursor_addr(struct m0_bufvec_cursor *cur)
Definition: vec.c:597
uint64_t m0_bcount_t
Definition: types.h:77
M0_INTERNAL size_t m0_bitmap_set_nr(const struct m0_bitmap *map)
Definition: bitmap.c:172
static void fails_sort(struct m0_reed_solomon *rs, uint8_t *fail, uint32_t total_count, uint32_t parity_count)
Definition: parity_math.c:1289
M0_INTERNAL int m0_parity_math_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
Definition: parity_math.c:383
#define M0_SET0(obj)
Definition: misc.h:64
M0_INTERNAL void m0_sns_ir_fini(struct m0_sns_ir *ir)
Definition: parity_math.c:488
M0_INTERNAL int m0_parity_math_diff(struct m0_parity_math *math, struct m0_buf *old, struct m0_buf *new, struct m0_buf *parity, uint32_t index)
Definition: parity_math.c:371
static int ir_recover(struct m0_sns_ir *ir, struct m0_sns_ir_block *alive_block)
Definition: parity_math.c:1143
struct m0_sns_ir_block * si_blocks
Definition: parity_math.h:141
static m0_bcount_t count
Definition: xcode.c:167
M0_INTERNAL int m0_sns_ir_failure_register(struct m0_bufvec *recov_addr, uint32_t failed_index, struct m0_sns_ir *ir)
Definition: parity_math.c:453
M0_INTERNAL int m0_sns_ir_init(const struct m0_parity_math *math, uint32_t local_nr, struct m0_sns_ir *ir)
Definition: parity_math.c:425
return M0_RC(rc)
M0_INTERNAL void m0_parity_math_calculate(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity)
Definition: parity_math.c:362
#define M0_ENTRY(...)
Definition: trace.h:170
static int reed_solomon_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
Definition: parity_math.c:1019
Definition: buf.h:37
static void fail_idx_reed_solomon_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t failure_index)
Definition: parity_math.c:727
M0_INTERNAL bool m0_bufvec_cursor_move(struct m0_bufvec_cursor *cur, m0_bcount_t count)
Definition: vec.c:574
M0_INTERNAL void m0_parity_math_refine(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, uint32_t data_ind_changed)
Definition: parity_math.c:406
static char * addr
Definition: node_k.c:37
int i
Definition: dir.c:1033
static uint32_t last_usable_block_id(const struct m0_sns_ir *ir, uint32_t block_idx)
Definition: parity_math.c:1455
struct m0_bufvec * sib_addr
Definition: parity_math.h:82
static void reed_solomon_encode(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
Definition: parity_math.c:921
#define UINT8_MAX
Definition: types.h:35
#define M0_ERR_INFO(rc, fmt,...)
Definition: trace.h:215
return M0_ERR(-EOPNOTSUPP)
void * b_addr
Definition: buf.h:231
static void xor_calculate(struct m0_parity_math *math, const struct m0_buf *data, struct m0_buf *parity)
Definition: parity_math.c:597
static int reed_solomon_init(struct m0_parity_math *math)
Definition: parity_math.c:853
M0_INTERNAL m0_bcount_t m0_bufvec_cursor_step(const struct m0_bufvec_cursor *cur)
Definition: vec.c:581
m0_bcount_t b_nob
Definition: buf.h:38
#define M0_ASSERT(cond)
static bool m0_is_aligned(uint64_t val, uint64_t alignment)
Definition: arith.h:179
M0_INTERNAL int m0_sns_ir_recover(struct m0_sns_ir *ir, struct m0_bufvec *bufvec, const struct m0_bitmap *bitmap, uint32_t failed_index, enum m0_sns_ir_block_type block_type)
Definition: parity_math.c:503
#define M0_BUF_INIT0
Definition: buf.h:71
M0_INTERNAL void m0_bufvec_cursor_init(struct m0_bufvec_cursor *cur, const struct m0_bufvec *bvec)
Definition: vec.c:563
M0_INTERNAL int m0_sns_ir_mat_compute(struct m0_sns_ir *ir)
Definition: parity_math.c:482
M0_INTERNAL int m0_parity_math_init(struct m0_parity_math *math, uint32_t data_count, uint32_t parity_count)
Definition: parity_math.c:333
M0_INTERNAL int m0_buf_alloc(struct m0_buf *buf, size_t size)
Definition: buf.c:43
struct m0_reed_solomon si_rs
Definition: parity_math.h:142
Definition: xcode.h:73
uint8_t * rs_failed_idx
Definition: parity_math.h:109
M0_INTERNAL void m0_bitmap_set(struct m0_bitmap *map, size_t idx, bool val)
Definition: bitmap.c:139
static bool is_valid_block_idx(const struct m0_sns_ir *ir, uint32_t block_idx)
Definition: parity_math.c:833
uint32_t v_nr
Definition: vec.h:51
M0_INTERNAL void m0_buf_free(struct m0_buf *buf)
Definition: buf.c:55
uint8_t * rs_decode_tbls
Definition: parity_math.h:105
m0_bcount_t * v_count
Definition: vec.h:53
M0_INTERNAL int m0_bitmap_ffs(const struct m0_bitmap *map)
Definition: bitmap.c:112
uint8_t ** rs_bufs_out
Definition: parity_math.h:117
#define BUF_ALLOC_ERR_INFO(ret, str, len)
Definition: parity_math.c:35
uint32_t si_data_nr
Definition: parity_math.h:134
M0_INTERNAL m0_bcount_t m0_vec_count(const struct m0_vec *vec)
Definition: vec.c:53
#define m0_forall(var, nr,...)
Definition: misc.h:112
static uint8_t fail[DATA_UNIT_COUNT_MAX+PARITY_UNIT_COUNT_MAX]
uint32_t si_local_nr
Definition: parity_math.h:139
static uint32_t fails_count(uint8_t *fail, uint32_t unit_count)
Definition: parity_math.c:578
static int(* recover[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
Definition: parity_math.c:299
uint64_t * b_words
Definition: bitmap.h:46
static void gfaxpy(struct m0_bufvec *y, struct m0_bufvec *x, m0_parity_elem_t alpha)
Definition: parity_math.c:765
uint8_t * rs_alive_idx
Definition: parity_math.h:111
uint32_t rs_failed_nr
Definition: parity_math.h:107
static int r[NR]
Definition: thread.c:46
M0_INTERNAL bool m0_bitmap_get(const struct m0_bitmap *map, size_t idx)
Definition: bitmap.c:105
static void ir_rs_init(const struct m0_parity_math *math, struct m0_sns_ir *ir)
Definition: parity_math.c:1083
static void(* fidx_recover[M0_PARITY_CAL_ALGO_NR])(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t fidx)
Definition: parity_math.c:308
#define _0C(exp)
Definition: assert.h:311
static int isal_gen_recov_coeff_tbl(uint32_t data_count, uint32_t parity_count, struct m0_reed_solomon *rs)
Definition: parity_math.c:1348
#define VALUE_MISMATCH_ASSERT_INFO(lhs, rhs)
Definition: parity_math.c:46
#define M0_PARITY_GALOIS_W
Definition: parity_ops.h:34
static void fail_idx_xor_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t failure_index)
Definition: parity_math.c:692
int m0_parity_elem_t
Definition: parity_ops.h:36
#define M0_CNT_DEC(cnt)
Definition: arith.h:219
#define M0_ASSERT_INFO(cond, fmt,...)
m0_sns_ir_block_type
Definition: parity_math.h:60
static int isal_encode_data_update(struct m0_buf *dest_bufs, struct m0_buf *src_buf, uint32_t vec_idx, uint8_t *g_tbls, uint32_t data_nr, uint32_t dest_nr)
Definition: parity_math.c:1254
static void reed_solomon_fini(struct m0_parity_math *math)
Definition: parity_math.c:840
size_t b_nr
Definition: bitmap.h:44
static m0_parity_elem_t m0_parity_mul(m0_parity_elem_t x, m0_parity_elem_t y)
Definition: parity_ops.h:51
uint8_t * rs_encode_tbls
Definition: parity_math.h:101
void m0_free(void *data)
Definition: memory.c:146
static struct m0_addb2_source * s
Definition: consumer.c:39
M0_INTERNAL void m0_parity_math_fail_index_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, const uint32_t fidx)
Definition: parity_math.c:396
static void buf_sort(struct m0_reed_solomon *rs, uint32_t total_count, uint32_t data_count, uint8_t *fail, struct m0_buf *data, struct m0_buf *parity)
Definition: parity_math.c:1314
struct m0_pdclust_src_addr src
Definition: fd.c:108
int32_t rc
Definition: trigger_fop.h:47
static int ir_si_blocks_init(struct m0_sns_ir *ir)
Definition: parity_math.c:734
static void ir_failure_register(struct m0_sns_ir *ir, uint32_t failed_index)
Definition: parity_math.c:1098
static bool parity_math_invariant(const struct m0_parity_math *math)
Definition: parity_math.c:589
uint32_t pmi_parity_count
Definition: parity_math.h:128
uint32_t si_parity_nr
Definition: parity_math.h:135
#define BLOCK_SIZE_ASSERT_INFO(blksz, index, block)
Definition: parity_math.c:52
static uint8_t parity[DATA_UNIT_COUNT_MAX][UNIT_BUFF_SIZE_MAX]
Definition: vec.h:145
m0_bcount_t b_nob
Definition: buf.h:230
static int xor_recover(struct m0_parity_math *math, struct m0_buf *data, struct m0_buf *parity, struct m0_buf *fails, enum m0_parity_linsys_algo algo)
Definition: parity_math.c:646
static int ir_mat_compute(struct m0_sns_ir *ir)
Definition: parity_math.c:1111
m0_parity_linsys_algo
Definition: parity_math.h:65