Motr  M0
Data Structures | Macros | Enumerations | Functions | Variables
beck.c File Reference
#include <err.h>
#include <stdio.h>
#include <string.h>
#include <sysexits.h>
#include <fcntl.h>
#include <unistd.h>
#include <time.h>
#include <pthread.h>
#include <signal.h>
#include <yaml.h>
#include <sys/mman.h>
#include "lib/trace.h"
#include "lib/getopts.h"
#include "lib/memory.h"
#include "lib/assert.h"
#include "lib/thread.h"
#include "motr/version.h"
#include "lib/uuid.h"
#include "motr/magic.h"
#include "motr/init.h"
#include "reqh/reqh.h"
#include "module/instance.h"
#include "format/format.h"
#include "format/format_xc.h"
#include "balloc/balloc.h"
#include "be/btree_internal.h"
#include "be/list.h"
#include "be/seg_internal.h"
#include "be/op.h"
#include "be/ut/helper.h"
#include "cas/ctg_store.h"
#include "cob/cob.h"
#include "pool/pool.h"
#include "stob/ad_private.h"
#include "dix/fid_convert.h"
#include "stob/ad.h"
#include "ioservice/io_service.h"
#include "ioservice/fid_convert.h"
#include "ioservice/cob_foms.h"
#include "be/extmap_internal.h"
#include "be/tx_bulk.h"
Include dependency graph for beck.c:

Go to the source code of this file.

Data Structures

struct  queue
 
struct  scanner
 
struct  stats
 
struct  rectype
 
struct  recops
 
struct  bstats
 
struct  btype
 
struct  gen
 
struct  action
 
struct  bnode_act
 
struct  cob_action
 
struct  action_ops
 
struct  cache_slot
 
struct  cache
 
struct  ad_dom_info
 
struct  builder
 
struct  emap_action
 
struct  worker_off_info
 
struct  scanner_off_info
 
struct  part_info
 
struct  nv_offset_info
 
struct  offset_info
 
struct  ctg_action
 

Macros

#define _FILE_OFFSET_BITS   64 /* for fseeko */
 
#define M0_TRACE_SUBSYSTEM   M0_TRACE_SUBSYS_BE
 
#define _FT(name)   M0_FORMAT_TYPE_ ## name
 
#define _TAG(name)   M0_ ## name ## _FORMAT_VERSION , _FT(name)
 
#define _T(name, str, field, ops)
 
#define _B(t, proc)   [t] = { .b_type = (t), .b_proc = (proc) }
 
#define FLOG(level, rc, s)
 
#define RLOG(level, prefix, s, r, tag)
 

Enumerations

enum  action_opcode {
  AO_INIT = 1, AO_DONE = 2, AO_CTG = 3, AO_COB = 4,
  AO_EMAP_FIRST = 5, AO_NR = 30
}
 
enum  { MAX_WORKERS_NR = 64 }
 
enum  { CACHE_SIZE = 1000000 }
 
enum  { NV_OFFSET_SAVE_DELTA_IN_BYTES = 0x40000000 }
 
enum  { NV_OFFSET_SAVE_ACT_DELTA = 1000 }
 
enum  {
  MAX_GEN = 256, MAX_SCAN_QUEUED = 10000000, MAX_QUEUED = 1000000, MAX_REC_SIZE = 64*1024,
  MAX_GEN_DIFF_SEC = 30, MAX_KEY_LEN = 256, MAX_VALUE_LEN = 256, DEFAULT_BE_SEG_LOAD_ADDR = 0x400000100000,
  DEFAULT_BE_MAX_TX_REG_SZ = (44 * 1024 * 1024ULL)
}
 
enum  { DELTA = 60 }
 

Functions

 M0_TL_DESCR_DECLARE (ad_domains, M0_EXTERN)
 
 M0_TL_DECLARE (ad_domains, M0_EXTERN, struct ad_domain_map)
 
static int init (void)
 
static void fini (void)
 
static int scan (struct scanner *s)
 
static void stats_print (void)
 
static int parse (struct scanner *s)
 
static int get (struct scanner *s, void *buf, size_t nob)
 
static int getat (struct scanner *s, off_t off, void *buf, size_t nob)
 
static int deref (struct scanner *s, const void *addr, void *buf, size_t nob)
 
static int recdo (struct scanner *s, const struct m0_format_tag *tag, struct rectype *rt)
 
static const char * recname (const struct rectype *rt)
 
static const char * bname (const struct btype *bt)
 
static int btree (struct scanner *s, struct rectype *r, char *buf)
 
static int bnode (struct scanner *s, struct rectype *r, char *buf)
 
static int seghdr (struct scanner *s, struct rectype *r, char *buf)
 
static void * scanner_action (size_t len, enum action_opcode opc, const struct action_ops *ops)
 
static void genadd (uint64_t gen)
 
static void generation_id_print (uint64_t gen)
 
static void generation_id_get (FILE *fp, uint64_t *gen_id)
 
static int generation_id_verify (struct scanner *s, uint64_t gen)
 
static void seg_get (FILE *fp, struct m0_be_seg *out)
 
static int scanner_cache_init (struct scanner *s)
 
static int scanner_init (struct scanner *s)
 
static void scanner_fini (struct scanner *s)
 
static int builder_init (struct builder *b)
 
static void builder_fini (struct builder *b)
 
static void ad_dom_fini (struct builder *b)
 
static void builder_thread (struct builder *b)
 
static void be_cfg_default_init (struct m0_be_domain_cfg *dom_cfg, struct m0_be_tx_bulk_cfg *tb_cfg)
 
static int format_header_verify (const struct m0_format_header *h, uint16_t rtype)
 
static bool btree_node_pre_is_valid (const struct m0_be_bnode *node, struct scanner *s)
 
static bool btree_node_post_is_valid (const struct m0_be_bnode *node, const struct m0_be_btree_kv_ops *ops)
 
static bool btree_kv_is_valid (struct m0_be_bnode *node, int index, struct m0_buf *key)
 
static void btree_bad_kv_count_update (uint64_t type, int count)
 
static void * builder_action (struct builder *b, size_t len, enum action_opcode opc, const struct action_ops *ops)
 
static bool fid_without_type_eq (const struct m0_fid *fid0, const struct m0_fid *fid1)
 
static struct cache_slot * cache_lookup (struct cache *c, struct m0_fid *fid)
 
static struct cache_slot * cache_insert (struct cache *c, const struct m0_fid *fid)
 
static void qinit (struct queue *q, uint64_t maxnr)
 
static void qfini (struct queue *q)
 
static void qput (struct queue *q, struct action *act)
 
static struct action * qget (struct queue *q)
 
static struct action * qtry (struct queue *q)
 
static struct action * qpeek (struct queue *q)
 
static bool isqempty (struct queue *q)
 
static int ctg_proc (struct scanner *s, struct btype *b, struct m0_be_bnode *node, off_t node_offset)
 
static int ctg_pver_fid_get (struct m0_fid *fid)
 
static void test (void)
 
static int cob_proc (struct scanner *s, struct btype *b, struct m0_be_bnode *node, off_t node_offset)
 
static int emap_proc (struct scanner *s, struct btype *b, struct m0_be_bnode *node, off_t node_offset)
 
static int emap_prep (struct action *act, struct m0_be_tx_credit *cred)
 
static void emap_act (struct action *act, struct m0_be_tx *tx)
 
static void emap_fini (struct action *act)
 
static int emap_kv_get (struct scanner *s, const struct be_btree_key_val *kv, struct m0_buf *key_buf, struct m0_buf *val_buf)
 
static void sig_handler (int num)
 
static int be_cfg_from_yaml_update (const char *yaml_file, struct m0_be_domain_cfg *cfg)
 
static int nv_scan_offset_init (uint64_t workers_nr, uint64_t partitions_nr)
 
static void nv_scan_offset_fini (void)
 
static off_t nv_scan_offset_get (off_t snapshot_size)
 
static void nv_scan_offset_update (void)
 
static void scanner_thread (struct scanner *s)
 
 M0_BASSERT (ARRAY_SIZE(rt)==M0_FORMAT_TYPE_NR+1)
 
int main (int argc, char **argv)
 
static bool seg_hdr_get (FILE *fp, struct m0_be_seg_hdr *out)
 
static int btree_check (struct scanner *s, struct rectype *r, char *buf)
 
static int bnode_check (struct scanner *s, struct rectype *r, char *buf)
 
static struct m0_stob_ad_domain * emap_dom_find (const struct action *act, const struct m0_fid *emap_fid, int *lockid)
 
static void emap_to_gob_convert (const struct m0_uint128 *emap_prefix, struct m0_fid *out)
 
static int emap_entry_lookup (struct m0_stob_ad_domain *adom, struct m0_uint128 prefix, m0_bindex_t offset, struct m0_be_emap_cursor *it)
 
static int seghdr_ver (struct scanner *s, struct rectype *r, char *buf)
 
static int seghdr_check (struct scanner *s, struct rectype *r, char *buf)
 
static void builder_do (struct m0_be_tx_bulk *tb, struct m0_be_tx *tx, struct m0_be_op *op, void *datum, void *user, uint64_t worker_index, uint64_t partition)
 
static void builder_done (struct m0_be_tx_bulk *tb, void *datum, void *user, uint64_t worker_index, uint64_t partition)
 
static void builder_work_put (struct m0_be_tx_bulk *tb, struct builder *b)
 
static int ad_dom_init (struct builder *b)
 
static void be_cfg_update (struct m0_be_domain_cfg *cfg, const char *str_key, const char *str_value)
 
static bool btree_kv_post_is_valid (struct scanner *s, struct m0_buf *key, struct m0_buf *val)
 
static int ctg_k_get (struct scanner *s, const void *addr, struct m0_buf *kbuf)
 
static int ctg_v_get (struct scanner *s, const void *addr, struct m0_buf *vbuf)
 
static int ctg_btree_fid_get (struct m0_buf *kbuf, struct m0_fid *fid)
 
static struct cache_slot * ctg_getslot_insertcred (struct ctg_action *ca, struct builder *b, struct m0_fid *cas_ctg_fid, struct m0_be_tx_credit *accum)
 
static int ctg_prep (struct action *act, struct m0_be_tx_credit *accum)
 
static struct m0_cas_ctg * ctg_create_meta (struct ctg_action *ca, struct m0_be_tx *tx)
 
static void ctg_act (struct action *act, struct m0_be_tx *tx)
 
static void ctg_fini (struct action *act)
 
static bool qinvariant (const struct queue *q)
 
static int cob_kv_get (struct scanner *s, const struct be_btree_key_val *kv, struct m0_buf *key, struct m0_buf *val)
 
static int cob_prep (struct action *act, struct m0_be_tx_credit *accum)
 
static void cob_act (struct action *act, struct m0_be_tx *tx)
 
static void cob_fini (struct action *act)
 
static int noop_prep (struct action *act, struct m0_be_tx_credit *cred)
 
static void noop_act (struct action *act, struct m0_be_tx *tx)
 
static void noop_fini (struct action *act)
 
static void test_queue (void)
 

Variables

static const struct recops btreeops
 
static const struct recops bnodeops
 
static const struct recops seghdrops
 
static const struct action_ops done_ops
 
static const struct action_ops ctg_ops
 
static const struct action_ops cob_ops
 
static struct rectype rt []
 
static struct btype bt []
 
static struct scanner beck_scanner
 
static struct builder beck_builder
 
static struct gen g [MAX_GEN] = {}
 
static struct m0_be_seg s_seg = {}
 
static struct nv_offset_info nv_off_info
 
static struct offset_info off_info
 
static bool dry_run = false
 
static bool disable_directio = false
 
static bool signaled = false
 
static bool resume_scan = false
 
static bool mmap_be_segment = false
 
static const char * offset_file = NULL
 
static struct m0_be_tx_bulk_cfg default_tb_cfg
 
static char iobuf [4 *1024 *1024]
 
static const struct action_ops emap_ops