Data Structures
struct	ha_note_handler_request

struct	m0_ha_note

struct	m0_ha_nvec

struct	m0_ha_state_single

struct	m0_ha_msg_nvec_array

struct	m0_ha_msg_nvec

struct	m0_ha_note_handler

struct	m0_ha_state_fop

Macros
#define	M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_HA

#define	M0_NVEC_PRINT(nvec_, label, level)

Enumerations
enum	m0_ha_obj_state { M0_NC_UNKNOWN, M0_NC_ONLINE, M0_NC_FAILED, M0_NC_TRANSIENT, M0_NC_REPAIR, M0_NC_REPAIRED, M0_NC_REBALANCE, M0_NC_DTM_RECOVERING, M0_NC_NR }

enum	m0_ha_state_update_defaults { M0_HA_STATE_UPDATE_LIMIT = 1024 }

enum	{ M0_HA_NVEC_SET, M0_HA_NVEC_GET }

Functions
static bool	note_invariant (const struct m0_ha_nvec *note, bool known)

M0_INTERNAL int	m0_ha_state_get (struct m0_ha_nvec note, struct m0_chan chan)

M0_INTERNAL void	m0_ha_state_set (const struct m0_ha_nvec *note)

M0_INTERNAL void	m0_ha_local_state_set (const struct m0_ha_nvec *nvec)

static void	ha_state_single_fop_data_free (struct m0_fop *fop)

static void	ha_state_single_replied (struct m0_rpc_item *item)

M0_INTERNAL void	m0_ha_state_single_post (struct m0_ha_nvec *nvec)

static void	ha_state_accept (struct m0_confc confc, const struct m0_ha_nvec note, uint64_t ignore_same_state)

M0_INTERNAL void	m0_ha_state_accept (const struct m0_ha_nvec *note, bool ignore_same_state)

M0_INTERNAL void	m0_ha_msg_accept (const struct m0_ha_msg msg, struct m0_ha_link hl)

M0_INTERNAL uint64_t	m0_ha_msg_nvec_send (const struct m0_ha_nvec nvec, uint64_t id_of_get, bool ignore_same_state, int direction, struct m0_ha_link hl)

	M0_TL_DESCR_DEFINE (ha_gets, "m0_ha_note_handler::hmh_gets", static, struct ha_note_handler_request, hsg_tlink, hsg_magic, 20, 21)

	M0_TL_DEFINE (ha_gets, static, struct ha_note_handler_request)

static void	ha_note_handler_msg (struct m0_ha_handler hh, struct m0_ha ha, struct m0_ha_link hl, struct m0_ha_msg msg, uint64_t tag, void *data)

M0_INTERNAL int	m0_ha_note_handler_init (struct m0_ha_note_handler hnh, struct m0_ha_dispatcher hd)

M0_INTERNAL void	m0_ha_note_handler_fini (struct m0_ha_note_handler *hnh)

M0_INTERNAL uint64_t	m0_ha_note_handler_add (struct m0_ha_note_handler hnh, struct m0_ha_nvec nvec_req, struct m0_chan *chan)

M0_INTERNAL void	m0_ha_note_handler_signal (struct m0_ha_note_handler hnh, struct m0_ha_nvec nvec_rep, uint64_t id)

M0_INTERNAL const char *	m0_ha_state2str (enum m0_ha_obj_state state)

struct m0_ha_note	M0_XCA_DOMAIN (rpc)

M0_INTERNAL void	m0_conf_ha_callback (struct m0_conf_obj *obj)

M0_INTERNAL int	m0_ha_state_fop_init (void)

M0_INTERNAL void	m0_ha_state_fop_fini (void)

Variables
struct m0_rpc_item_ops	ha_ri_ops

struct m0_chan *	hsg_wait_chan

struct m0_tlink	hsg_tlink

struct m0_ha_nvec *	hsg_nvec

uint64_t	hsg_magic

uint64_t	hsg_id

struct m0_ha_state_single	M0_XCA_DOMAIN

struct m0_fop_type	m0_ha_state_get_fopt

struct m0_fop_type	m0_ha_state_get_rep_fopt

struct m0_fop_type	m0_ha_state_set_fopt

const struct m0_fom_type_ops *	m0_ha_state_get_fom_type_ops

const struct m0_fom_type_ops *	m0_ha_state_set_fom_type_ops

Detailed Description

TODO handle memory allocation failure in m0_ha_note_handler_add()

TODO update

This module defines protocols and functions used to communicate HA-related events between HA and Motr core.

Any HA-related event is represented as a state change of a configuration object. Configuration objects are stored in the Motr configuration data-base hosted by confd services and accessible to Motr and HA instances through confc module. A configuration object is identified by a unique 128-bit identifier.

HA-related state of a configuration object is represented by enum m0_ha_obj_state. It is important to understand that this state is not stored in confd. confd stores the "basic" information describing the nomenclature of system elements (nodes, services, devices, pools, etc.) and their relationships. HA maintains additional state on top of confd, which describes the run-time behaviour of configuration elements.

Among other things, confd stores, for certain types of objects, their "delegation pointers". A delegation pointer of an object X is some object Y that should be used when X fails. For example, to organise a fail-over pair, 2 services should have delegation pointers set to each other. The delegation pointer of a pool points to the pool to which writes should be re-directed in case of an NBA event. When an object fails, the chain formed by delegation pointers is followed until a usable object is found. If the chain is exhausted before a usable object is found, a system error is declared. All consecutive attempts to use the object would return the error until HA state or confd state changes.

Use cases

0. Motr initialisation

On startup, Motr instance connects to confd and populates its local confc with configuration objects. Then, Motr instance calls m0_ha_state_get() (one or more times). This function accepts as an input parameter a vector (m0_ha_nvec) that identifies objects for which state is queried (m0_ha_note::no_state field is ignored). m0_ha_state_get() constructs a fop of m0_ha_state_get_fopt type with nvec as data and sends it to the local HA instance (via supplied session).

HA replies with the same vector with m0_ha_note::no_state fields set. m0_ha_state_get() stores received object states in confc and notifies the caller about completion through the supplied channel.

Motr core notifies HA about failure.

On detecting a failure, Motr core calls m0_ha_state_set(), which takes as an input an nvec (with filled states), describing the failures, constructs m0_ha_state_set_fopt fop and sends it to the local HA instance via supplied session. HA replies with generic fop reply (m0_fop_generic_reply).

Note: that there is a separate mechanism, based on m0ctl, which is used to notify HA about failures which cannot be reported through RPC.

HA notifies Motr about failure.

When HA agrees about a failure, it sends to each Motr instance a m0_ha_state_set_fopt fop. Motr replies with generic fop reply.

m0_ha_state_accept() is called when a m0_ha_state_set_fopt fop is received.

Macro Definition Documentation

◆ M0_NVEC_PRINT

#define M0_NVEC_PRINT	(	nvec_,
		label,
		level
	)

Value:

({                        \
        int i;                                                    \
        struct m0_ha_nvec *nvec = nvec_;                          \
        char *lbl = label;                                        \
        for (i = 0; i < nvec->nv_nr; i++) {                       \
                M0_LOG(level, "%s [%d] " FID_F ", (%d)", lbl, i,  \
                       FID_P(&nvec->nv_note[i].no_id),            \
                       nvec->nv_note[i].no_state);                \
        }                                                         \
})

Definition at line 268 of file note.h.

◆ M0_TRACE_SUBSYSTEM

#define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_HA

Definition at line 31 of file note.c.

Enumeration Type Documentation

◆ anonymous enum

anonymous enum

Enumerator
M0_HA_NVEC_SET
M0_HA_NVEC_GET

Definition at line 215 of file note.h.

◆ m0_ha_obj_state

enum m0_ha_obj_state

Enumeration of possible object states.

Enumerator
M0_NC_UNKNOWN	Object state is unknown.
M0_NC_ONLINE	Object can be used normally.
M0_NC_FAILED	Object has experienced a permanent failure and cannot be recovered.
M0_NC_TRANSIENT	Object is experiencing a temporary failure. Halon will notify Motr when the object is available for use again.
M0_NC_REPAIR	This state is only applicable to the pool objects. In this state, the pool is undergoing repair, i.e., the process of reconstructing data lost due to a failure and storing them in spare space.
M0_NC_REPAIRED	This state is only applicable to the pool objects. In this state, the pool device has completed sns repair. Its data is re-constructed on its corresponding spare space.
M0_NC_REBALANCE	This state is only applicable to the pool objects. Rebalance process is complementary to repair: previously reconstructed data is being copied from spare space to the replacement storage.
M0_NC_DTM_RECOVERING	Recovery is triggered by the incoming NC_DTM_RECOVERING HA state message. During this state, processes iterate over their DTM logs, pack elements of them, TXRs, into REDO messages and send REDO messages to corresponding participants of TXR. Any subsequent failure of the process in the cluster restarts the recovery process on all alive participants. DTM_RECOVERING state is transitive for client applications and mkfs, meaning that transition from NC_DTM_RECOVERING to NC_ONLINE is immediate. For m0ds (CAS-, IO-services) indicates that the process waits for the completion of ongoing DTM recovery process. When the client learns the process in DTM_RECOVERING state it treats the process as read-only, at least for the meta-data, and skips sending modification requests to it.
M0_NC_NR

Definition at line 119 of file note.h.

◆ m0_ha_state_update_defaults

enum m0_ha_state_update_defaults

Enumerator
M0_HA_STATE_UPDATE_LIMIT

Definition at line 211 of file note.h.

Function Documentation

◆ ha_note_handler_msg()

static void ha_note_handler_msg	(	struct m0_ha_handler *	hh,
		struct m0_ha *	ha,
		struct m0_ha_link *	hl,
		struct m0_ha_msg *	msg,
		uint64_t	tag,
		void *	data
	)

static

Definition at line 331 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ha_state_accept()

static void ha_state_accept	(	struct m0_confc *	confc,
		const struct m0_ha_nvec *	note,
		uint64_t	ignore_same_state
	)

static

Callback used in m0_ha_state_accept(). Updates HA states for particular confc instance during iteration through HA clients list.

For internal details see comments provided for m0_ha_state_accept().

Definition at line 149 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ha_state_single_fop_data_free()

static void ha_state_single_fop_data_free ( struct m0_fop * fop )

static

Definition at line 111 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ha_state_single_replied()

static void ha_state_single_replied ( struct m0_rpc_item * item )

static

Definition at line 122 of file note.c.

Here is the call graph for this function:

◆ m0_conf_ha_callback()

M0_INTERNAL void m0_conf_ha_callback ( struct m0_conf_obj * obj )

◆ m0_ha_local_state_set()

M0_INTERNAL void m0_ha_local_state_set ( const struct m0_ha_nvec * nvec )

Notify local HA about state of configuration objects.

Definition at line 105 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_msg_accept()

M0_INTERNAL void m0_ha_msg_accept	(	const struct m0_ha_msg *	msg,
		struct m0_ha_link *	hl
	)

Definition at line 196 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_msg_nvec_send()

M0_INTERNAL uint64_t m0_ha_msg_nvec_send	(	const struct m0_ha_nvec *	nvec,
		uint64_t	id_of_get,
		bool	ignore_same_state,
		int	direction,
		struct m0_ha_link *	hl
	)

Definition at line 279 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_note_handler_add()

M0_INTERNAL uint64_t m0_ha_note_handler_add	(	struct m0_ha_note_handler *	hnh,
		struct m0_ha_nvec *	nvec_req,
		struct m0_chan *	chan
	)

Definition at line 382 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_note_handler_fini()

M0_INTERNAL void m0_ha_note_handler_fini ( struct m0_ha_note_handler * hnh )

Definition at line 364 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_note_handler_init()

M0_INTERNAL int m0_ha_note_handler_init	(	struct m0_ha_note_handler *	hnh,
		struct m0_ha_dispatcher *	hd
	)

Definition at line 345 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_note_handler_signal()

M0_INTERNAL void m0_ha_note_handler_signal	(	struct m0_ha_note_handler *	hnh,
		struct m0_ha_nvec *	nvec_rep,
		uint64_t	id
	)

Definition at line 401 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_state2str()

M0_INTERNAL const char * m0_ha_state2str ( enum m0_ha_obj_state state )

Definition at line 433 of file note.c.

Here is the caller graph for this function:

◆ m0_ha_state_accept()

M0_INTERNAL void m0_ha_state_accept	(	const struct m0_ha_nvec *	note,
		bool	ignore_same_state
	)

Incorporates received failure state changes in the cache of every confc instance registered with the global HA context (see m0_ha_client_add()).

Failure states of configuration objects are received from HA (not confd), but are stored in the same data-structure (conf client cache (m0_confc) consisting of configuration objects (m0_conf_obj)).

This function updates failures states of configuration objects according to "nvec".

Use cases:

this function is called when a Motr instance receives a failure state
update (m0_ha_state_set_fopt) from HA. This is a "push" notification
mechanism (HA sends updates) as opposed to m0_ha_state_get(), where Motr
"pulls" updates.

Note: : m0_conf_obj should be modified to hold HA-related state. Valery Vorotyntsev (valer.nosp@m.y.vo.nosp@m.rotyn.nosp@m.tsev.nosp@m.@seag.nosp@m.ate..nosp@m.com) is the configuration sub-system maintainer.

Precondition: m0_forall(i, note->nv_nr, note->nv_note[i].no_state != M0_NC_UNKNOWN && m0_conf_fid_is_valid(&note->nv_note[i].no_id) && ergo(M0_IN(note->nv_note[i].no_state, (M0_NC_REPAIR, M0_NC_REBALANCE)), m0_conf_fid_type(&note->nv_note[i].no_id) == &M0_CONF_POOL_TYPE))

Actual cache update is done by ha_state_accept() called on per-client basis in the course of iterating global HA context client list.

See m0_ha_msg_nvec::hmnv_ignore_same_state for ignore_same_state flag description.

Definition at line 189 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_state_fop_fini()

M0_INTERNAL void m0_ha_state_fop_fini ( void )

Definition at line 37 of file note_fops.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_state_fop_init()

M0_INTERNAL int m0_ha_state_fop_init ( void )

Definition at line 44 of file note_fops.c.

Here is the caller graph for this function:

◆ m0_ha_state_get()

M0_INTERNAL int m0_ha_state_get	(	struct m0_ha_nvec *	note,
		struct m0_chan *	chan
	)

Queries HA about the current the failure state for a set of objects.

Constructs a m0_ha_state_get_fopt from the "note" parameter and sends it to an HA instance, returning immediately after the fop is sent. When the reply (m0_ha_state_get_rep_fopt) is received, fills m0_ha_note::no_state from the reply and signals the provided channel.

On error (e.g., time-out), the function signals the channel, leaving m0_ha_note::no_state intact, so that the caller can determine that failure state wasn't fetched.

Use cases:

this function is called by a Motr instance when it joins the cluster
right after it received configuration information from the confd or
afterwards, when the instance wants to access an object for the first
time. The caller of m0_ha_state_get() is likely to call
m0_ha_state_accept() when the reply is received.

The caller must guarantee that on successful return from this function "note" parameter is valid (i.e., not deallocated) until the channel is signalled.

Precondition: m0_forall(i, note->nv_nr, note->nv_note[i].no_state == M0_NC_UNKNOWN && m0_conf_fid_is_valid(&note->nv_note[i].no_id))

Definition at line 74 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_state_set()

M0_INTERNAL void m0_ha_state_set ( const struct m0_ha_nvec * note )

Notifies HA about tentative change in the failure state for a set of objects.

Constructs a m0_ha_state_set_fopt from the "note" parameter, sends it to an HA instance and returns immediately. This function is used to report failures (and "unfailures") to HA.

Use cases:

this function is called by a Motr instance when it detects a
change in object behaviour. E.g., a timeout or increased
latency of a particular service or device.

Note that the failure state change is only tentative. It is up to HA to accumulate and analyse the stream of failure notifications and to declare failures. Specifically, a Motr instance should not assume that the object failure state changed, unless explicitly told so by HA.

Because failure state change is tentative, no error reporting is needed.

Precondition: m0_forall(i, note->nv_nr, note->nv_note[i].no_state != M0_NC_UNKNOWN && m0_conf_fid_is_valid(&note->nv_note[i].no_id) && ergo(M0_IN(note->nv_note[i].no_state, (M0_NC_REPAIR, M0_NC_REBALANCE)), m0_conf_fid_type(&note->nv_note[i].no_id) == &M0_CONF_POOL_TYPE))

Definition at line 95 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ m0_ha_state_single_post()

M0_INTERNAL void m0_ha_state_single_post ( struct m0_ha_nvec * nvec )

Asynchronous version of m0_ha_state_set() intended for posting single state.

Definition at line 134 of file note.c.

Here is the call graph for this function:

◆ M0_TL_DEFINE()

M0_TL_DEFINE	(	ha_gets	,
		static	,
		struct ha_note_handler_request
	)

◆ M0_TL_DESCR_DEFINE()

M0_TL_DESCR_DEFINE	(	ha_gets	,
		"m0_ha_note_handler::hmh_gets"	,
		static	,
		struct ha_note_handler_request	,
		hsg_tlink	,
		hsg_magic	,
		20	,
		21
	)

◆ M0_XCA_DOMAIN()

struct m0_ha_note M0_XCA_DOMAIN ( rpc )

◆ note_invariant()

static bool note_invariant	(	const struct m0_ha_nvec *	note,
		bool	known
	)

static

See also: : confc_fop_release()

Definition at line 58 of file note.c.

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ ha_ri_ops

struct m0_rpc_item_ops ha_ri_ops

Initial value:

= {
        .rio_replied = ha_state_single_replied,
}

Definition at line 130 of file note.c.

◆ hsg_id

uint64_t hsg_id

Definition at line 323 of file note.c.

◆ hsg_magic

uint64_t hsg_magic

Definition at line 322 of file note.c.

◆ hsg_nvec

struct m0_ha_nvec* hsg_nvec

Definition at line 321 of file note.c.

◆ hsg_tlink

struct m0_tlink hsg_tlink

Definition at line 320 of file note.c.

◆ hsg_wait_chan

struct m0_chan* hsg_wait_chan

Definition at line 319 of file note.c.

◆ m0_ha_state_get_fom_type_ops

const struct m0_fom_type_ops* m0_ha_state_get_fom_type_ops

Definition at line 191 of file note_foms.c.

◆ m0_ha_state_get_fopt

struct m0_fop_type m0_ha_state_get_fopt

Definition at line 33 of file note_fops.c.

◆ m0_ha_state_get_rep_fopt

struct m0_fop_type m0_ha_state_get_rep_fopt

Definition at line 34 of file note_fops.c.

◆ m0_ha_state_set_fom_type_ops

const struct m0_fom_type_ops* m0_ha_state_set_fom_type_ops

Definition at line 192 of file note_foms.c.

◆ m0_ha_state_set_fopt

struct m0_fop_type m0_ha_state_set_fopt

Definition at line 35 of file note_fops.c.

◆ M0_XCA_DOMAIN

struct m0_ha_note_handler M0_XCA_DOMAIN

Data Structures

Macros

Enumerations

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ M0_NVEC_PRINT

◆ M0_TRACE_SUBSYSTEM

Enumeration Type Documentation

◆ anonymous enum

◆ m0_ha_obj_state

◆ m0_ha_state_update_defaults

Function Documentation

◆ ha_note_handler_msg()

◆ ha_state_accept()

◆ ha_state_single_fop_data_free()

◆ ha_state_single_replied()

◆ m0_conf_ha_callback()

◆ m0_ha_local_state_set()

◆ m0_ha_msg_accept()

◆ m0_ha_msg_nvec_send()

◆ m0_ha_note_handler_add()

◆ m0_ha_note_handler_fini()

◆ m0_ha_note_handler_init()

◆ m0_ha_note_handler_signal()

◆ m0_ha_state2str()

◆ m0_ha_state_accept()

◆ m0_ha_state_fop_fini()

◆ m0_ha_state_fop_init()

◆ m0_ha_state_get()

◆ m0_ha_state_set()

◆ m0_ha_state_single_post()

◆ M0_TL_DEFINE()

◆ M0_TL_DESCR_DEFINE()

◆ M0_XCA_DOMAIN()

◆ note_invariant()

Variable Documentation

◆ ha_ri_ops

◆ hsg_id

◆ hsg_magic

◆ hsg_nvec

◆ hsg_tlink

◆ hsg_wait_chan

◆ m0_ha_state_get_fom_type_ops

◆ m0_ha_state_get_fopt

◆ m0_ha_state_get_rep_fopt

◆ m0_ha_state_set_fom_type_ops

◆ m0_ha_state_set_fopt

◆ M0_XCA_DOMAIN