net.h

Go to the documentation of this file.

/* -*- C -*- */
/*
 * Copyright (c) 2012-2020 Seagate Technology LLC and/or its Affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For any questions about this software or licensing,
 * please email opensource@seagate.com or cortx-questions@seagate.com.
 *
 */


#pragma once

#ifndef __MOTR_NET_NET_H__
#define __MOTR_NET_NET_H__

#include <stdarg.h>

#include "lib/rwlock.h"
#include "lib/list.h"
#include "lib/tlist.h"
#include "lib/queue.h"
#include "lib/refs.h"
#include "lib/chan.h"
#include "lib/cond.h"
#include "lib/mutex.h"
#include "lib/time.h"
#include "lib/thread.h"
#include "lib/vec.h"
#include "net/ip.h"
#include "net/net_otw_types.h"
#include "net/net_otw_types_xc.h"
#include "sm/sm.h"

/* import */
struct m0_bitmap;

/* export */
struct m0_net_xprt;
struct m0_net_xprt_ops;
struct m0_net_domain;
struct m0_net_transfer_mc;
struct m0_net_tm_event;
struct m0_net_tm_callbacks;
struct m0_net_end_point;
struct m0_net_buffer;
struct m0_net_buf_desc;
struct m0_net_buffer_event;
struct m0_net_buffer_callbacks;
struct m0_net_qstats;

#ifdef ENABLE_LIBFAB
#define M0_NET_XPRT_PREFIX_DEFAULT "libfab"
#define USE_LIBFAB                 1
#else
#define M0_NET_XPRT_PREFIX_DEFAULT "lnet"
#define USE_LIBFAB                 0
#endif

M0_INTERNAL int m0_net_init(void);

M0_INTERNAL void m0_net_fini(void);

enum {
        M0_NET_TM_RECV_QUEUE_DEF_LEN = 2,
};

struct m0_net_xprt {
        const char                   *nx_name;
        const struct m0_net_xprt_ops *nx_ops;
};

struct m0_net_xprt_ops {
        int  (*xo_dom_init)(const struct m0_net_xprt *xprt,
                            struct m0_net_domain *dom);
        void (*xo_dom_fini)(struct m0_net_domain *dom);

        int (*xo_tm_init)(struct m0_net_transfer_mc *tm);

        int (*xo_tm_confine)(struct m0_net_transfer_mc *tm,
                             const struct m0_bitmap *processors);

        int (*xo_tm_start)(struct m0_net_transfer_mc *tm, const char *addr);

        int (*xo_tm_stop)(struct m0_net_transfer_mc *tm, bool cancel);

        void (*xo_tm_fini)(struct m0_net_transfer_mc *tm);

        int (*xo_end_point_create)(struct m0_net_end_point **epp,
                                   struct m0_net_transfer_mc *tm,
                                   const char *addr);

        int (*xo_buf_register)(struct m0_net_buffer *nb);

        void (*xo_buf_deregister)(struct m0_net_buffer *nb);

        int (*xo_buf_add)(struct m0_net_buffer *nb);

        void (*xo_buf_del)(struct m0_net_buffer *nb);

        int  (*xo_bev_deliver_sync)(struct m0_net_transfer_mc *tm);

        void (*xo_bev_deliver_all)(struct m0_net_transfer_mc *tm);

        bool (*xo_bev_pending)(struct m0_net_transfer_mc *tm);

        void (*xo_bev_notify)(struct m0_net_transfer_mc *tm,
                              struct m0_chan *chan);

        m0_bcount_t (*xo_get_max_buffer_size)(const struct m0_net_domain *dom);

        m0_bcount_t (*xo_get_max_buffer_segment_size)(const struct m0_net_domain
                                                      *dom);

        int32_t (*xo_get_max_buffer_segments)(const struct m0_net_domain *dom);

        m0_bcount_t (*xo_get_max_buffer_desc_size)(const struct m0_net_domain
                                                   *dom);

        m0_bcount_t (*xo_rpc_max_seg_size) (struct m0_net_domain *ndom);

        uint32_t    (*xo_rpc_max_segs_nr)  (struct m0_net_domain *ndom);

        m0_bcount_t (*xo_rpc_max_msg_size) (struct m0_net_domain *ndom,
                                            m0_bcount_t rpc_size);

        uint32_t    (*xo_rpc_max_recv_msgs)(struct m0_net_domain *ndom,
                                            m0_bcount_t rpc_size);
};

struct m0_net_domain {
        struct m0_mutex     nd_mutex;

        struct m0_list      nd_registered_bufs;

        struct m0_list      nd_tms;

        void               *nd_xprt_private;

        const struct m0_net_xprt *nd_xprt;

        struct m0_tlink     nd_app_linkage;

        uint32_t            nd_get_max_buffer_segments;

        m0_bcount_t         nd_get_max_buffer_segment_size;

        m0_bcount_t         nd_get_max_buffer_size;

        m0_bcount_t         nd_get_max_buffer_desc_size;

        uint64_t            nd_magix;
};

int m0_net_domain_init(struct m0_net_domain *dom,
                        const struct m0_net_xprt *xprt);

void m0_net_domain_fini(struct m0_net_domain *dom);

M0_INTERNAL m0_bcount_t m0_net_domain_get_max_buffer_size(struct m0_net_domain
                                                          *dom);

M0_INTERNAL m0_bcount_t m0_net_domain_get_max_buffer_segment_size(struct
                                                                  m0_net_domain
                                                                  *dom);

M0_INTERNAL m0_bcount_t m0_net_domain_get_max_buffer_desc_size(struct
                                                               m0_net_domain
                                                               *dom);

M0_INTERNAL int32_t m0_net_domain_get_max_buffer_segments(struct m0_net_domain
                                                          *dom);

struct m0_net_end_point {
        uint64_t                   nep_magix;
        struct m0_ref              nep_ref;
        struct m0_net_transfer_mc *nep_tm;
        struct m0_tlink            nep_tm_linkage;
        const char                *nep_addr;
        void                      *nep_xprt_pvt;
};

M0_TL_DESCR_DECLARE(m0_nep, M0_EXTERN);
M0_TL_DECLARE(m0_nep, M0_INTERNAL, struct m0_net_end_point);

M0_INTERNAL int m0_net_end_point_create(struct m0_net_end_point **epp,
                                        struct m0_net_transfer_mc *tm,
                                        const char *addr);

M0_INTERNAL void m0_net_end_point_get(struct m0_net_end_point *ep);

void m0_net_end_point_put(struct m0_net_end_point *ep);

enum m0_net_queue_type {
        M0_NET_QT_MSG_RECV = 0,

        M0_NET_QT_MSG_SEND,

        M0_NET_QT_PASSIVE_BULK_RECV,

        M0_NET_QT_PASSIVE_BULK_SEND,

        M0_NET_QT_ACTIVE_BULK_RECV,

        M0_NET_QT_ACTIVE_BULK_SEND,

        M0_NET_QT_NR
};

enum m0_net_tm_state {
        M0_NET_TM_UNDEFINED = 0, 
        M0_NET_TM_INITIALIZED,   
        M0_NET_TM_STARTING,      
        M0_NET_TM_STARTED,       
        M0_NET_TM_STOPPING,      
        M0_NET_TM_STOPPED,       
        M0_NET_TM_FAILED         
};

enum m0_net_tm_ev_type {
        M0_NET_TEV_ERROR = 0,      
        M0_NET_TEV_STATE_CHANGE,   
        M0_NET_TEV_DIAGNOSTIC,     
        M0_NET_TEV_NR
};

struct m0_net_tm_event {
        enum m0_net_tm_ev_type     nte_type;

        struct m0_net_transfer_mc *nte_tm;

        m0_time_t                  nte_time;

        int32_t                    nte_status;

        enum m0_net_tm_state       nte_next_state;

        struct m0_net_end_point   *nte_ep;

        void                      *nte_payload;
};

struct m0_net_tm_callbacks {
        void (*ntc_event_cb)(const struct m0_net_tm_event *ev);
};

struct m0_net_qstats {
        uint64_t        nqs_num_adds;

        uint64_t        nqs_num_dels;

        uint64_t        nqs_num_s_events;

        uint64_t        nqs_num_f_events;

        m0_time_t       nqs_time_in_queue;

        uint64_t        nqs_total_bytes;

        uint64_t        nqs_max_bytes;
};

struct m0_net_transfer_mc {
        const struct m0_net_tm_callbacks *ntm_callbacks;

        enum m0_net_tm_state        ntm_state;

        struct m0_sm_group          ntm_group;

#define ntm_mutex ntm_group.s_lock

        uint32_t                    ntm_callback_counter;

        struct m0_net_domain       *ntm_dom;

        struct m0_tl                ntm_end_points;

        struct m0_net_end_point    *ntm_ep;

        struct m0_chan              ntm_chan;

        struct m0_tl                ntm_q[M0_NET_QT_NR];

        struct m0_net_qstats        ntm_qstats[M0_NET_QT_NR];

        struct m0_list_link         ntm_dom_linkage;

        void                       *ntm_xprt_private;

        bool                        ntm_bev_auto_deliver;

        struct m0_net_buffer_pool  *ntm_recv_pool;

        const struct m0_net_buffer_callbacks *ntm_recv_pool_callbacks;

        uint32_t                    ntm_recv_queue_min_length;

        struct m0_atomic64          ntm_recv_queue_deficit;

        uint32_t                    ntm_pool_colour;

        m0_bcount_t                 ntm_recv_queue_min_recv_size;

        uint32_t                    ntm_recv_queue_max_recv_msgs;
};

M0_INTERNAL int m0_net_tm_init(struct m0_net_transfer_mc *tm,
                               struct m0_net_domain      *dom);

M0_INTERNAL void m0_net_tm_fini(struct m0_net_transfer_mc *tm);

M0_INTERNAL int m0_net_tm_confine(struct m0_net_transfer_mc *tm,
                                  const struct m0_bitmap *processors);

M0_INTERNAL int m0_net_tm_start(struct m0_net_transfer_mc *tm,
                                const char *addr);

M0_INTERNAL int m0_net_tm_stop(struct m0_net_transfer_mc *tm, bool abort);

M0_INTERNAL int m0_net_tm_stats_get(struct m0_net_transfer_mc *tm,
                                    enum m0_net_queue_type qtype,
                                    struct m0_net_qstats *qs, bool reset);

M0_INTERNAL void m0_net_tm_event_post(const struct m0_net_tm_event *ev);

M0_INTERNAL void m0_net_tm_colour_set(struct m0_net_transfer_mc *tm,
                                      uint32_t colour);

M0_INTERNAL uint32_t m0_net_tm_colour_get(struct m0_net_transfer_mc *tm);

M0_INTERNAL int m0_net_tm_pool_attach(struct m0_net_transfer_mc *tm,
                                      struct m0_net_buffer_pool *bufpool,
                                      const struct m0_net_buffer_callbacks
                                      *callbacks, m0_bcount_t min_recv_size,
                                      uint32_t max_recv_msgs,
                                      uint32_t min_recv_queue_len);

M0_INTERNAL void m0_net_tm_pool_length_set(struct m0_net_transfer_mc *tm,
                                           uint32_t len);

M0_INTERNAL void
m0_net_domain_buffer_pool_not_empty(struct m0_net_buffer_pool *pool);

struct m0_net_buffer_event {
        struct m0_net_buffer      *nbe_buffer;

        m0_time_t                  nbe_time;

        int32_t                    nbe_status;

        m0_bcount_t                nbe_length;

        m0_bindex_t                nbe_offset;

        struct m0_net_end_point   *nbe_ep;
};

typedef void (*m0_net_buffer_cb_proc_t)(const struct m0_net_buffer_event *ev);

struct m0_net_buffer_callbacks {
        m0_net_buffer_cb_proc_t nbc_cb[M0_NET_QT_NR];
};

enum m0_net_buf_flags {
        M0_NET_BUF_REGISTERED  = 1 << 0,
        M0_NET_BUF_QUEUED      = 1 << 1,
        M0_NET_BUF_IN_USE      = 1 << 2,
        M0_NET_BUF_CANCELLED   = 1 << 3,
        M0_NET_BUF_TIMED_OUT   = 1 << 4,
        M0_NET_BUF_RETAIN      = 1 << 5,
};

struct m0_net_buffer {
        struct m0_bufvec           nb_buffer;

        m0_bcount_t                nb_length;

        m0_bindex_t                nb_offset;

        struct m0_net_domain      *nb_dom;

        struct m0_net_transfer_mc *nb_tm;

        enum m0_net_queue_type     nb_qtype;

        const struct m0_net_buffer_callbacks *nb_callbacks;

        m0_time_t                  nb_timeout;

        m0_time_t                  nb_add_time;

        struct m0_net_buf_desc     nb_desc;

        struct m0_net_end_point   *nb_ep;

        struct m0_tlink            nb_tm_linkage;

        struct m0_tlink            nb_lru;

        struct m0_tlink            nb_extern_linkage;

        uint64_t                   nb_magic;

        struct m0_list_link        nb_dom_linkage;

        void                      *nb_xprt_private;

        void                      *nb_app_private;

        uint64_t                   nb_flags;

        m0_bcount_t                nb_min_receive_size;

        uint32_t                   nb_max_receive_msgs;

        struct m0_net_buffer_pool *nb_pool;

        uint32_t                   nb_msgs_received;
};

M0_INTERNAL int m0_net_buffer_register(struct m0_net_buffer *buf,
                                       struct m0_net_domain *dom);

M0_INTERNAL void m0_net_buffer_deregister(struct m0_net_buffer *buf,
                                          struct m0_net_domain *dom);

M0_INTERNAL int m0_net_buffer_add(struct m0_net_buffer *buf,
                                  struct m0_net_transfer_mc *tm);

M0_INTERNAL bool m0_net_buffer_del(struct m0_net_buffer *buf,
                                   struct m0_net_transfer_mc *tm);

M0_INTERNAL void m0_net_buffer_event_post(const struct m0_net_buffer_event *ev);

M0_INTERNAL void m0_net_buffer_event_deliver_all(struct m0_net_transfer_mc *tm);

M0_INTERNAL int
m0_net_buffer_event_deliver_synchronously(struct m0_net_transfer_mc *tm);

M0_INTERNAL bool m0_net_buffer_event_pending(struct m0_net_transfer_mc *tm);

M0_INTERNAL void m0_net_buffer_event_notify(struct m0_net_transfer_mc *tm,
                                            struct m0_chan *chan);

M0_INTERNAL int m0_net_desc_copy(const struct m0_net_buf_desc *from_desc,
                                 struct m0_net_buf_desc *to_desc);

M0_INTERNAL void m0_net_desc_free(struct m0_net_buf_desc *desc);

/* Descriptor for the tlist of buffers. */
M0_TL_DESCR_DECLARE(m0_net_pool, M0_EXTERN);
M0_TL_DESCR_DECLARE(m0_net_tm, M0_EXTERN);
M0_TL_DECLARE(m0_net_pool, M0_INTERNAL, struct m0_net_buffer);
M0_TL_DECLARE(m0_net_tm, M0_INTERNAL, struct m0_net_buffer);

#ifndef __KERNEL__

M0_INTERNAL bool m0_net_endpoint_is_valid(const char *endpoint);

#endif /* __KERNEL__ */

M0_INTERNAL void m0_net_xprt_default_set(const struct m0_net_xprt *xprt);
M0_INTERNAL void m0_net_xprt_register(const struct m0_net_xprt *xprt);
M0_INTERNAL void m0_net_xprt_deregister(const struct m0_net_xprt *xprt);
struct m0_net_xprt *m0_net_xprt_default_get(void);
struct m0_net_xprt **m0_net_all_xprt_get(void);
int m0_net_xprt_nr(void);
M0_INTERNAL void m0_net_print_xprt(void);
M0_INTERNAL bool m0_net_check_xprt(const struct m0_net_xprt *xprt);

M0_INTERNAL m0_bcount_t default_xo_rpc_max_seg_size(struct m0_net_domain *ndom);
M0_INTERNAL uint32_t    default_xo_rpc_max_segs_nr(struct m0_net_domain *ndom);
M0_INTERNAL m0_bcount_t default_xo_rpc_max_msg_size(struct m0_net_domain *ndom,
                                                    m0_bcount_t rpc_size);
M0_INTERNAL uint32_t    default_xo_rpc_max_recv_msgs(struct m0_net_domain *ndom,
                                                     m0_bcount_t rpc_size);


#endif /* __MOTR_NET_NET_H__ */

/*
 *  Local variables:
 *  c-indentation-style: "K&R"
 *  c-basic-offset: 8
 *  tab-width: 8
 *  fill-column: 79
 *  scroll-step: 1
 *  End:
 */