klnet_core.h

Go to the documentation of this file.

/* -*- C -*- */
/*
 * Copyright (c) 2012-2020 Seagate Technology LLC and/or its Affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For any questions about this software or licensing,
 * please email opensource@seagate.com or cortx-questions@seagate.com.
 *
 */


#pragma once

#ifndef __MOTR_NET_KLNET_CORE_H__
#define __MOTR_NET_KLNET_CORE_H__

#include "lib/semaphore.h"
#include "lib/tlist.h"
#include "net/lnet/lnet_core.h"

#include <linux/spinlock.h>

/* LUSTRE_VERSION_CODE, OBD_OCD_VERSION */
#if M0_LUSTRE_VERSION < 2110
#include <lustre_ver.h>
#else
#include <lustre/lustre_ver.h>
#include <lustre/lustre_idl.h>
#endif

#if LUSTRE_VERSION_CODE > OBD_OCD_VERSION(2, 10, 51, 0)
typedef struct lnet_eq lnet_eq_t;
typedef struct lnet_event lnet_event_t;
typedef struct lnet_handle_eq lnet_handle_eq_t;
typedef struct lnet_handle_md lnet_handle_md_t;
typedef struct lnet_handle_me lnet_handle_me_t;
typedef struct lnet_libhandle lnet_libhandle_t;
typedef struct lnet_libmd lnet_libmd_t;
typedef struct lnet_md lnet_md_t;
typedef struct lnet_me lnet_me_t;
typedef struct lnet_msg lnet_msg_t;
typedef struct lnet_process_id lnet_process_id_t;
typedef enum lnet_event_kind lnet_event_kind_t;
#endif

enum {
        M0_NET_LNET_MAX_PORTALS     = 64, 
#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 2, 54, 0)
        M0_NET_LNET_EQ_SIZE         = 8,
#else
        M0_NET_LNET_EQ_SIZE         = 0,
#endif

        M0_NET_LNET_PORTAL_MASK     = M0_NET_LNET_BUFFER_ID_MAX,
};

struct nlx_kcore_ops;

struct nlx_kcore_domain {
        uint64_t                      kd_magic;

        struct nlx_core_kmem_loc      kd_cd_loc;

        struct m0_mutex               kd_drv_mutex;

        struct nlx_kcore_ops         *kd_drv_ops;

        struct m0_tl                  kd_drv_tms;

        struct m0_tl                  kd_drv_bufs;
};

struct nlx_kcore_transfer_mc {
        uint64_t                      ktm_magic;

        struct nlx_core_kmem_loc      ktm_ctm_loc;

        struct m0_tlink               ktm_tm_linkage;

        struct m0_tlink               ktm_drv_linkage;

        struct m0_tl                  ktm_drv_bevs;

        struct nlx_core_ep_addr       ktm_addr;

        spinlock_t                    ktm_bevq_lock;

        wait_queue_head_t             ktm_wq;

        lnet_handle_eq_t              ktm_eqh;
        unsigned                      _debug_;
};

struct nlx_kcore_buffer {
        uint64_t                      kb_magic;

        struct nlx_core_kmem_loc      kb_cb_loc;

        struct nlx_kcore_transfer_mc *kb_ktm;

        struct m0_tlink               kb_drv_linkage;

        nlx_core_opaque_ptr_t         kb_buffer_id;

        enum m0_net_queue_type        kb_qtype;

        m0_time_t                     kb_add_time;

        lnet_kiov_t                  *kb_kiov;

        size_t                        kb_kiov_len;

        size_t                        kb_kiov_adj_idx;

        unsigned                      kb_kiov_orig_len;

        lnet_handle_md_t              kb_mdh;

        bool                          kb_ooo_reply;

        unsigned                      kb_ooo_mlength;

        int                           kb_ooo_status;

        unsigned                      kb_ooo_offset;
};

struct nlx_kcore_buffer_event {
        uint64_t                      kbe_magic;

        struct nlx_core_kmem_loc      kbe_bev_loc;

        struct m0_tlink               kbe_drv_linkage;
};

#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 9, 54, 0)
#define LNetInvalidateMDHandle(h)  LNetInvalidateHandle(h)
#define LNetMDHandleIsInvalid(h)   LNetHandleIsInvalid(h)
#else
#define LNetHandleIsEqual(h1, h2)  ((h1).cookie == (h2).cookie)
#define LNetHandleIsInvalid(h)     ((h).cookie == LNET_WIRE_HANDLE_COOKIE_NONE)
#endif

typedef int (*nlx_kcore_queue_op_t)(struct nlx_kcore_transfer_mc *ktm,
                                    struct nlx_core_buffer *cb,
                                    struct nlx_kcore_buffer *kb);


struct nlx_kcore_ops {
        int (*ko_dom_init)(struct nlx_kcore_domain *kd,
                           struct nlx_core_domain *cd);
        void (*ko_dom_fini)(struct nlx_kcore_domain *kd,
                            struct nlx_core_domain *cd);

        int (*ko_buf_register)(struct nlx_kcore_domain *kd,
                               nlx_core_opaque_ptr_t buffer_id,
                               struct nlx_core_buffer *cb,
                               struct nlx_kcore_buffer *kb);

        void (*ko_buf_deregister)(struct nlx_core_buffer *cb,
                                  struct nlx_kcore_buffer *kb);

        int (*ko_tm_start)(struct nlx_kcore_domain      *kd,
                           struct nlx_core_transfer_mc  *ctm,
                           struct nlx_kcore_transfer_mc *ktm);

        void (*ko_tm_stop)(struct nlx_core_transfer_mc *ctm,
                           struct nlx_kcore_transfer_mc *ktm);

        nlx_kcore_queue_op_t ko_buf_msg_recv;

        nlx_kcore_queue_op_t ko_buf_msg_send;

        nlx_kcore_queue_op_t ko_buf_active_recv;

        nlx_kcore_queue_op_t ko_buf_active_send;

        nlx_kcore_queue_op_t ko_buf_passive_recv;

        nlx_kcore_queue_op_t ko_buf_passive_send;

        int (*ko_buf_del)(struct nlx_kcore_transfer_mc *ktm,
                          struct nlx_kcore_buffer *kb);

        int (*ko_buf_event_wait)(struct nlx_core_transfer_mc *ctm,
                                 struct nlx_kcore_transfer_mc *ktm,
                                 m0_time_t timeout);
};

static void nlx_core_kmem_loc_set(struct nlx_core_kmem_loc *loc,
                                  struct page* pg, uint32_t off);

static bool nlx_kcore_domain_invariant(const struct nlx_kcore_domain *kd);
static bool nlx_kcore_buffer_invariant(const struct nlx_kcore_buffer *kcb);
static bool nlx_kcore_buffer_event_invariant(
                                     const struct nlx_kcore_buffer_event *kbe);
static bool nlx_kcore_tm_invariant(const struct nlx_kcore_transfer_mc *kctm);
static int nlx_kcore_kcore_dom_init(struct nlx_kcore_domain *kd);
static void nlx_kcore_kcore_dom_fini(struct nlx_kcore_domain *kd);
static int nlx_kcore_buffer_kla_to_kiov(struct nlx_kcore_buffer *kb,
                                        const struct m0_bufvec *bvec);
static int nlx_kcore_buffer_uva_to_kiov(struct nlx_kcore_buffer *kb,
                                        const struct m0_bufvec *bvec);
static bool nlx_kcore_kiov_invariant(const lnet_kiov_t *k, size_t len);

#define NLX_PAGE_OFFSET(addr) ((addr) & ~PAGE_MASK)
 /* KLNetCore */

#endif /* __MOTR_NET_KLNET_CORE_H__ */

/*
 *  Local variables:
 *  c-indentation-style: "K&R"
 *  c-basic-offset: 8
 *  tab-width: 8
 *  fill-column: 79
 *  scroll-step: 1
 *  End:
 */