klnet_drv.c

Go to the documentation of this file.

/* -*- C -*- */
/*
 * Copyright (c) 2012-2020 Seagate Technology LLC and/or its Affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For any questions about this software or licensing,
 * please email opensource@seagate.com or cortx-questions@seagate.com.
 *
 */


#include <linux/miscdevice.h>

M0_BASSERT(sizeof(struct nlx_xo_domain) < PAGE_SIZE);
M0_BASSERT(sizeof(struct nlx_xo_transfer_mc) < PAGE_SIZE);
M0_BASSERT(sizeof(struct nlx_xo_buffer) < PAGE_SIZE);
M0_BASSERT(sizeof(struct nlx_core_buffer_event) < PAGE_SIZE);

/* LNET_NIDSTR_SIZE is only defined in the kernel */
M0_BASSERT(M0_NET_LNET_NIDSTR_SIZE == LNET_NIDSTR_SIZE);

static int nlx_dev_ioctl_dom_init(struct nlx_kcore_domain *kd,
                                  struct m0_lnet_dev_dom_init_params *p)

{
        struct page *pg;
        struct nlx_core_domain *cd;
        uint32_t off = NLX_PAGE_OFFSET((unsigned long) p->ddi_cd);
        int rc;

        m0_mutex_lock(&kd->kd_drv_mutex);
        if (!nlx_core_kmem_loc_is_empty(&kd->kd_cd_loc)) {
                m0_mutex_unlock(&kd->kd_drv_mutex);
                return M0_ERR(-EBADR);
        }
        if (off + sizeof *cd > PAGE_SIZE)
                return M0_ERR(-EBADR);

        /* note: these calls can block */
        down_read(&current->mm->mmap_sem);
        rc = WRITABLE_USER_PAGE_GET(p->ddi_cd, pg);
        up_read(&current->mm->mmap_sem);

        if (rc >= 0) {
                M0_ASSERT(rc == 1);
                nlx_core_kmem_loc_set(&kd->kd_cd_loc, pg, off);
                cd = nlx_kcore_core_domain_map(kd);
                rc = kd->kd_drv_ops->ko_dom_init(kd, cd);
                if (rc == 0) {
                        p->ddi_max_buffer_size =
                            nlx_core_get_max_buffer_size(cd);
                        p->ddi_max_buffer_segment_size =
                            nlx_core_get_max_buffer_segment_size(cd);
                        p->ddi_max_buffer_segments =
                            nlx_core_get_max_buffer_segments(cd);
                        M0_ASSERT(nlx_kcore_domain_invariant(kd));
                        M0_ASSERT(!nlx_core_kmem_loc_is_empty(&kd->kd_cd_loc));
                }
                nlx_kcore_core_domain_unmap(kd);
        }
        m0_mutex_unlock(&kd->kd_drv_mutex);
        return M0_RC(rc);
}

static int nlx_dev_ioctl_buf_register(struct nlx_kcore_domain *kd,
                                      struct m0_lnet_dev_buf_register_params *p)
{
        struct page *pg;
        struct nlx_core_buffer *cb;
        struct nlx_kcore_buffer *kb;
        uint32_t off = NLX_PAGE_OFFSET((unsigned long) p->dbr_lcbuf);
        m0_bcount_t sz;
        void **buf;
        m0_bcount_t *count;
        int n = p->dbr_bvec.ov_vec.v_nr;
        int rc;

        if (off + sizeof *cb > PAGE_SIZE)
                return M0_ERR(-EBADR);
        if (p->dbr_buffer_id == 0)
                return M0_ERR(-EBADR);
        NLX_ALLOC_ARR(buf, n);
        if (buf == NULL)
                return M0_ERR(-ENOMEM);
        NLX_ALLOC_ARR(count, n);
        if (count == NULL) {
                rc = M0_ERR(-ENOMEM);
                goto fail_count;
        }

        sz = n * sizeof *buf;
        if (copy_from_user(buf, (void __user *) p->dbr_bvec.ov_buf, sz)) {
                rc = M0_ERR(-EFAULT);
                goto fail_copy;
        }
        sz = n * sizeof *count;
        if (copy_from_user(count,
                           (void __user *) p->dbr_bvec.ov_vec.v_count, sz)) {
                rc = M0_ERR(-EFAULT);
                goto fail_copy;
        }

        NLX_ALLOC_PTR(kb);
        if (kb == NULL) {
                rc = M0_ERR(-ENOMEM);
                goto fail_copy;
        }
        kb->kb_magic = M0_NET_LNET_KCORE_BUF_MAGIC;

        down_read(&current->mm->mmap_sem);
        rc = WRITABLE_USER_PAGE_GET(p->dbr_lcbuf, pg);
        up_read(&current->mm->mmap_sem);
        if (rc < 0)
                goto fail_page;
        nlx_core_kmem_loc_set(&kb->kb_cb_loc, pg, off);
        cb = nlx_kcore_core_buffer_map(kb);
        if (cb->cb_magic != 0 || cb->cb_buffer_id != 0 || cb->cb_kpvt != NULL) {
                rc = -EBADR;
                goto fail_cb;
        }
        rc = kd->kd_drv_ops->ko_buf_register(kd, p->dbr_buffer_id, cb, kb);
        if (rc != 0)
                goto fail_cb;

        p->dbr_bvec.ov_buf = buf;
        p->dbr_bvec.ov_vec.v_count = count;
        rc = nlx_kcore_buffer_uva_to_kiov(kb, &p->dbr_bvec);
        p->dbr_bvec.ov_buf = NULL;
        p->dbr_bvec.ov_vec.v_count = NULL;
        if (rc != 0)
                goto fail_kiov;

        M0_ASSERT(kb->kb_kiov != NULL && kb->kb_kiov_len > 0);
        M0_POST(nlx_kcore_buffer_invariant(cb->cb_kpvt));
        nlx_kcore_core_buffer_unmap(kb);
        m0_mutex_lock(&kd->kd_drv_mutex);
        drv_bufs_tlist_add(&kd->kd_drv_bufs, kb);
        m0_mutex_unlock(&kd->kd_drv_mutex);
        m0_free(count);
        m0_free(buf);
        return 0;

fail_kiov:
        kd->kd_drv_ops->ko_buf_deregister(cb, kb);
fail_cb:
        nlx_kcore_core_buffer_unmap(kb);
        WRITABLE_USER_PAGE_PUT(kb->kb_cb_loc.kl_page);
fail_page:
        kb->kb_magic = 0;
        m0_free(kb);
fail_copy:
        m0_free(count);
fail_count:
        m0_free(buf);
        M0_ASSERT(rc < 0);
        return M0_RC(rc);
}

static void nlx_dev_buf_pages_unpin(const struct nlx_kcore_buffer *kb)
{
        size_t i;

        for (i = 0; i < kb->kb_kiov_len; ++i)
                WRITABLE_USER_PAGE_PUT(kb->kb_kiov[i].kiov_page);
}

static int nlx_dev_buf_deregister(struct nlx_kcore_domain *kd,
                                  struct nlx_kcore_buffer *kb)
{
        struct nlx_core_buffer *cb;

        if (!nlx_kcore_buffer_invariant(kb))
                return M0_ERR(-EBADR);
        m0_mutex_lock(&kd->kd_drv_mutex);
        drv_bufs_tlist_del(kb);
        m0_mutex_unlock(&kd->kd_drv_mutex);
        nlx_dev_buf_pages_unpin(kb);
        cb = nlx_kcore_core_buffer_map(kb);
        kd->kd_drv_ops->ko_buf_deregister(cb, kb);
        nlx_kcore_core_buffer_unmap(kb);
        WRITABLE_USER_PAGE_PUT(kb->kb_cb_loc.kl_page);
        m0_free(kb);
        return 0;
}

static int nlx_dev_ioctl_buf_deregister(struct nlx_kcore_domain *kd,
                                   struct m0_lnet_dev_buf_deregister_params *p)
{
        struct nlx_kcore_buffer *kb = p->dbd_kb;

        /* protect against user space passing invalid ptr */
        if (!virt_addr_valid(kb))
                return M0_ERR(-EBADR);
        return nlx_dev_buf_deregister(kd, kb);
}

static int nlx_dev_ioctl_buf_queue_op(
                                  const struct m0_lnet_dev_buf_queue_params *p,
                                  nlx_kcore_queue_op_t op)
{
        struct nlx_kcore_transfer_mc *ktm = p->dbq_ktm;
        struct nlx_kcore_buffer *kb = p->dbq_kb;
        struct nlx_core_buffer *cb;
        int rc;

        M0_PRE(op != NULL);
        if (!virt_addr_valid(ktm) || !virt_addr_valid(kb))
                return M0_ERR(-EBADR);
        if (!nlx_kcore_tm_invariant(ktm))
                return M0_ERR(-EBADR);
        if (!nlx_kcore_buffer_invariant(kb))
                return M0_ERR(-EBADR);
        cb = nlx_kcore_core_buffer_map(kb);
        if (!nlx_core_buffer_invariant(cb))
                rc = -EBADR;
        else
                rc = op(ktm, cb, kb);
        nlx_kcore_core_buffer_unmap(kb);

        return M0_RC(rc);
}

static int nlx_dev_ioctl_buf_del(const struct nlx_kcore_domain *kd,
                                 const struct m0_lnet_dev_buf_queue_params *p)
{
        struct nlx_kcore_transfer_mc *ktm = p->dbq_ktm;
        struct nlx_kcore_buffer *kb = p->dbq_kb;

        if (!virt_addr_valid(ktm) || !virt_addr_valid(kb))
                return M0_ERR(-EBADR);
        if (!nlx_kcore_tm_invariant(ktm))
                return M0_ERR(-EBADR);
        if (!nlx_kcore_buffer_invariant(kb))
                return M0_ERR(-EBADR);
        return kd->kd_drv_ops->ko_buf_del(ktm, kb);
}

static int nlx_dev_ioctl_buf_event_wait(const struct nlx_kcore_domain *kd,
                             const struct m0_lnet_dev_buf_event_wait_params *p)
{
        struct nlx_kcore_transfer_mc *ktm = p->dbw_ktm;
        struct nlx_core_transfer_mc *ctm;
        int rc;

        if (!virt_addr_valid(ktm))
                return M0_ERR(-EBADR);
        M0_ASSERT(nlx_kcore_tm_invariant(ktm));
        ctm = nlx_kcore_core_tm_map(ktm);
        if (!nlx_core_tm_invariant(ctm))
                rc = M0_ERR(-EBADR);
        else
                rc = kd->kd_drv_ops->ko_buf_event_wait(ctm, ktm,
                                                       p->dbw_timeout);
        nlx_kcore_core_tm_unmap(ktm);

        return M0_RC(rc);
}

static int nlx_dev_ioctl_nidstr_decode(struct m0_lnet_dev_nid_encdec_params *p)
{
        return nlx_kcore_nidstr_decode(p->dn_buf, &p->dn_nid);
}

static int nlx_dev_ioctl_nidstr_encode(struct m0_lnet_dev_nid_encdec_params *p)
{
        return nlx_kcore_nidstr_encode(p->dn_nid, p->dn_buf);
}

static int nlx_dev_ioctl_nidstrs_get(struct nlx_kcore_domain *kd,
                                     struct m0_lnet_dev_nidstrs_get_params *p)
{
        char **nidstrs;
        int rc = nlx_core_nidstrs_get(NULL, &nidstrs);
        char *buf;
        m0_bcount_t sz;
        int i;

        if (rc != 0)
                return M0_RC(rc);
        for (i = 0, sz = 1; nidstrs[i] != NULL; ++i)
                sz += strlen(nidstrs[i]) + 1;
        if (sz > p->dng_size) {
                nlx_core_nidstrs_put(NULL, &nidstrs);
                return M0_ERR_INFO(-EFBIG, "sz=%" PRIu64 " p->dng_size=%"PRIu64,
                                   sz, p->dng_size);
        }
        NLX_ALLOC(buf, sz);
        if (buf == NULL) {
                nlx_core_nidstrs_put(NULL, &nidstrs);
                return M0_ERR(-ENOMEM);
        }
        for (i = 0, sz = 0; nidstrs[i] != NULL; ++i) {
                strcpy(&buf[sz], nidstrs[i]);
                sz += strlen(nidstrs[i]) + 1;
        }
        nlx_core_nidstrs_put(NULL, &nidstrs);
        if (copy_to_user((void __user *) p->dng_buf, buf, sz))
                rc = -EFAULT;
        else
                rc = i;
        m0_free(buf);

        return M0_RC(rc);
}

static int nlx_dev_ioctl_tm_start(struct nlx_kcore_domain *kd,
                                  struct m0_lnet_dev_tm_start_params *p)
{
        struct page *pg;
        struct nlx_core_transfer_mc *ctm;
        struct nlx_kcore_transfer_mc *ktm;
        unsigned long utmp = (unsigned long) p->dts_ctm;
        uint32_t off = NLX_PAGE_OFFSET(utmp);
        int rc;

        if (off + sizeof *ctm > PAGE_SIZE)
                return M0_ERR(-EBADR);
        NLX_ALLOC_PTR(ktm);
        if (ktm == NULL)
                return M0_ERR(-ENOMEM);
        ktm->ktm_magic = M0_NET_LNET_KCORE_TM_MAGIC;

        down_read(&current->mm->mmap_sem);
        rc = WRITABLE_USER_PAGE_GET(utmp, pg);
        up_read(&current->mm->mmap_sem);
        if (rc < 0) {
                M0_LOG(M0_ERROR, "WRITABLE_USER_PAGE_GET() failed: rc=%d", rc);
                goto fail_page;
        }
        nlx_core_kmem_loc_set(&ktm->ktm_ctm_loc, pg, off);
        ctm = nlx_kcore_core_tm_map(ktm);
        if (ctm->ctm_magic != 0 || ctm->ctm_mb_counter != 0 ||
            ctm->ctm_kpvt != NULL) {
                rc = M0_ERR(-EBADR);
                goto fail_ctm;
        }
        rc = kd->kd_drv_ops->ko_tm_start(kd, ctm, ktm);
        if (rc != 0)
                goto fail_ctm;
        nlx_kcore_core_tm_unmap(ktm);
        m0_mutex_lock(&kd->kd_drv_mutex);
        drv_tms_tlist_add(&kd->kd_drv_tms, ktm);
        m0_mutex_unlock(&kd->kd_drv_mutex);
        return M0_RC(0);

fail_ctm:
        nlx_kcore_core_tm_unmap(ktm);
        WRITABLE_USER_PAGE_PUT(ktm->ktm_ctm_loc.kl_page);
fail_page:
        ktm->ktm_magic = 0;
        m0_free(ktm);
        M0_ASSERT(rc != 0);
        return M0_ERR(rc);
}

static int nlx_dev_tm_cleanup(struct nlx_kcore_domain *kd,
                              struct nlx_kcore_transfer_mc *ktm)
{
        struct nlx_kcore_buffer_event *kbev;
        struct nlx_core_transfer_mc *ctm;

        if (!nlx_kcore_tm_invariant(ktm))
                return M0_ERR(-EBADR);
        m0_tl_for(drv_bevs, &ktm->ktm_drv_bevs, kbev) {
                WRITABLE_USER_PAGE_PUT(kbev->kbe_bev_loc.kl_page);
                m0_mutex_lock(&kd->kd_drv_mutex);
                drv_bevs_tlist_del(kbev);
                m0_mutex_unlock(&kd->kd_drv_mutex);
                drv_bevs_tlink_fini(kbev);
                m0_free(kbev);
        } m0_tl_endfor;
        m0_mutex_lock(&kd->kd_drv_mutex);
        drv_tms_tlist_del(ktm);
        m0_mutex_unlock(&kd->kd_drv_mutex);

        ctm = nlx_kcore_core_tm_map(ktm);
        kd->kd_drv_ops->ko_tm_stop(ctm, ktm);
        nlx_kcore_core_tm_unmap(ktm);
        WRITABLE_USER_PAGE_PUT(ktm->ktm_ctm_loc.kl_page);
        m0_free(ktm);
        return 0;
}

static int nlx_dev_ioctl_tm_stop(struct nlx_kcore_domain *kd,
                                 struct m0_lnet_dev_tm_stop_params *p)
{
        struct nlx_kcore_transfer_mc *ktm = p->dts_ktm;

        /* protect against user space passing invalid ptr */
        if (!virt_addr_valid(ktm))
                return M0_ERR(-EBADR);
        return nlx_dev_tm_cleanup(kd, ktm);
}

static int nlx_dev_ioctl_bev_bless(struct nlx_kcore_domain *kd,
                                   struct m0_lnet_dev_bev_bless_params *p)
{
        struct page *pg;
        struct nlx_kcore_transfer_mc *ktm = p->dbb_ktm;
        struct nlx_kcore_buffer_event *kbe;
        struct nlx_core_buffer_event *cbe;
        uint32_t off = NLX_PAGE_OFFSET((unsigned long) p->dbb_bev);
        int rc;

        if (!virt_addr_valid(ktm))
                return M0_ERR(-EBADR);
        if (!nlx_kcore_tm_invariant(ktm))
                return M0_ERR(-EBADR);
        if (off + sizeof *cbe > PAGE_SIZE)
                return M0_ERR(-EBADR);

        NLX_ALLOC_PTR(kbe);
        if (kbe == NULL)
                return M0_ERR(-ENOMEM);
        drv_bevs_tlink_init(kbe);

        down_read(&current->mm->mmap_sem);
        rc = WRITABLE_USER_PAGE_GET(p->dbb_bev, pg);
        up_read(&current->mm->mmap_sem);
        if (rc < 0)
                goto fail_page;
        nlx_core_kmem_loc_set(&kbe->kbe_bev_loc, pg, off);
        cbe = nlx_kcore_core_bev_map(kbe);
        if (cbe->cbe_kpvt != NULL ||
            !nlx_core_kmem_loc_is_empty(&cbe->cbe_tm_link.cbl_p_self_loc)) {
                rc = -EBADR;
                goto fail_cbe;
        }
        bev_link_bless(&cbe->cbe_tm_link, pg);
        cbe->cbe_kpvt = kbe;
        nlx_kcore_core_bev_unmap(kbe);
        m0_mutex_lock(&kd->kd_drv_mutex);
        drv_bevs_tlist_add(&ktm->ktm_drv_bevs, kbe);
        m0_mutex_unlock(&kd->kd_drv_mutex);
        return 0;

fail_cbe:
        nlx_kcore_core_bev_unmap(kbe);
        WRITABLE_USER_PAGE_PUT(kbe->kbe_bev_loc.kl_page);
fail_page:
        drv_bevs_tlink_fini(kbe);
        kbe->kbe_magic = 0;
        m0_free(kbe);
        M0_ASSERT(rc != 0);
        return M0_RC(rc);
}

static long nlx_dev_ioctl(struct file *file,
                          unsigned int cmd, unsigned long arg)
{
        struct nlx_kcore_domain *kd = file->private_data;
        union {
                struct m0_lnet_dev_dom_init_params       dip;
                struct m0_lnet_dev_tm_start_params       tsp;
                struct m0_lnet_dev_tm_stop_params        tpp;
                struct m0_lnet_dev_buf_register_params   brp;
                struct m0_lnet_dev_buf_deregister_params bdp;
                struct m0_lnet_dev_buf_queue_params      bqp;
                struct m0_lnet_dev_buf_event_wait_params bep;
                struct m0_lnet_dev_nid_encdec_params     nep;
                struct m0_lnet_dev_nidstrs_get_params    ngp;
                struct m0_lnet_dev_bev_bless_params      bbp;
        } p;
        unsigned sz = _IOC_SIZE(cmd);
        int rc;

        M0_THREAD_ENTER;
        M0_PRE(nlx_kcore_domain_invariant(kd));

        if (_IOC_TYPE(cmd) != M0_LNET_IOC_MAGIC ||
            _IOC_NR(cmd) < M0_LNET_IOC_MIN_NR  ||
            _IOC_NR(cmd) > M0_LNET_IOC_MAX_NR || sz > sizeof p) {
                rc = M0_ERR(-ENOTTY);
                goto done;
        }

        if (_IOC_DIR(cmd) & _IOC_WRITE) {
                if (copy_from_user(&p, (void __user *) arg, sz)) {
                        rc = M0_ERR(-EFAULT);
                        goto done;
                }
        }

        switch (cmd) {
        case M0_LNET_DOM_INIT:
                rc = nlx_dev_ioctl_dom_init(kd, &p.dip);
                break;
        case M0_LNET_TM_START:
                rc = nlx_dev_ioctl_tm_start(kd, &p.tsp);
                break;
        case M0_LNET_TM_STOP:
                rc = nlx_dev_ioctl_tm_stop(kd, &p.tpp);
                break;
        case M0_LNET_BUF_REGISTER:
                rc = nlx_dev_ioctl_buf_register(kd, &p.brp);
                break;
        case M0_LNET_BUF_DEREGISTER:
                rc = nlx_dev_ioctl_buf_deregister(kd, &p.bdp);
                break;
        case M0_LNET_BUF_MSG_RECV:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                              kd->kd_drv_ops->ko_buf_msg_recv);
                break;
        case M0_LNET_BUF_MSG_SEND:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                              kd->kd_drv_ops->ko_buf_msg_send);
                break;
        case M0_LNET_BUF_ACTIVE_RECV:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                           kd->kd_drv_ops->ko_buf_active_recv);
                break;
        case M0_LNET_BUF_ACTIVE_SEND:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                           kd->kd_drv_ops->ko_buf_active_send);
                break;
        case M0_LNET_BUF_PASSIVE_RECV:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                          kd->kd_drv_ops->ko_buf_passive_recv);
                break;
        case M0_LNET_BUF_PASSIVE_SEND:
                rc = nlx_dev_ioctl_buf_queue_op(&p.bqp,
                                          kd->kd_drv_ops->ko_buf_passive_send);
                break;
        case M0_LNET_BUF_DEL:
                rc = nlx_dev_ioctl_buf_del(kd, &p.bqp);
                break;
        case M0_LNET_BUF_EVENT_WAIT:
                rc = nlx_dev_ioctl_buf_event_wait(kd, &p.bep);
                break;
        case M0_LNET_BEV_BLESS:
                rc = nlx_dev_ioctl_bev_bless(kd, &p.bbp);
                break;
        case M0_LNET_NIDSTR_DECODE:
                rc = nlx_dev_ioctl_nidstr_decode(&p.nep);
                break;
        case M0_LNET_NIDSTR_ENCODE:
                rc = nlx_dev_ioctl_nidstr_encode(&p.nep);
                break;
        case M0_LNET_NIDSTRS_GET:
                rc = nlx_dev_ioctl_nidstrs_get(kd, &p.ngp);
                break;
        default:
                rc = M0_ERR(-ENOTTY);
                break;
        }

        if (rc >= 0 && (_IOC_DIR(cmd) & _IOC_READ)) {
                if (copy_to_user((void __user *) arg, &p, sz))
                        rc = M0_ERR(-EFAULT);
        }

done:
        if (rc < 0 && ergo(cmd == M0_LNET_BUF_EVENT_WAIT,
                           !M0_IN(rc, (-ETIMEDOUT, -ERESTARTSYS))))
                M0_LOG(M0_ERROR, "cmd=0x%x rc=%d", cmd, rc);
        if (rc == -ERESTARTSYS)
                rc = -EINTR;
        return M0_RC(rc);
}

static int nlx_dev_open(struct inode *inode, struct file *file)
{
        struct nlx_kcore_domain *kd;
        int rc;
        M0_THREAD_ENTER;

        if (!(file->f_flags & O_RDWR))
                return M0_ERR(-EACCES);

        NLX_ALLOC_PTR(kd);
        if (kd == NULL)
                return M0_ERR(-ENOMEM);
        rc = nlx_kcore_kcore_dom_init(kd);
        if (rc != 0) {
                m0_free(kd);
        } else {
                file->private_data = kd;
        }
        return M0_RC(rc);
}

M0_INTERNAL int nlx_dev_close(struct inode *inode, struct file *file)
{
        struct nlx_kcore_domain *kd =
            (struct nlx_kcore_domain *) file->private_data;
        struct nlx_core_domain *cd;
        struct nlx_kcore_transfer_mc *ktm;
        struct nlx_kcore_buffer *kb;
        bool cleanup = false;
        int rc;
        M0_THREAD_ENTER;

        M0_PRE(nlx_kcore_domain_invariant(kd));
        file->private_data = NULL;

        /*
         * user program may not unmap all areas, eg if it was killed.
         * 1. Cancel all outstanding buffer operations.
         * 2. Clean up (stop, et al) all running TMs, this can take a while.
         * 3. De-register all buffers.
         */
        m0_tl_for(drv_bufs, &kd->kd_drv_bufs, kb) {
                ktm = kb->kb_ktm;
                if (ktm != NULL) {
                        /*
                         * Only LNetMDUnlink() causes nlx_kcore_LNetMDUnlink()
                         * failure.  That can happen if the operation completes
                         * concurrently with the execution of this loop.
                         * Such failures are OK in this context.
                         */
                        nlx_kcore_LNetMDUnlink(ktm, kb);
                }
        } m0_tl_endfor;
        m0_tl_for(drv_tms, &kd->kd_drv_tms, ktm) {
                /*
                 * Wait until no more buffers are associated with this TM and
                 * the event callback is no longer using the ktm.  Must be in
                 * the ktm-based loop because it needs to synchronize use of the
                 * kb_ktm with the LNet event callback using the spinlock. Only
                 * holds spinlock for extremely short periods to avoid seriously
                 * blocking event callback.  Depends on:
                 * 1. There can be no other threads down-ing ktm_sem or adding
                 *    new buffers to queues while in nlx_dev_close().
                 * 2. kb_ktm is set to NULL in the spinlock and before ktm_sem
                 *    is up'd.
                 * 3. The final reference to the ktm in nlx_kcore_eq_cb() is to
                 *    unlock the spinlock, after up-ing ktm_sem.
                 * 4. LNet events involving unlink are not dropped.
                 */
                m0_tlist_for(&drv_bufs_tl, &kd->kd_drv_bufs, kb) {
                        spin_lock(&ktm->ktm_bevq_lock);
                        while (kb->kb_ktm == ktm) {
                                spin_unlock(&ktm->ktm_bevq_lock);
                                wait_event(ktm->ktm_wq, kb->kb_ktm == NULL);
                                spin_lock(&ktm->ktm_bevq_lock);
                        }
                        spin_unlock(&ktm->ktm_bevq_lock);
                } m0_tlist_endfor;

                rc = nlx_dev_tm_cleanup(kd, ktm);
                M0_ASSERT(rc == 0);
                cleanup = true;
        } m0_tl_endfor;
        m0_tl_for(drv_bufs, &kd->kd_drv_bufs, kb) {
                rc = nlx_dev_buf_deregister(kd, kb);
                M0_ASSERT(rc == 0);
                cleanup = true;
        } m0_tl_endfor;

        if (cleanup)
                M0_LOG(M0_NOTICE, "close cleanup");

        /* user program may not successfully perform M0_NET_DOM_INIT ioctl */
        if (!nlx_core_kmem_loc_is_empty(&kd->kd_cd_loc)) {
                cd = nlx_kcore_core_domain_map(kd);
                kd->kd_drv_ops->ko_dom_fini(kd, cd);
                nlx_kcore_core_domain_unmap(kd);
                WRITABLE_USER_PAGE_PUT(kd->kd_cd_loc.kl_page);
                nlx_core_kmem_loc_set(&kd->kd_cd_loc, NULL, 0);
        }

        nlx_kcore_kcore_dom_fini(kd);
        m0_free(kd);
        return M0_RC(0);
}

static struct file_operations nlx_dev_file_ops = {
        .owner          = THIS_MODULE,
        .unlocked_ioctl = nlx_dev_ioctl,
        .open           = nlx_dev_open,
        .release        = nlx_dev_close
};

static struct miscdevice nlx_dev = {
        .minor   = MISC_DYNAMIC_MINOR,
        .name    = M0_LNET_DEV,
        .fops    = &nlx_dev_file_ops
};
static bool nlx_dev_registered = false;
 /* LNetDevInternal */

M0_INTERNAL int nlx_dev_init(void)
{
        int rc;

        rc = misc_register(&nlx_dev);
        if (rc != 0)
                return M0_RC(rc);
        nlx_dev_registered = true;
        printk("Motr %s registered with minor %d\n",
               nlx_dev.name, nlx_dev.minor);
        return M0_RC(rc);
}

M0_INTERNAL void nlx_dev_fini(void)
{
        if (nlx_dev_registered) {
                misc_deregister(&nlx_dev);
                nlx_dev_registered = false;
                printk("Motr %s deregistered\n", nlx_dev.name);
        }
}
 /* LNetDev */

/*
 *  Local variables:
 *  c-indentation-style: "K&R"
 *  c-basic-offset: 8
 *  tab-width: 8
 *  fill-column: 80
 *  scroll-step: 1
 *  End:
 */