processor.c

Go to the documentation of this file.

/* -*- C -*- */
/*
 * Copyright (c) 2012-2020 Seagate Technology LLC and/or its Affiliates
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For any questions about this software or licensing,
 * please email opensource@seagate.com or cortx-questions@seagate.com.
 *
 */


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/syscall.h>  /* syscall, SYS_getcpu */
#include <dirent.h>
#include <unistd.h>
#include <linux/limits.h>
#include <sched.h>        /* sched_getcpu() */

#define M0_TRACE_SUBSYSTEM M0_TRACE_SUBSYS_OTHER
#include "lib/trace.h"
#include "lib/errno.h"
#include "lib/memory.h"
#include "lib/processor.h"
#include "lib/list.h"
#include "lib/thread.h"
#include "module/instance.h"

#define PROCESSORS_INFO_ENV             "M0_PROCESSORS_INFO_DIR"

#define PROCESSORS_SYSFS_DIR            "/sys/devices/system"
#define PROCESSORS_CPU_DIR              "cpu/"
#define PROCESSORS_NODE_DIR             "node/"

#define PROCESSORS_MAX_FILE             "cpu/kernel_max"
#define PROCESSORS_POSS_FILE            "cpu/possible"
#define PROCESSORS_PRESENT_FILE         "cpu/present"
#define PROCESSORS_ONLINE_FILE          "cpu/online"

#define PROCESSORS_CACHE1_LEVEL_FILE    "cache/index0/level"
#define PROCESSORS_CACHE2_LEVEL_FILE    "cache/index1/level"
#define PROCESSORS_CACHE3_LEVEL_FILE    "cache/index2/level"

#define PROCESSORS_CACHE1_SHCPUMAP_FILE "cache/index0/shared_cpu_map"
#define PROCESSORS_CACHE2_SHCPUMAP_FILE "cache/index1/shared_cpu_map"
#define PROCESSORS_CACHE3_SHCPUMAP_FILE "cache/index2/shared_cpu_map"

#define PROCESSORS_CACHE1_SIZE_FILE     "cache/index0/size"
#define PROCESSORS_CACHE2_SIZE_FILE     "cache/index1/size"
#define PROCESSORS_CACHE3_SIZE_FILE     "cache/index2/size"

#define PROCESSORS_COREID_FILE          "topology/core_id"
#define PROCESSORS_PHYSID_FILE          "topology/physical_package_id"

#define PROCESSORS_CPU_DIR_PREFIX       "cpu/cpu"

#define PROCESSORS_NODE_STR             "node"
#define PROCESSORS_CPU_STR              "cpu"

enum {
        PROCESSORS_RANGE_SET_SEPARATOR = ',',
        PROCESSORS_RANGE_SEPARATOR     = '-',

        PROCESSORS_L1                  = 1,
        PROCESSORS_L2                  = 2,

        MAX_LINE_LEN                   = 256,
};

enum map {
        PROCESSORS_POSS_MAP  = 0,
        PROCESSORS_AVAIL_MAP = 1,
        PROCESSORS_ONLN_MAP  = 2,
};

struct processor_sys_summary {
        struct m0_list    pss_head;

        m0_processor_nr_t pss_max;

        struct m0_bitmap  pss_poss_map;

        struct m0_bitmap  pss_avail_map;

        struct m0_bitmap  pss_onln_map;
};

struct processor_node {
        struct m0_list_link       pn_link;

        struct m0_processor_descr pn_info;
};

struct processor_global {
        bool pg_getcpu_inited;
        bool pg_getcpu_workaround;
};

struct getcpu_thr_data {
        m0_processor_nr_t  ptd_idx;
        struct m0_thread   ptd_thread;
        struct m0_bitmap   ptd_cpu_map;
        bool               ptd_success;
};

/* Global Variables. */
static struct processor_sys_summary sys_cpus;
static bool processor_init = false;

static void processor_map_set(struct m0_bitmap *map, const char *mapstr)
{
        uint32_t id;
        uint32_t to_id;
        char    *ptr;

        M0_PRE(map != NULL);
        M0_PRE(mapstr != NULL);

        /*
         * Walk the string, parsing separate cpu ranges.
         * For example, the string looks like 0-3,7,8-12
         */
        for (; mapstr != NULL && *mapstr != 0; mapstr = ptr) {
                /*
                 * Parse from and to indices within the range.
                 */
                id = strtoul(mapstr, &ptr, 0);
                if (*ptr == PROCESSORS_RANGE_SEPARATOR)
                        to_id = strtoul(ptr + 1, &ptr, 0);
                else
                        to_id = id;
                /*
                 * Set the bitmap for the given range.
                 */
                M0_ASSERT(to_id < map->b_nr);
                while (id <= to_id)
                        m0_bitmap_set(map, id++, true);
                if (*ptr == PROCESSORS_RANGE_SET_SEPARATOR)
                        ++ptr;
                else
                        ptr = NULL;
        } /* for - string is parsed */
}

static int processor_map_type_set(enum map map_type)
{
        int                rc;
        char               buf[MAX_LINE_LEN + 1];
        char              *str;
        FILE              *fp;
        struct m0_bitmap  *pbitmap;
        static const char *fname[] = {
                [PROCESSORS_POSS_MAP]  = PROCESSORS_POSS_FILE,
                [PROCESSORS_AVAIL_MAP] = PROCESSORS_PRESENT_FILE,
                [PROCESSORS_ONLN_MAP]  = PROCESSORS_ONLINE_FILE,
        };

        M0_PRE((unsigned) map_type <= PROCESSORS_ONLN_MAP);
        fp = fopen(fname[map_type], "r");
        if (fp == NULL)
                return -errno;

        str = fgets(buf, MAX_LINE_LEN, fp);
        fclose(fp);
        if (str == NULL)
                return M0_ERR(-ENODATA);

        switch (map_type) {
        case PROCESSORS_POSS_MAP :
                pbitmap = &sys_cpus.pss_poss_map;
                break;

        case PROCESSORS_AVAIL_MAP :
                pbitmap = &sys_cpus.pss_avail_map;
                break;

        case PROCESSORS_ONLN_MAP :
                pbitmap = &sys_cpus.pss_onln_map;
                break;
        default:
                M0_IMPOSSIBLE("Incorrect map_type value");
                pbitmap = NULL;
                break;
        }

        rc = m0_bitmap_init(pbitmap, sys_cpus.pss_max);
        if (rc == 0)
                processor_map_set(pbitmap, buf);
        return M0_RC(rc);
}

static uint32_t number_read(const char *filename)
{
        uint32_t val = M0_PROCESSORS_INVALID_ID;
        int      rc;
        FILE    *fp;

        fp = fopen(filename, "r");
        if (fp == NULL)
                return val;

        rc = fscanf(fp, "%u", &val);
        if (rc <= 0)
                val = M0_PROCESSORS_INVALID_ID;

        fclose(fp);
        return val;
}

static void processor_maxsz_get()
{
        /*
         * cpu/kernel_max contains maximum index which starts from 0. Therefore,
         * we need to convert it to maximum number of CPUs.
         */
        sys_cpus.pss_max = number_read(PROCESSORS_MAX_FILE) + 1;
}

static uint32_t processor_numanodeid_get(m0_processor_nr_t id)
{
        uint32_t       numa_node_id = 0;
        bool           gotid = false;
        int            rc;
        char           dirname[PATH_MAX];
        DIR           *dirp;
        struct stat    statbuf;
        struct dirent *fname;

        sprintf(dirname, PROCESSORS_CPU_DIR_PREFIX"%u", id);
        dirp = opendir(dirname);
        if (dirp == NULL)
                return numa_node_id;

        /*
         * Find node id under .../cpu/cpu<id>/node<id>
         */
        while ((fname = readdir(dirp)) != NULL) {
                if (strncmp(fname->d_name, PROCESSORS_NODE_STR,
                            sizeof(PROCESSORS_NODE_STR) - 1) == 0) {
                        sscanf(fname->d_name,
                               PROCESSORS_NODE_STR"%u", &numa_node_id);
                        gotid = true;
                        break;
                }
        }/* while - entire cpuX dir is scanned */

        closedir(dirp);
        if (gotid)
                return numa_node_id;

        /*
         * If nodeid file is not found in previous search, look for cpuX file
         * under node/nod<id>/cpu<id>
         */
        dirp = opendir(PROCESSORS_NODE_DIR);
        if (dirp == NULL)
                return numa_node_id;

        while ((fname = readdir(dirp)) != NULL) {
                if (strncmp(fname->d_name, PROCESSORS_NODE_STR,
                            sizeof(PROCESSORS_NODE_STR) - 1) == 0) {
                        sprintf(dirname, "%s%s/%s%u",
                                PROCESSORS_NODE_DIR, fname->d_name,
                                PROCESSORS_CPU_STR, id);
                        rc = stat(dirname, &statbuf);
                        if (rc == 0) {
                                sscanf(fname->d_name,
                                       PROCESSORS_NODE_STR"%u", &numa_node_id);
                                break;
                        }
                }
        }/* while - entire node dir is scanned */
        closedir(dirp);

        /*
         * Note : If no numa node id is found, it's assumed to be 0.
         */
        return numa_node_id;
}

static uint32_t processor_coreid_get(m0_processor_nr_t id)
{
        char filebuf[PATH_MAX];

        sprintf(filebuf, PROCESSORS_CPU_DIR_PREFIX"%u/"
                         PROCESSORS_COREID_FILE, id);
        return number_read(filebuf);
}

static uint32_t processor_physid_get(m0_processor_nr_t id)
{
        char filebuf[PATH_MAX];

        sprintf(filebuf, PROCESSORS_CPU_DIR_PREFIX"%u/"
                         PROCESSORS_PHYSID_FILE, id);
        return number_read(filebuf);
}

static bool processor_is_cache_shared(const char *mapstr)
{
        char    *ptr;
        uint32_t num;
        int      shared_cpus = 0;

        /*
         * The string is a bitmap. Each byte string is separated by ','.
         */
        for (; mapstr != NULL && *mapstr != 0; mapstr = ptr) {
                /*
                 * Convert the "bytes" hexstring into a number, add to bits set.
                 */
                for (num = strtoul(mapstr, &ptr, 16); num != 0; num >>= 1)
                        shared_cpus += num & 1;
                /*
                 * If we have already found more than one cpu, don't
                 * scan further.
                 */
                if (shared_cpus > 1) /* Is cache shared? */
                        break;
                if (*ptr == PROCESSORS_RANGE_SET_SEPARATOR)
                        ++ptr;
                else
                        ptr = NULL;
        } /* for - entire string is parsed */

        M0_POST(shared_cpus >= 1);
        return shared_cpus > 1;
}

static size_t processor_l1_size_get(m0_processor_nr_t id)
{
        uint32_t level;
        uint32_t sz;
        int      rc;
        size_t   size = 0;
        char     filename[PATH_MAX];
        FILE    *fp;

        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                   PROCESSORS_CACHE1_LEVEL_FILE);
        level = number_read(filename);
        if (level == M0_PROCESSORS_INVALID_ID)
                return size;

        M0_ASSERT(level == PROCESSORS_L1);

        /*
         * Set path to appropriate cache size file
         */
        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                   PROCESSORS_CACHE1_SIZE_FILE);

        /*
         * Get the size string. It's in format 32K, 6144K etc.
         */
        fp = fopen(filename, "r");
        if (fp == NULL)
                return size;

        rc = fscanf(fp, "%uK", &sz);
        fclose(fp);
        if (rc <= 0)
                return size;

        size = (size_t) sz * 1024;
        return size;
}

static size_t processor_l2_size_get(m0_processor_nr_t id)
{
        uint32_t level;
        uint32_t sz;
        int      rc;
        size_t   size = 0;
        char     filename[PATH_MAX];
        FILE    *fp;

        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                   PROCESSORS_CACHE3_LEVEL_FILE);
        level = number_read(filename);
        if (level == M0_PROCESSORS_INVALID_ID)
                return size;

        if (level != PROCESSORS_L2) { /* If L2 level is not found */
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE2_LEVEL_FILE);
                level = number_read(filename);
                if (level == M0_PROCESSORS_INVALID_ID)
                        return size;

                M0_ASSERT(level == PROCESSORS_L2);
                /*
                 * Set path to appropriate cache size file
                 */
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE2_SIZE_FILE);
        } else {
                /*
                 * Set path to appropriate cache size file
                 */
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE3_SIZE_FILE);
        }

        /*
         * Get the size string. It's in format 32K, 6144K etc.
         */
        fp = fopen(filename, "r");
        if (fp == NULL)
                return size;

        rc = fscanf(fp, "%uK", &sz);
        fclose(fp);
        if (rc <= 0)
                return size;

        size = (size_t)(sz * 1024);
        return size;
}

static uint32_t processor_l1_cacheid_get(m0_processor_nr_t id)
{
        uint32_t level;
        uint32_t coreid;
        uint32_t physid;
        uint32_t l1_id = id;

        bool     is_shared;

        char     filename[PATH_MAX];
        char     buf[MAX_LINE_LEN + 1];
        char    *str;

        FILE    *fp;

        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                   PROCESSORS_CACHE1_LEVEL_FILE);
        level = number_read(filename);
        if (level == M0_PROCESSORS_INVALID_ID)
                return l1_id;

        M0_ASSERT(level == PROCESSORS_L1);
        /*
         * Set path to appropriate shared cpu file name
         */
        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                          PROCESSORS_CACHE1_SHCPUMAP_FILE);

        /*
         * Get the shared cpu map string.
         */
        fp = fopen(filename, "r");
        if (fp == NULL)
                return l1_id;

        str = fgets(buf, MAX_LINE_LEN, fp);
        fclose(fp);
        if (str == NULL)
                return l1_id;

        /*
         * Scan the map string to find how many bits are set in the string
         * If more than one bit is set, then cache is shared.
         */
        is_shared = processor_is_cache_shared(buf);
        if (is_shared) { /* L1 cache is shared */
                physid = processor_physid_get(id);
                coreid = processor_coreid_get(id);
                l1_id = physid << 16 | coreid;
        }

        return l1_id;
}

static uint32_t processor_l2_cacheid_get(m0_processor_nr_t id)
{
        uint32_t level;
        uint32_t l2_id = id;
        uint32_t coreid;
        uint32_t physid;

        bool     is_shared;
        bool     l3_cache_present = false;

        char     filename[PATH_MAX];
        char     buf[MAX_LINE_LEN + 1];
        char    *str;

        FILE    *fp;

        sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                   PROCESSORS_CACHE3_LEVEL_FILE);
        level = number_read(filename);
        if (level == M0_PROCESSORS_INVALID_ID)
                return l2_id;

        if (level != PROCESSORS_L2) {
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE2_LEVEL_FILE);
                level = number_read(filename);
                if (level == M0_PROCESSORS_INVALID_ID)
                        return l2_id;

                M0_ASSERT(level == PROCESSORS_L2);
                l3_cache_present = true;
                /*
                 * Set path to appropriate shared cpu file name
                 */
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE2_SHCPUMAP_FILE);
        } else {
                /*
                 * Set path to appropriate shared cpu file name
                 */
                sprintf(filename, "%s%u/%s", PROCESSORS_CPU_DIR_PREFIX, id,
                                           PROCESSORS_CACHE3_SHCPUMAP_FILE);
        }

        /*
         * Get the shared cpu map string.
         */
        fp = fopen(filename, "r");
        if (fp == NULL)
                return l2_id;

        str = fgets(buf, MAX_LINE_LEN, fp);
        fclose(fp);
        if (str == NULL)
                return l2_id;

        /*
         * Scan the map string to find how many bits are set in the string
         */
        is_shared = processor_is_cache_shared(buf);
        if (is_shared) { /* L2 cache is shared */
                physid = processor_physid_get(id);

                if (l3_cache_present) { /* L3 cache is present */
                        coreid = processor_coreid_get(id);
                        l2_id = physid << 16 | coreid;
                } else
                        l2_id = physid;
        }

        return l2_id;
}

static inline uint32_t processor_pipelineid_get(m0_processor_nr_t id)
{
        return id;
}

static int processor_info_get(m0_processor_nr_t id, struct processor_node *pn)
{
        M0_PRE(pn != NULL);

        pn->pn_info.pd_numa_node = processor_numanodeid_get(id);
        if (pn->pn_info.pd_numa_node == M0_PROCESSORS_INVALID_ID)
                return M0_ERR(-EINVAL);

        pn->pn_info.pd_l1 = processor_l1_cacheid_get(id);
        if (pn->pn_info.pd_l1 == M0_PROCESSORS_INVALID_ID)
                return M0_ERR(-EINVAL);

        pn->pn_info.pd_l1_sz = processor_l1_size_get(id);
        if (pn->pn_info.pd_l1_sz == M0_PROCESSORS_INVALID_ID)
                return M0_ERR(-EINVAL);

        pn->pn_info.pd_l2 = processor_l2_cacheid_get(id);
        if (pn->pn_info.pd_l2 == M0_PROCESSORS_INVALID_ID)
                return M0_ERR(-EINVAL);

        pn->pn_info.pd_l2_sz = processor_l2_size_get(id);
        if (pn->pn_info.pd_l2_sz == M0_PROCESSORS_INVALID_ID)
                return M0_ERR(-EINVAL);

        pn->pn_info.pd_id = id;
        pn->pn_info.pd_pipeline = processor_pipelineid_get(id);

        return 0;
}

static void processor_cache_destroy(void)
{
        struct m0_list_link   *node;
        struct processor_node *pinfo;

        m0_bitmap_fini(&sys_cpus.pss_poss_map);
        m0_bitmap_fini(&sys_cpus.pss_avail_map);
        m0_bitmap_fini(&sys_cpus.pss_onln_map);
        sys_cpus.pss_max = 0;

        /* Remove all the processor nodes. */
        node = sys_cpus.pss_head.l_head;
        while((struct m0_list *)node != &sys_cpus.pss_head) {
                pinfo = m0_list_entry(node, struct processor_node, pn_link);
                m0_list_del(&pinfo->pn_link);
                m0_free(pinfo);
                node = sys_cpus.pss_head.l_head;
        }
        m0_list_fini(&sys_cpus.pss_head);
}

static int processors_summary_get()
{
        int                    rc;
        uint32_t               cpuid;
        char                  *dirp = getenv(PROCESSORS_INFO_ENV);
        char                  *str;
        char                   cwd[PATH_MAX];
        struct processor_node *pn;

        if (dirp == NULL)
                dirp = PROCESSORS_SYSFS_DIR;

        /*
         * Obtain current working directory.
         */
        str = getcwd(cwd, sizeof(cwd) - 1);
        if (str == NULL)
                return -errno;

        /*
         * Change directory to desired "sysfs" directory.
         * Subsequent functions will use file names "relative" to syfs dir.
         * Subsequent function will work in the context of "sysfs" directory.
         */
        rc = chdir(dirp);
        if (rc != 0)
                return -errno;

        /*
         * Now get the summary of max/possible/avail/online CPUs
         * All these functions will collect summary in 'sys_cpus'.
         */

        processor_maxsz_get();
        if (sys_cpus.pss_max == M0_PROCESSORS_INVALID_ID) {
                rc = chdir(cwd);
                if (rc != 0)
                        M0_LOG(M0_ERROR, "failed to chdir to '%s'", (char*)cwd);
                return M0_ERR(-EINVAL);
        }

        rc = processor_map_type_set(PROCESSORS_POSS_MAP);
        if (rc != 0) {
                int rc2 = chdir(cwd);
                if (rc2 != 0)
                        M0_LOG(M0_ERROR, "failed to chdir to '%s'", (char*)cwd);
                return M0_RC(rc);
        }

        rc = processor_map_type_set(PROCESSORS_AVAIL_MAP);
        if (rc != 0) {
                int rc2;
                m0_bitmap_fini(&sys_cpus.pss_poss_map);
                rc2 = chdir(cwd);
                if (rc2 != 0)
                        M0_LOG(M0_ERROR, "failed to chdir to '%s'", (char*)cwd);
                return M0_RC(rc);
        }

        rc = processor_map_type_set(PROCESSORS_ONLN_MAP);
        if (rc != 0) {
                int rc2;
                m0_bitmap_fini(&sys_cpus.pss_poss_map);
                m0_bitmap_fini(&sys_cpus.pss_avail_map);
                rc2 = chdir(cwd);
                if (rc2 != 0)
                        M0_LOG(M0_ERROR, "failed to chdir to '%s'", (char*)cwd);
                return M0_RC(rc);
        }

        m0_list_init(&sys_cpus.pss_head);

        /*
         * Using present/available CPU mask get details of each processor.
         */
        for (cpuid = 0; cpuid < sys_cpus.pss_avail_map.b_nr; ++cpuid) {
                if (m0_bitmap_get(&sys_cpus.pss_avail_map, cpuid)) {
                        M0_ALLOC_PTR(pn);
                        if (pn == NULL) {
                                processor_cache_destroy();
                                return M0_ERR(-ENOMEM);
                        }
                        rc = processor_info_get(cpuid, pn);
                        if (rc != 0)
                                m0_free(pn);
                        else
                                m0_list_add(&sys_cpus.pss_head, &pn->pn_link);

                } /* if - processor is present on the system */

        }/* for - scan all the available processors */

        if (m0_list_is_empty(&sys_cpus.pss_head)) {
                processor_cache_destroy();
                rc = chdir(cwd);
                if (rc != 0)
                        M0_LOG(M0_ERROR, "failed to chdir to '%s'", (char*)cwd);
                return M0_ERR(-ENODATA);
        }

        /*
         * Change back to previous working dir
         */
        rc = chdir(cwd);
        if (rc != 0)
                return -errno;
        return 0;
}

static void processors_m0bitmap_copy(struct m0_bitmap *dst,
                                     const struct m0_bitmap *src)
{
        M0_PRE(dst->b_nr >= sys_cpus.pss_max);
        m0_bitmap_copy(dst, src);
}

static int getcpu_thr_init(struct getcpu_thr_data *data)
{
        return m0_thread_confine(&data->ptd_thread, &data->ptd_cpu_map);
}

static void getcpu_thr_func(struct getcpu_thr_data *data)
{
        m0_processor_nr_t idx = m0_processor_id_get();

        data->ptd_success = idx == data->ptd_idx;
}

static int processor_getcpu_check(bool *success)
{
        struct getcpu_thr_data  *tdata;
        struct m0_bitmap         map_online = {};
        size_t                   cpu_max;
        size_t                   cpu_nr;
        size_t                   i;
        size_t                   j;
        bool                     result = true;
        int                      rc;
        int                      rc2;

        cpu_max = m0_processor_nr_max();
        rc = m0_bitmap_init(&map_online, cpu_max);
        if (rc != 0)
                return M0_RC(rc);
        m0_processors_online(&map_online);
        cpu_nr = m0_bitmap_set_nr(&map_online);

        M0_ALLOC_ARR(tdata, cpu_nr);
        if (tdata == NULL) {
                m0_bitmap_fini(&map_online);
                return M0_ERR(-ENOMEM);
        }

        for (i = 0, j = 0; i < map_online.b_nr; ++i) {
                if (!m0_bitmap_get(&map_online, i))
                        continue;

                tdata[j].ptd_idx = (m0_processor_nr_t)i;
                rc = m0_bitmap_init(&tdata[j].ptd_cpu_map, cpu_max);
                if (rc != 0)
                        break;
                m0_bitmap_set(&tdata[j].ptd_cpu_map, i, true);

                rc = M0_THREAD_INIT(&tdata[j].ptd_thread,
                                    struct getcpu_thr_data *,
                                    getcpu_thr_init, &getcpu_thr_func,
                                    &tdata[j], "m0_getcpu_check");
                if (rc != 0) {
                        m0_bitmap_fini(&tdata[i].ptd_cpu_map);
                        break;
                }
                ++j;
        }
        M0_ASSERT(ergo(rc == 0, j == cpu_nr));

        for (i = 0; i < j; ++i) {
                rc2 = m0_thread_join(&tdata[i].ptd_thread);
                rc  = rc ?: rc2;
                result = rc2 == 0 && result && tdata[i].ptd_success;
                m0_thread_fini(&tdata[i].ptd_thread);
                m0_bitmap_fini(&tdata[i].ptd_cpu_map);
        }

        m0_free(tdata);
        m0_bitmap_fini(&map_online);

        if (rc == 0)
                *success = result;
        return M0_RC(rc);
}

static int processor_getcpu_init(void)
{
        struct processor_global *pg = m0_get()->i_moddata[M0_MODULE_PROCESSOR];
        bool                     success;
        int                      rc;

        pg->pg_getcpu_inited = true;
        pg->pg_getcpu_workaround = false;
        rc = processor_getcpu_check(&success);
        if (rc == 0 && !success) {
                M0_LOG(M0_WARN, "sched_getcpu(3) doesn't return expected"
                                "values, fall back to syscall getcpu(2).");
                pg->pg_getcpu_workaround = true;
                rc = processor_getcpu_check(&success);
                if (rc == 0 && !success)
                        rc = M0_ERR(-ENODEV);
        }
        if (rc != 0)
                pg->pg_getcpu_inited = false;

        return M0_RC(rc);
}

static void processor_getcpu_fini(void)
{
}

#ifdef CONFIG_X86_64
static void processor_cpuid_reg_get(unsigned opc, unsigned *r)
{
        __asm__ __volatile__ ("cpuid\n\t"
                              : "=a" (r[0]), "=b" (r[1]),
                                "=c" (r[2]), "=d" (r[3])
                              : "0" (opc), "1" (0), "2" (0));
}
#endif

/* ---- Processor Interface Implementation ---- */

M0_INTERNAL int m0_processors_init()
{
        struct processor_global *pg;
        int                      rc;

        M0_PRE(!processor_init);
        M0_ALLOC_PTR(pg);
        m0_get()->i_moddata[M0_MODULE_PROCESSOR] = pg;
        rc = pg == NULL ? M0_ERR(-ENOMEM) : 0;
        rc = rc ?: processors_summary_get();
        /* Required by processor_getcpu_init(). */
        processor_init = (rc == 0);
        rc = rc ?: processor_getcpu_init();

        if (rc != 0) {
                m0_free(pg);
                m0_get()->i_moddata[M0_MODULE_PROCESSOR] = NULL;
                processor_init = false;
        }
        return M0_RC(rc);
}

M0_INTERNAL void m0_processors_fini()
{
        M0_PRE(processor_init);
        processor_getcpu_fini();
        processor_cache_destroy();
        m0_free0(&m0_get()->i_moddata[M0_MODULE_PROCESSOR]);
        processor_init = false;
}

M0_INTERNAL m0_processor_nr_t m0_processor_nr_max(void)
{
        M0_PRE(processor_init);
        return sys_cpus.pss_max;
}

M0_INTERNAL void m0_processors_possible(struct m0_bitmap *map)
{
        M0_PRE(processor_init);
        M0_PRE(map != NULL);
        processors_m0bitmap_copy(map, &sys_cpus.pss_poss_map);
}

M0_INTERNAL void m0_processors_available(struct m0_bitmap *map)
{
        M0_PRE(processor_init);
        M0_PRE(map != NULL);
        processors_m0bitmap_copy(map, &sys_cpus.pss_avail_map);
}

M0_INTERNAL void m0_processors_online(struct m0_bitmap *map)
{
        M0_PRE(processor_init);
        M0_PRE(map != NULL);
        processors_m0bitmap_copy(map, &sys_cpus.pss_onln_map);
}

M0_INTERNAL int m0_processor_describe(m0_processor_nr_t id,
                                      struct m0_processor_descr *pd)
{
        struct processor_node *pinfo;

        M0_PRE(processor_init);
        M0_PRE(pd != NULL);

        m0_list_for_each_entry(&sys_cpus.pss_head, pinfo,
                               struct processor_node, pn_link) {
                if (pinfo->pn_info.pd_id == id) {
                        *pd = pinfo->pn_info;
                        return 0;
                }/* if - matching CPU id found */
        }/* for - iterate over all the processor nodes */

        return M0_ERR(-EINVAL);
}

M0_INTERNAL m0_processor_nr_t m0_processor_id_get(void)
{
        struct processor_global *pg = m0_get()->i_moddata[M0_MODULE_PROCESSOR];
        unsigned                 cpu;
        int                      rc;

        M0_PRE(pg->pg_getcpu_inited);

        if (pg->pg_getcpu_workaround) {
                rc = syscall(SYS_getcpu, &cpu, NULL, NULL);
                if (rc < 0)
                        cpu = M0_PROCESSORS_INVALID_ID;
        } else {
                rc = sched_getcpu();
                cpu = rc < 0 ? M0_PROCESSORS_INVALID_ID : (unsigned)rc;
        }
        if (rc < 0)
                M0_LOG(M0_ERROR, "%s(): rc=%d errno=%d",
                       pg->pg_getcpu_workaround ? "syscall" : "sched_getcpu",
                       rc, errno);
        return cpu;
}

#ifdef CONFIG_X86_64
M0_INTERNAL bool m0_processor_is_vm(void)
{
        unsigned reg[4] = {};

        processor_cpuid_reg_get(0x1, reg);
        return (reg[2] & (1U << 31)) != 0;
}
#else
M0_INTERNAL bool m0_processor_is_vm(void)
{
        return false;
}
#endif


#undef M0_TRACE_SUBSYSTEM

/*
 *  Local variables:
 *  c-indentation-style: "K&R"
 *  c-basic-offset: 8
 *  tab-width: 8
 *  fill-column: 80
 *  scroll-step: 1
 *  End:
 */