odp_timer_perf.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2019-2023 Nokia
 */
 
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <inttypes.h>
#include <signal.h>
#include <stdlib.h>
#include <getopt.h>
 
#include <odp_api.h>
#include <odp/helper/odph_api.h>
 
#define MODE_SCHED_OVERH  0
#define MODE_START_CANCEL 1
#define MODE_START_EXPIRE 2
#define MAX_TIMER_POOLS   32
#define MAX_TIMERS        10000
#define START_NS          (100 * ODP_TIME_MSEC_IN_NS)
 
typedef struct test_options_t {
    uint32_t num_cpu;
    uint32_t num_tp;
    uint32_t num_timer;
    uint64_t res_ns;
    uint64_t period_ns;
    int      shared;
    int      mode;
    uint64_t test_rounds;
 
} test_options_t;
 
typedef struct time_stat_t {
    uint64_t num;
    uint64_t sum_ns;
    uint64_t max_ns;
 
} time_stat_t;
 
typedef struct test_stat_t {
    uint64_t rounds;
    uint64_t events;
    uint64_t nsec;
    uint64_t cycles_0;
    uint64_t cycles_1;
 
    uint64_t cancels;
    uint64_t sets;
 
    time_stat_t before;
    time_stat_t after;
 
} test_stat_t;
 
typedef struct test_stat_sum_t {
    uint64_t rounds;
    uint64_t events;
    uint64_t nsec;
    uint64_t cycles_0;
    uint64_t cycles_1;
 
    uint64_t cancels;
    uint64_t sets;
 
    time_stat_t before;
    time_stat_t after;
 
    double   time_ave;
    uint32_t num;
 
} test_stat_sum_t;
 
typedef struct thread_arg_t {
    void *global;
    int   worker_idx;
 
} thread_arg_t;
 
typedef struct timer_ctx_t {
    uint64_t target_ns;
    uint64_t target_tick;
    uint32_t tp_idx;
    uint32_t timer_idx;
    int last;
 
} timer_ctx_t;
 
typedef struct timer_pool_t {
    odp_timer_pool_t tp;
    uint64_t start_tick;
    uint64_t period_tick;
 
} timer_pool_t;
 
typedef struct test_global_t {
    test_options_t test_options;
    odp_atomic_u32_t exit_test;
    odp_atomic_u32_t timers_started;
    odp_barrier_t barrier;
    odp_cpumask_t cpumask;
    timer_pool_t timer_pool[MAX_TIMER_POOLS];
    odp_pool_t pool[MAX_TIMER_POOLS];
    odp_queue_t queue[MAX_TIMER_POOLS];
    odp_timer_t timer[MAX_TIMER_POOLS][MAX_TIMERS];
    timer_ctx_t timer_ctx[MAX_TIMER_POOLS][MAX_TIMERS];
    odph_thread_t thread_tbl[ODP_THREAD_COUNT_MAX];
    test_stat_t stat[ODP_THREAD_COUNT_MAX];
    thread_arg_t thread_arg[ODP_THREAD_COUNT_MAX];
    test_stat_sum_t stat_sum;
 
} test_global_t;
 
test_global_t *test_global;
 
static void print_usage(void)
{
    printf("\n"
           "Timer performance test\n"
           "\n"
           "Usage: odp_timer_perf [options]\n"
           "\n"
           "  -c, --num_cpu          Number of CPUs (worker threads). 0: all available CPUs. Default: 1\n"
           "  -n, --num_tp           Number of timer pools. Default: 1\n"
           "  -t, --num_timer        Number of timers per timer pool. Default: 10\n"
           "  -r, --res_ns           Resolution in nsec.     Default:  10000000\n"
           "  -p, --period_ns        Timeout period in nsec. Default: 100000000\n"
           "  -s, --shared           Shared vs private timer pool. Currently, private pools can be\n"
           "                         tested only with single CPU. Default: 1\n"
           "                           0: Private timer pools\n"
           "                           1: Shared timer pools\n"
           "  -m, --mode             Select test mode. Default: 0\n"
           "                           0: Measure odp_schedule() overhead when using timers\n"
           "                           1: Measure timer set + cancel performance\n"
           "                           2: Measure schedule and timer start overhead while continuously\n"
           "                              restarting expiring timers\n"
           "  -R, --rounds           Number of test rounds in timer set + cancel test.\n"
           "                           Default: 100000\n"
           "  -h, --help             This help\n"
           "\n");
}
 
static int parse_options(int argc, char *argv[], test_options_t *test_options)
{
    int opt;
    int ret = 0;
 
    static const struct option longopts[] = {
        {"num_cpu",   required_argument, NULL, 'c'},
        {"num_tp   ", required_argument, NULL, 'n'},
        {"num_timer", required_argument, NULL, 't'},
        {"res_ns",    required_argument, NULL, 'r'},
        {"period_ns", required_argument, NULL, 'p'},
        {"shared",    required_argument, NULL, 's'},
        {"mode",      required_argument, NULL, 'm'},
        {"rounds",    required_argument, NULL, 'R'},
        {"help",      no_argument,       NULL, 'h'},
        {NULL, 0, NULL, 0}
    };
 
    static const char *shortopts = "+c:n:t:r:p:s:m:R:h";
 
    test_options->num_cpu   = 1;
    test_options->num_tp    = 1;
    test_options->num_timer = 10;
    test_options->res_ns    = 10 * ODP_TIME_MSEC_IN_NS;
    test_options->period_ns = 100 * ODP_TIME_MSEC_IN_NS;
    test_options->shared    = 1;
    test_options->mode      = 0;
    test_options->test_rounds = 100000;
 
    while (1) {
        opt = getopt_long(argc, argv, shortopts, longopts, NULL);
 
        if (opt == -1)
            break;
 
        switch (opt) {
        case 'c':
            test_options->num_cpu = atoi(optarg);
            break;
        case 'n':
            test_options->num_tp = atoi(optarg);
            break;
        case 't':
            test_options->num_timer = atoi(optarg);
            break;
        case 'r':
            test_options->res_ns = atoll(optarg);
            break;
        case 'p':
            test_options->period_ns = atoll(optarg);
            break;
        case 's':
            test_options->shared = atoi(optarg);
            break;
        case 'm':
            test_options->mode = atoi(optarg);
            break;
        case 'R':
            test_options->test_rounds = atoll(optarg);
            break;
        case 'h':
            /* fall through */
        default:
            print_usage();
            ret = -1;
            break;
        }
    }
 
    if (test_options->num_timer > MAX_TIMERS) {
        ODPH_ERR("Too many timers. Max %u\n", MAX_TIMERS);
        ret = -1;
    }
 
    return ret;
}
 
static int set_num_cpu(test_global_t *global)
{
    int ret;
    test_options_t *test_options = &global->test_options;
    int num_cpu = test_options->num_cpu;
    int shared = test_options->shared;
 
    /* One thread used for the main thread */
    if (num_cpu > ODP_THREAD_COUNT_MAX - 1) {
        ODPH_ERR("Too many workers. Maximum is %i.\n", ODP_THREAD_COUNT_MAX - 1);
        return -1;
    }
 
    ret = odp_cpumask_default_worker(&global->cpumask, num_cpu);
 
    if (num_cpu && ret != num_cpu) {
        ODPH_ERR("Too many workers. Max supported %i\n.", ret);
        return -1;
    }
 
    if (shared == 0 && num_cpu != 1) {
        ODPH_ERR("Private pool test supports only single CPU\n.");
        return -1;
    }
 
    /* Zero: all available workers */
    if (num_cpu == 0) {
        num_cpu = ret;
        test_options->num_cpu = num_cpu;
    }
 
    if (shared) /* Main thread + all workers */
        odp_barrier_init(&global->barrier, num_cpu + 1);
    else /* Only the main thread */
        odp_barrier_init(&global->barrier, 1);
 
    return 0;
}
 
static int create_timer_pools(test_global_t *global)
{
    odp_timer_capability_t timer_capa;
    odp_timer_res_capability_t timer_res_capa;
    odp_timer_pool_param_t timer_pool_param;
    odp_timer_pool_t tp;
    odp_queue_param_t queue_param;
    odp_queue_t queue;
    odp_pool_param_t pool_param;
    odp_pool_t pool;
    uint64_t max_tmo_ns, min_tmo_ns;
    uint32_t i, j;
    uint32_t max_timers;
    int priv;
    test_options_t *test_options = &global->test_options;
    uint32_t num_cpu   = test_options->num_cpu;
    uint32_t num_tp    = test_options->num_tp;
    uint32_t num_timer = test_options->num_timer;
    uint64_t res_ns    = test_options->res_ns;
    uint64_t period_ns = test_options->period_ns;
    int mode = test_options->mode;
    char tp_name[] = "timer_pool_00";
 
    max_tmo_ns = START_NS + (num_timer * period_ns);
    min_tmo_ns = START_NS / 2;
 
    if (test_options->mode == MODE_START_EXPIRE) {
        /*
         * Timers are set to 1-2 periods from current time. Add an
         * arbitrary margin of one period, resulting in maximum of
         * three periods.
         */
        max_tmo_ns = period_ns * 3;
        min_tmo_ns = test_options->res_ns / 2;
    }
 
    priv = 0;
    if (test_options->shared == 0)
        priv = 1;
 
    printf("\nTimer performance test\n");
    printf("  mode             %i\n", mode);
    printf("  num cpu          %u\n", num_cpu);
    printf("  private pool     %i\n", priv);
    printf("  num timer pool   %u\n", num_tp);
    printf("  num timer        %u\n", num_timer);
    printf("  resolution       %" PRIu64 " nsec\n", res_ns);
    printf("  period           %" PRIu64 " nsec\n", period_ns);
    printf("  max timeout      %" PRIu64 " nsec\n", max_tmo_ns);
    printf("  min timeout      %" PRIu64 " nsec\n", min_tmo_ns);
    printf("  first timer at   %.2f sec\n", (double)START_NS / ODP_TIME_SEC_IN_NS);
    if (mode == MODE_SCHED_OVERH)
        printf("  test duration    %.2f sec\n", (double)max_tmo_ns / ODP_TIME_SEC_IN_NS);
    else
        printf("  test rounds      %" PRIu64 "\n", test_options->test_rounds);
 
    for (i = 0; i < MAX_TIMER_POOLS; i++) {
        global->timer_pool[i].tp = ODP_TIMER_POOL_INVALID;
        global->pool[i]  = ODP_POOL_INVALID;
        global->queue[i] = ODP_QUEUE_INVALID;
 
        for (j = 0; j < MAX_TIMERS; j++)
            global->timer[i][j] = ODP_TIMER_INVALID;
    }
 
    if (odp_timer_capability(ODP_CLOCK_DEFAULT, &timer_capa)) {
        ODPH_ERR("Timer capability failed\n");
        return -1;
    }
 
    memset(&timer_res_capa, 0, sizeof(odp_timer_res_capability_t));
    timer_res_capa.res_ns = res_ns;
    if (odp_timer_res_capability(ODP_CLOCK_DEFAULT, &timer_res_capa)) {
        ODPH_ERR("Timer resolution capability failed\n");
        return -1;
    }
 
    if (res_ns < timer_capa.max_res.res_ns) {
        ODPH_ERR("Too high resolution\n");
        return -1;
    }
 
    if (min_tmo_ns < timer_res_capa.min_tmo) {
        ODPH_ERR("Too short min timeout\n");
        return -1;
    }
 
    if (max_tmo_ns > timer_res_capa.max_tmo) {
        ODPH_ERR("Too long max timeout\n");
        return -1;
    }
 
    max_timers = timer_capa.max_timers;
    if (max_timers && num_timer > max_timers) {
        ODPH_ERR("Too many timers (max %u)\n", max_timers);
        return -1;
    }
 
    if (num_tp > timer_capa.max_pools) {
        ODPH_ERR("Too many timer pools (max %u)\n", timer_capa.max_pools);
        return -1;
    }
 
    odp_timer_pool_param_init(&timer_pool_param);
    timer_pool_param.res_ns     = res_ns;
    timer_pool_param.min_tmo    = min_tmo_ns;
    timer_pool_param.max_tmo    = max_tmo_ns;
    timer_pool_param.num_timers = num_timer;
    timer_pool_param.priv       = priv;
    timer_pool_param.clk_src    = ODP_CLOCK_DEFAULT;
 
    odp_pool_param_init(&pool_param);
    pool_param.type    = ODP_POOL_TIMEOUT;
    pool_param.tmo.num = num_timer;
 
    odp_queue_param_init(&queue_param);
    queue_param.type        = ODP_QUEUE_TYPE_SCHED;
    queue_param.sched.prio  = odp_schedule_default_prio();
    queue_param.sched.sync  = ODP_SCHED_SYNC_ATOMIC;
    queue_param.sched.group = ODP_SCHED_GROUP_ALL;
 
    for (i = 0; i < num_tp; i++) {
        if (num_tp < 100) {
            tp_name[11] = '0' + i / 10;
            tp_name[12] = '0' + i % 10;
        }
 
        tp = odp_timer_pool_create(tp_name, &timer_pool_param);
        global->timer_pool[i].tp = tp;
        if (tp == ODP_TIMER_POOL_INVALID) {
            ODPH_ERR("Timer pool create failed (%u)\n", i);
            return -1;
        }
 
        if (odp_timer_pool_start_multi(&tp, 1) != 1) {
            ODPH_ERR("Timer pool start failed (%u)\n", i);
            return -1;
        }
 
        pool = odp_pool_create(tp_name, &pool_param);
        global->pool[i] = pool;
        if (pool == ODP_POOL_INVALID) {
            ODPH_ERR("Pool create failed (%u)\n", i);
            return -1;
        }
 
        queue = odp_queue_create(tp_name, &queue_param);
        global->queue[i] = queue;
        if (queue == ODP_QUEUE_INVALID) {
            ODPH_ERR("Queue create failed (%u)\n", i);
            return -1;
        }
 
        global->timer_pool[i].period_tick = odp_timer_ns_to_tick(tp,
                                     test_options->period_ns);
        global->timer_pool[i].start_tick = odp_timer_ns_to_tick(tp, START_NS);
    }
 
    printf("  start            %" PRIu64 " tick\n",  global->timer_pool[0].start_tick);
    printf("  period           %" PRIu64 " ticks\n", global->timer_pool[0].period_tick);
    printf("\n");
 
    return 0;
}
 
static int set_timers(test_global_t *global)
{
    odp_timer_pool_info_t timer_pool_info;
    odp_timer_pool_t tp;
    odp_timer_t timer;
    odp_pool_t pool;
    odp_queue_t queue;
    odp_time_t time;
    uint64_t tick_cur, nsec, time_ns;
    uint64_t max_tmo_ns;
    uint32_t i, j;
    test_options_t *test_options = &global->test_options;
    uint32_t num_tp    = test_options->num_tp;
    uint32_t num_timer = test_options->num_timer;
    uint64_t period_ns = test_options->period_ns;
 
    max_tmo_ns = START_NS + (num_timer * period_ns);
 
    for (i = 0; i < num_tp; i++) {
        tp    = global->timer_pool[i].tp;
        pool  = global->pool[i];
        queue = global->queue[i];
 
        nsec     = max_tmo_ns;
        tick_cur = odp_timer_current_tick(tp);
        time     = odp_time_global();
        time_ns  = odp_time_to_ns(time);
 
        for (j = 0; j < num_timer; j++) {
            uint64_t tick_ns;
            odp_timeout_t timeout;
            odp_event_t ev;
            int status;
            timer_ctx_t *ctx = &global->timer_ctx[i][j];
            odp_timer_start_t start_param;
 
            /* Set timers backwards, the last timer is set first */
            if (j == 0)
                ctx->last = 1;
 
            ctx->target_ns = time_ns + nsec;
            ctx->tp_idx    = i;
            ctx->timer_idx = j;
 
            timeout = odp_timeout_alloc(pool);
            ev = odp_timeout_to_event(timeout);
 
            timer = odp_timer_alloc(tp, queue, ctx);
            global->timer[i][j] = timer;
 
            tick_ns = odp_timer_ns_to_tick(tp, nsec);
            nsec    = nsec - period_ns;
 
            start_param.tick_type = ODP_TIMER_TICK_ABS;
            start_param.tick = tick_cur + tick_ns;
            start_param.tmo_ev = ev;
 
            if (test_options->mode == MODE_START_EXPIRE) {
                uint64_t offset_ns = period_ns + j * period_ns / num_timer;
 
                ctx->target_ns = time_ns + offset_ns;
                ctx->target_tick = tick_cur + odp_timer_ns_to_tick(tp, offset_ns);
                start_param.tick = ctx->target_tick;
            }
 
            status = odp_timer_start(timer, &start_param);
            if (status != ODP_TIMER_SUCCESS) {
                ODPH_ERR("Timer set %i/%i (ret %i)\n", i, j, status);
                return -1;
            }
        }
 
        if (odp_timer_pool_info(tp, &timer_pool_info)) {
            ODPH_ERR("Timer pool info failed\n");
            return -1;
        }
 
        printf("Timer pool info [%i]:\n", i);
        printf("  cur_timers       %u\n", timer_pool_info.cur_timers);
        printf("  hwm_timers       %u\n", timer_pool_info.hwm_timers);
        printf("\n");
    }
 
    return 0;
}
 
static int destroy_timer_pool(test_global_t *global)
{
    odp_timer_pool_t tp;
    odp_pool_t pool;
    odp_queue_t queue;
    odp_timer_t timer;
    uint32_t i, j;
    test_options_t *test_options = &global->test_options;
    uint32_t num_timer = test_options->num_timer;
    uint32_t num_tp = test_options->num_tp;
 
    for (i = 0; i < num_tp; i++) {
        for (j = 0; j < num_timer; j++) {
            timer = global->timer[i][j];
 
            if (timer == ODP_TIMER_INVALID)
                break;
 
            if (odp_timer_free(timer))
                printf("Timer free failed: %i/%i\n", i, j);
        }
 
        queue = global->queue[i];
        if (queue != ODP_QUEUE_INVALID)
            odp_queue_destroy(queue);
 
        pool = global->pool[i];
        if (pool != ODP_POOL_INVALID)
            odp_pool_destroy(pool);
 
        tp = global->timer_pool[i].tp;
        if (tp != ODP_TIMER_POOL_INVALID)
            odp_timer_pool_destroy(tp);
    }
 
    return 0;
}
 
static int sched_mode_worker(void *arg)
{
    int thr;
    uint32_t exit_test;
    odp_event_t ev;
    odp_timeout_t tmo;
    uint64_t c2, diff, nsec, time_ns, target_ns;
    odp_time_t t1, t2, time;
    time_stat_t before, after;
    timer_ctx_t *ctx;
    thread_arg_t *thread_arg = arg;
    test_global_t *global = thread_arg->global;
    test_options_t *test_options = &global->test_options;
    uint32_t num_tp = test_options->num_tp;
    uint64_t cycles = 0;
    uint64_t events = 0;
    uint64_t rounds = 0;
    uint64_t c1 = 0;
    int meas = 1;
    int ret = 0;
 
    memset(&before, 0, sizeof(time_stat_t));
    memset(&after, 0, sizeof(time_stat_t));
 
    thr = odp_thread_id();
 
    /* Start all workers at the same time */
    odp_barrier_wait(&global->barrier);
 
    t1 = odp_time_local();
 
    while (1) {
        if (meas) {
            c1   = odp_cpu_cycles();
            meas = 0;
        }
 
        ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT);
        rounds++;
 
        exit_test = odp_atomic_load_u32(&global->exit_test);
        if (odp_likely(ev == ODP_EVENT_INVALID && exit_test < num_tp))
            continue;
 
        c2      = odp_cpu_cycles();
        diff    = odp_cpu_cycles_diff(c2, c1);
        cycles += diff;
 
        if (ev == ODP_EVENT_INVALID && exit_test >= num_tp)
            break;
 
        time = odp_time_global();
        time_ns = odp_time_to_ns(time);
        events++;
        meas = 1;
 
        tmo  = odp_timeout_from_event(ev);
        ctx  = odp_timeout_user_ptr(tmo);
        odp_timeout_free(tmo);
 
        target_ns = ctx->target_ns;
        if (time_ns < target_ns) {
            diff = target_ns - time_ns;
            before.num++;
            before.sum_ns += diff;
            if (diff > before.max_ns)
                before.max_ns = diff;
 
            ODPH_DBG("before %" PRIu64 "\n", diff);
        } else {
            diff = time_ns - target_ns;
            after.num++;
            after.sum_ns += diff;
            if (diff > after.max_ns)
                after.max_ns = diff;
 
            ODPH_DBG("after %" PRIu64 "\n", time_ns - target_ns);
        }
 
        if (ctx->last)
            odp_atomic_inc_u32(&global->exit_test);
    }
 
    t2   = odp_time_local();
    nsec = odp_time_diff_ns(t2, t1);
 
    /* Update stats*/
    global->stat[thr].events = events;
    global->stat[thr].cycles_0 = cycles;
    global->stat[thr].rounds = rounds;
    global->stat[thr].nsec   = nsec;
    global->stat[thr].before = before;
    global->stat[thr].after  = after;
 
    return ret;
}
 
static int cancel_timers(test_global_t *global, uint32_t worker_idx)
{
    uint32_t i, j;
    int r;
    odp_timer_t timer;
    odp_event_t ev;
    test_options_t *test_options = &global->test_options;
    uint32_t num_tp = test_options->num_tp;
    uint32_t num_timer = test_options->num_timer;
    uint32_t num_worker = test_options->num_cpu;
    int ret = 0;
 
    for (i = 0; i < num_tp; i++) {
        for (j = worker_idx; j < num_timer; j += num_worker) {
            timer = global->timer[i][j];
            if (timer == ODP_TIMER_INVALID)
                continue;
 
            r = odp_timer_cancel(timer, &ev);
 
            if (r == ODP_TIMER_SUCCESS) {
                odp_event_free(ev);
            } else if (r == ODP_TIMER_TOO_NEAR) {
                ret = 1;
            } else {
                ret = -1;
                break;
            }
        }
    }
 
    return ret;
}
 
static int set_cancel_mode_worker(void *arg)
{
    uint64_t tick, start_tick, period_tick, nsec;
    uint64_t c1, c2;
    int thr, status;
    uint32_t i, j, worker_idx;
    odp_event_t ev;
    odp_time_t t1, t2;
    odp_timer_t timer;
    odp_timer_pool_t tp;
    odp_timeout_t tmo;
    odp_timer_start_t start_param;
    thread_arg_t *thread_arg = arg;
    test_global_t *global = thread_arg->global;
    test_options_t *test_options = &global->test_options;
    uint32_t num_tp = test_options->num_tp;
    uint32_t num_timer = test_options->num_timer;
    uint32_t num_worker = test_options->num_cpu;
    int ret = 0;
    int started = 0;
    uint64_t test_rounds = test_options->test_rounds;
    uint64_t num_tmo = 0;
    uint64_t num_cancel = 0;
    uint64_t num_set = 0;
    uint64_t cancel_cycles = 0, start_cycles = 0;
    odp_event_t ev_tbl[MAX_TIMERS];
 
    thr = odp_thread_id();
    worker_idx = thread_arg->worker_idx;
    t1 = ODP_TIME_NULL;
 
    /* Start all workers at the same time */
    odp_barrier_wait(&global->barrier);
 
    while (1) {
        ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT);
 
        if (odp_unlikely(ev != ODP_EVENT_INVALID)) {
            /* Timeout, set timer again. When start_tick is large enough, this should
             * not happen. */
            timer_ctx_t *ctx;
 
            tmo   = odp_timeout_from_event(ev);
            ctx   = odp_timeout_user_ptr(tmo);
            i     = ctx->tp_idx;
            j     = ctx->timer_idx;
            timer = global->timer[i][j];
            start_tick  = global->timer_pool[i].start_tick;
            period_tick = global->timer_pool[i].period_tick;
            tick = start_tick + j * period_tick;
 
            start_param.tick_type = ODP_TIMER_TICK_REL;
            start_param.tick = tick;
            start_param.tmo_ev = ev;
 
            status = odp_timer_start(timer, &start_param);
            num_tmo++;
            num_set++;
 
            if (status != ODP_TIMER_SUCCESS) {
                ODPH_ERR("Timer set (tmo) failed (ret %i)\n", status);
                ret = -1;
                break;
            }
 
            continue;
        }
 
        if (odp_unlikely(odp_atomic_load_u32(&global->exit_test)))
            break;
 
        if (odp_unlikely(started == 0)) {
            /* Run schedule loop while waiting for timers to be created */
            if (odp_atomic_load_acq_u32(&global->timers_started) == 0)
                continue;
 
            /* Start measurements */
            started = 1;
            t1 = odp_time_local();
        }
 
        /* Cancel and set timers again */
        for (i = 0; i < num_tp; i++) {
            tp = global->timer_pool[i].tp;
            if (tp == ODP_TIMER_POOL_INVALID)
                continue;
 
            start_tick  = global->timer_pool[i].start_tick;
            period_tick = global->timer_pool[i].period_tick;
 
            tick = odp_timer_current_tick(tp) + start_tick;
            c1 = odp_cpu_cycles();
 
            for (j = worker_idx; j < num_timer; j += num_worker) {
                ev_tbl[j] = ODP_EVENT_INVALID;
 
                timer = global->timer[i][j];
                if (timer == ODP_TIMER_INVALID)
                    continue;
 
                status = odp_timer_cancel(timer, &ev_tbl[j]);
                num_cancel++;
 
                if (odp_unlikely(status == ODP_TIMER_TOO_NEAR)) {
                    continue;
                } else if (odp_unlikely(status != ODP_TIMER_SUCCESS)) {
                    ODPH_ERR("Timer (%u/%u) cancel failed (ret %i)\n", i, j,
                         status);
                    ret = -1;
                    break;
                }
            }
 
            c2 = odp_cpu_cycles();
            cancel_cycles += odp_cpu_cycles_diff(c2, c1);
            c1 = c2;
 
            for (j = worker_idx; j < num_timer; j += num_worker) {
                if (ev_tbl[j] == ODP_EVENT_INVALID)
                    continue;
 
                timer = global->timer[i][j];
                if (timer == ODP_TIMER_INVALID)
                    continue;
 
                start_param.tick_type = ODP_TIMER_TICK_ABS;
                start_param.tick = tick + j * period_tick;
                start_param.tmo_ev = ev_tbl[j];
 
                status = odp_timer_start(timer, &start_param);
                num_set++;
 
                if (status != ODP_TIMER_SUCCESS) {
                    ODPH_ERR("Timer (%u/%u) set failed (ret %i)\n", i, j,
                         status);
                    ret = -1;
                    break;
                }
            }
 
            c2 = odp_cpu_cycles();
            start_cycles += odp_cpu_cycles_diff(c2, c1);
        }
 
        if (test_rounds) {
            test_rounds--;
            if (test_rounds == 0)
                break;
        }
    }
 
    t2   = odp_time_local();
    nsec = odp_time_diff_ns(t2, t1);
 
    /* Cancel all timers that belong to this thread */
    if (cancel_timers(global, worker_idx))
        ODPH_ERR("Timer cancel failed\n");
 
    /* Update stats */
    global->stat[thr].events = num_tmo;
    global->stat[thr].rounds = test_options->test_rounds - test_rounds;
    global->stat[thr].nsec   = nsec;
    global->stat[thr].cycles_0 = cancel_cycles;
    global->stat[thr].cycles_1 = start_cycles;
 
    global->stat[thr].cancels = num_cancel;
    global->stat[thr].sets    = num_set;
 
    return ret;
}
 
static int set_expire_mode_worker(void *arg)
{
    int status, thr;
    uint32_t i, j, exit_test;
    odp_event_t ev;
    odp_timeout_t tmo;
    uint64_t c2, c3, c4, diff, nsec, time_ns, target_ns, period_tick, wait;
    odp_timer_t timer;
    odp_timer_start_t start_param;
    odp_time_t t1, t2;
    time_stat_t before, after;
    timer_ctx_t *ctx;
    thread_arg_t *thread_arg = arg;
    test_global_t *global = thread_arg->global;
    test_options_t *opt = &global->test_options;
    uint32_t num_tp = opt->num_tp;
    uint64_t sched_cycles = 0;
    uint64_t start_cycles = 0;
    uint64_t events = 0;
    uint64_t rounds = 0;
    uint64_t c1 = 0;
    int meas = 1;
    int ret = 0;
 
    memset(&before, 0, sizeof(time_stat_t));
    memset(&after, 0, sizeof(time_stat_t));
 
    thr = odp_thread_id();
 
    /* Start all workers at the same time */
    odp_barrier_wait(&global->barrier);
 
    t1 = odp_time_local();
 
    while (events < opt->test_rounds * opt->num_timer / opt->num_cpu) {
        if (meas) {
            c1   = odp_cpu_cycles();
            meas = 0;
        }
 
        ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT);
        rounds++;
 
        exit_test = odp_atomic_load_u32(&global->exit_test);
        if (odp_likely(ev == ODP_EVENT_INVALID && exit_test < num_tp))
            continue;
 
        c2      = odp_cpu_cycles();
        diff    = odp_cpu_cycles_diff(c2, c1);
        sched_cycles += diff;
 
        if (ev == ODP_EVENT_INVALID && exit_test >= num_tp)
            break;
 
        events++;
        meas = 1;
        tmo = odp_timeout_from_event(ev);
        ctx = odp_timeout_user_ptr(tmo);
        i = ctx->tp_idx;
        j = ctx->timer_idx;
        timer = global->timer[i][j];
        period_tick = global->timer_pool[i].period_tick;
        time_ns = odp_time_global_ns();
        target_ns = ctx->target_ns;
 
        if (time_ns < target_ns) {
            diff = target_ns - time_ns;
            before.num++;
            before.sum_ns += diff;
            if (diff > before.max_ns)
                before.max_ns = diff;
 
            ODPH_DBG("before %" PRIu64 "\n", diff);
        } else {
            diff = time_ns - target_ns;
            after.num++;
            after.sum_ns += diff;
            if (diff > after.max_ns)
                after.max_ns = diff;
 
            ODPH_DBG("after %" PRIu64 "\n", diff);
        }
 
        /* Start the timer again */
        start_param.tick_type = ODP_TIMER_TICK_ABS;
        ctx->target_ns += opt->period_ns;
        ctx->target_tick += period_tick;
        start_param.tick = ctx->target_tick;
        start_param.tmo_ev = ev;
        c3 = odp_cpu_cycles();
 
        status = odp_timer_start(timer, &start_param);
 
        c4 = odp_cpu_cycles();
        diff = odp_cpu_cycles_diff(c4, c3);
        start_cycles += diff;
 
        if (status != ODP_TIMER_SUCCESS) {
            ODPH_ERR("Timer set (tmo) failed (ret %i)\n", status);
            ret = -1;
            break;
        }
    }
 
    t2   = odp_time_local();
    nsec = odp_time_diff_ns(t2, t1);
 
    /* Cancel all timers that belong to this thread */
    status = cancel_timers(global, thread_arg->worker_idx);
 
    wait = ODP_SCHED_NO_WAIT;
    if (status > 0)
        wait = odp_schedule_wait_time(opt->period_ns);
 
    /* Wait and free remaining events */
    while (1) {
        ev = odp_schedule(NULL, wait);
        if (ev == ODP_EVENT_INVALID)
            break;
        odp_event_free(ev);
    }
 
    /* Update stats*/
    global->stat[thr].events = events;
    global->stat[thr].cycles_0 = sched_cycles;
    global->stat[thr].cycles_1 = start_cycles;
    global->stat[thr].rounds = rounds;
    global->stat[thr].nsec   = nsec;
    global->stat[thr].before = before;
    global->stat[thr].after  = after;
 
    return ret;
}
 
static int start_workers(test_global_t *global, odp_instance_t instance)
{
    odph_thread_common_param_t thr_common;
    int i, ret;
    test_options_t *test_options = &global->test_options;
    int num_cpu   = test_options->num_cpu;
    odph_thread_param_t thr_param[num_cpu];
 
    memset(global->thread_tbl, 0, sizeof(global->thread_tbl));
    odph_thread_common_param_init(&thr_common);
 
    thr_common.instance = instance;
    thr_common.cpumask  = &global->cpumask;
 
    for (i = 0; i < num_cpu; i++) {
        odph_thread_param_init(&thr_param[i]);
 
        if (test_options->mode == MODE_SCHED_OVERH)
            thr_param[i].start = sched_mode_worker;
        else if (test_options->mode == MODE_START_CANCEL)
            thr_param[i].start = set_cancel_mode_worker;
        else
            thr_param[i].start = set_expire_mode_worker;
 
        thr_param[i].arg      = &global->thread_arg[i];
        thr_param[i].thr_type = ODP_THREAD_WORKER;
    }
 
    ret = odph_thread_create(global->thread_tbl, &thr_common, thr_param,
                 num_cpu);
 
    if (ret != num_cpu) {
        ODPH_ERR("Thread create failed %i\n", ret);
        return -1;
    }
 
    return 0;
}
 
static void sum_stat(test_global_t *global)
{
    int i;
    test_stat_sum_t *sum = &global->stat_sum;
 
    memset(sum, 0, sizeof(test_stat_sum_t));
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        if (global->stat[i].rounds == 0)
            continue;
 
        sum->num++;
        sum->events  += global->stat[i].events;
        sum->rounds  += global->stat[i].rounds;
        sum->cycles_0 += global->stat[i].cycles_0;
        sum->cycles_1 += global->stat[i].cycles_1;
        sum->nsec    += global->stat[i].nsec;
        sum->cancels += global->stat[i].cancels;
        sum->sets    += global->stat[i].sets;
 
        sum->before.num    += global->stat[i].before.num;
        sum->before.sum_ns += global->stat[i].before.sum_ns;
        sum->after.num     += global->stat[i].after.num;
        sum->after.sum_ns  += global->stat[i].after.sum_ns;
 
        if (global->stat[i].before.max_ns > sum->before.max_ns)
            sum->before.max_ns = global->stat[i].before.max_ns;
 
        if (global->stat[i].after.max_ns > sum->after.max_ns)
            sum->after.max_ns = global->stat[i].after.max_ns;
    }
 
    if (sum->num)
        sum->time_ave = ((double)sum->nsec / sum->num) / ODP_TIME_SEC_IN_NS;
}
 
static void print_stat_sched_mode(test_global_t *global)
{
    int i;
    test_stat_sum_t *sum = &global->stat_sum;
    double round_ave = 0.0;
    double before_ave = 0.0;
    double after_ave = 0.0;
    int num = 0;
 
    printf("\n");
    printf("RESULTS - schedule() cycles per thread:\n");
    printf("----------------------------------------------\n");
    printf("        1      2      3      4      5      6      7      8      9     10");
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        if (global->stat[i].rounds) {
            if ((num % 10) == 0)
                printf("\n   ");
 
            printf("%6.1f ", (double)global->stat[i].cycles_0 / global->stat[i].rounds);
            num++;
        }
    }
 
    printf("\n\n");
 
    if (sum->num)
        round_ave = (double)sum->rounds / sum->num;
 
    if (sum->before.num)
        before_ave = (double)sum->before.sum_ns / sum->before.num;
 
    if (sum->after.num)
        after_ave = (double)sum->after.sum_ns / sum->after.num;
 
    printf("TOTAL (%i workers)\n", sum->num);
    printf("  events:             %" PRIu64 "\n", sum->events);
    printf("  ave time:           %.2f sec\n", sum->time_ave);
    printf("  ave rounds per sec: %.2fM\n", (round_ave / sum->time_ave) / 1000000.0);
    printf("  num before:         %" PRIu64 "\n", sum->before.num);
    printf("  ave before:         %.1f nsec\n", before_ave);
    printf("  max before:         %" PRIu64 " nsec\n", sum->before.max_ns);
    printf("  num after:          %" PRIu64 "\n", sum->after.num);
    printf("  ave after:          %.1f nsec\n", after_ave);
    printf("  max after:          %" PRIu64 " nsec\n", sum->after.max_ns);
    printf("\n");
}
 
static void print_stat_set_cancel_mode(test_global_t *global)
{
    int i;
    test_stat_sum_t *sum = &global->stat_sum;
    double set_ave = 0.0;
    int num = 0;
 
    printf("\n");
    printf("RESULTS\n");
    printf("odp_timer_cancel() cycles per thread:\n");
    printf("-------------------------------------------------\n");
    printf("        1      2      3      4      5      6      7      8      9     10");
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        const test_stat_t *si = &global->stat[i];
 
        if (si->cancels) {
            if ((num % 10) == 0)
                printf("\n   ");
 
            printf("%6.1f ", (double)si->cycles_0 / si->cancels);
            num++;
        }
    }
 
    printf("\n\n");
 
    num = 0;
    printf("odp_timer_start() cycles per thread:\n");
    printf("-------------------------------------------------\n");
    printf("        1      2      3      4      5      6      7      8      9     10");
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        const test_stat_t *si = &global->stat[i];
 
        if (si->sets) {
            if ((num % 10) == 0)
                printf("\n   ");
 
            printf("%6.1f ", (double)si->cycles_1 / si->sets);
            num++;
        }
    }
 
    if (sum->num)
        set_ave = (double)sum->sets / sum->num;
 
    printf("\n\n");
    printf("TOTAL (%i workers)\n", sum->num);
    printf("  rounds:              %" PRIu64 "\n", sum->rounds);
    printf("  timeouts:            %" PRIu64 "\n", sum->events);
    printf("  timer cancels:       %" PRIu64 "\n", sum->cancels);
    printf("  cancels failed:      %" PRIu64 "\n", sum->cancels - sum->sets);
    printf("  timer sets:          %" PRIu64 "\n", sum->sets);
    printf("  ave time:            %.2f sec\n", sum->time_ave);
    printf("  cancel+set per cpu:  %.2fM per sec\n", (set_ave / sum->time_ave) / 1000000.0);
    printf("\n");
}
 
static void print_stat_expire_mode(test_global_t *global)
{
    int i;
    test_stat_sum_t *sum = &global->stat_sum;
    double round_ave = 0.0;
    double before_ave = 0.0;
    double after_ave = 0.0;
    int num = 0;
 
    printf("\n");
    printf("RESULTS\n");
    printf("odp_schedule() cycles per thread:\n");
    printf("-------------------------------------------------\n");
    printf("        1      2      3      4      5      6      7      8      9     10");
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        if (global->stat[i].rounds) {
            if ((num % 10) == 0)
                printf("\n   ");
 
            printf("%6.1f ", (double)global->stat[i].cycles_0 / global->stat[i].rounds);
            num++;
        }
    }
 
    printf("\n\n");
 
    num = 0;
    printf("odp_timer_start() cycles per thread:\n");
    printf("-------------------------------------------------\n");
    printf("        1      2      3      4      5      6      7      8      9     10");
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        if (global->stat[i].events) {
            if ((num % 10) == 0)
                printf("\n   ");
 
            printf("%6.1f ", (double)global->stat[i].cycles_1 / global->stat[i].events);
            num++;
        }
    }
 
    printf("\n\n");
 
    if (sum->num)
        round_ave = (double)sum->rounds / sum->num;
 
    if (sum->before.num)
        before_ave = (double)sum->before.sum_ns / sum->before.num;
 
    if (sum->after.num)
        after_ave = (double)sum->after.sum_ns / sum->after.num;
 
    printf("TOTAL (%i workers)\n", sum->num);
    printf("  events:             %" PRIu64 "\n", sum->events);
    printf("  ave time:           %.2f sec\n", sum->time_ave);
    printf("  ave rounds per sec: %.2fM\n", (round_ave / sum->time_ave) / 1000000.0);
    printf("  num before:         %" PRIu64 "\n", sum->before.num);
    printf("  ave before:         %.1f nsec\n", before_ave);
    printf("  max before:         %" PRIu64 " nsec\n", sum->before.max_ns);
    printf("  num after:          %" PRIu64 "\n", sum->after.num);
    printf("  ave after:          %.1f nsec\n", after_ave);
    printf("  max after:          %" PRIu64 " nsec\n", sum->after.max_ns);
    printf("\n");
}
 
static void sig_handler(int signo)
{
    (void)signo;
 
    if (test_global == NULL)
        return;
    odp_atomic_add_u32(&test_global->exit_test, MAX_TIMER_POOLS);
}
 
int main(int argc, char **argv)
{
    odph_helper_options_t helper_options;
    odp_instance_t instance;
    odp_init_t init;
    odp_shm_t shm;
    test_global_t *global;
    test_options_t *test_options;
    int i, shared, mode;
 
    signal(SIGINT, sig_handler);
 
    /* Let helper collect its own arguments (e.g. --odph_proc) */
    argc = odph_parse_options(argc, argv);
    if (odph_options(&helper_options)) {
        ODPH_ERR("Reading ODP helper options failed.\n");
        exit(EXIT_FAILURE);
    }
 
    /* List features not to be used */
    odp_init_param_init(&init);
    init.not_used.feat.cls      = 1;
    init.not_used.feat.compress = 1;
    init.not_used.feat.crypto   = 1;
    init.not_used.feat.ipsec    = 1;
    init.not_used.feat.tm       = 1;
 
    init.mem_model = helper_options.mem_model;
 
    /* Init ODP before calling anything else */
    if (odp_init_global(&instance, &init, NULL)) {
        ODPH_ERR("Global init failed.\n");
        return -1;
    }
 
    /* Init this thread */
    if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
        ODPH_ERR("Local init failed.\n");
        return -1;
    }
 
    shm = odp_shm_reserve("timer_perf_global", sizeof(test_global_t), ODP_CACHE_LINE_SIZE, 0);
    if (shm == ODP_SHM_INVALID) {
        ODPH_ERR("Shared mem reserve failed.\n");
        exit(EXIT_FAILURE);
    }
 
    global = odp_shm_addr(shm);
    if (global == NULL) {
        ODPH_ERR("Shared mem alloc failed\n");
        exit(EXIT_FAILURE);
    }
    test_global = global;
 
    memset(global, 0, sizeof(test_global_t));
    odp_atomic_init_u32(&global->exit_test, 0);
    odp_atomic_init_u32(&global->timers_started, 0);
 
    for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) {
        global->thread_arg[i].global = global;
        global->thread_arg[i].worker_idx = i;
    }
 
    if (parse_options(argc, argv, &global->test_options))
        return -1;
 
    test_options = &global->test_options;
    shared = test_options->shared;
    mode   = test_options->mode;
 
    odp_sys_info_print();
 
    odp_schedule_config(NULL);
 
    if (set_num_cpu(global))
        return -1;
 
    if (create_timer_pools(global))
        return -1;
 
    if (shared) {
        /* Start worker threads */
        start_workers(global, instance);
 
        /* Wait until workers have started.
         * Scheduler calls from workers may be needed to run timer
         * pools in a software implementation. Wait 1 msec to ensure
         * that timer pools are running before setting timers. */
        odp_barrier_wait(&global->barrier);
        odp_time_wait_ns(ODP_TIME_MSEC_IN_NS);
    }
 
    /* Set timers. Force workers to exit on failure. */
    if (set_timers(global))
        odp_atomic_add_u32(&global->exit_test, MAX_TIMER_POOLS);
    else
        odp_atomic_store_rel_u32(&global->timers_started, 1);
 
    if (!shared) {
        /* Test private pools on the master thread */
        if (mode == MODE_SCHED_OVERH) {
            if (sched_mode_worker(&global->thread_arg[0])) {
                ODPH_ERR("Sched_mode_worker failed\n");
                return -1;
            }
        } else if (mode == MODE_START_CANCEL) {
            if (set_cancel_mode_worker(&global->thread_arg[0])) {
                ODPH_ERR("Set_cancel_mode_worker failed\n");
                return -1;
            }
        } else {
            if (set_expire_mode_worker(&global->thread_arg[0])) {
                ODPH_ERR("Set_expire_mode_worker failed\n");
                return -1;
            }
        }
    } else {
        /* Wait workers to exit */
        odph_thread_join(global->thread_tbl,
                 global->test_options.num_cpu);
    }
 
    sum_stat(global);
 
    if (mode == MODE_SCHED_OVERH)
        print_stat_sched_mode(global);
    else if (mode == MODE_START_CANCEL)
        print_stat_set_cancel_mode(global);
    else
        print_stat_expire_mode(global);
 
    destroy_timer_pool(global);
 
    if (odp_shm_free(shm)) {
        ODPH_ERR("Shared mem free failed.\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_term_local()) {
        ODPH_ERR("Term local failed.\n");
        return -1;
    }
 
    if (odp_term_global(instance)) {
        ODPH_ERR("Term global failed.\n");
        return -1;
    }
 
    return 0;
}