odp_bench_buffer.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2017-2018 Linaro Limited
 * Copyright (c) 2022-2025 Nokia
 */
 
#include <odp_api.h>
#include <odp/helper/odph_api.h>
 
#include <bench_common.h>
#include <export_results.h>
 
#include <getopt.h>
#include <inttypes.h>
#include <signal.h>
#include <stdlib.h>
#include <unistd.h>
 
#define TEST_BUF_SIZE 1024
 
#define TEST_UAREA_SIZE 8
 
#define TEST_REPEAT_COUNT 1000
 
#define TEST_ROUNDS 100u
 
#define TEST_MAX_BURST 64
 
#define TEST_DEF_BURST 8
 
#define TEST_MAX_BENCH 40
 
#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
                strrchr((file_name), '/') + 1 : (file_name))
 
#define BENCH_INFO(run_fn, init_fn, term_fn, alt_name) \
    {.name = #run_fn, .run = run_fn, .init = init_fn, .term = term_fn, .desc = alt_name}
 
#define BENCH_INFO_COND(run_fn, init_fn, term_fn, alt_name, cond_fn) \
    {.name = #run_fn, .run = run_fn, .init = init_fn, .term = term_fn, .desc = alt_name, \
     .cond = cond_fn}
 
#define BENCH_INFO_TM(run_fn, init_fn, term_fn, cond_fn) \
    {.name = #run_fn, .run = run_fn, .init = init_fn, .term = term_fn, .cond = cond_fn,\
     .max_rounds = 0}
 
typedef enum {
    M_TPUT,
    M_LATENCY
} meas_mode_t;
 
typedef struct {
    int bench_idx;    
    int burst_size;   
    int cache_size;   
    int time;         
    meas_mode_t mode; 
    uint32_t rounds;  
} appl_args_t;
 
typedef struct {
    appl_args_t appl;
 
    struct {
        union {
            bench_suite_t b;
            bench_tm_suite_t t;
        };
 
        odp_atomic_u32_t *exit;
        int *retval;
        void *args;
        int (*suite_fn)(void *args);
        int (*export_fn)(void *data);
    } suite;
 
    odp_pool_t pool;
    uint32_t buf_size;
    uint32_t uarea_size;
    uint32_t max_flow_id;
    odp_buffer_t buf_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST];
    odp_event_t event_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST];
    void *ptr_tbl[TEST_REPEAT_COUNT];
    odp_pool_t pool_tbl[TEST_REPEAT_COUNT];
    odp_event_type_t event_type_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST];
    odp_event_subtype_t event_subtype_tbl[TEST_REPEAT_COUNT * TEST_MAX_BURST];
    char cpumask_str[ODP_CPUMASK_STR_SIZE];
    union{
        double b[TEST_MAX_BENCH];
        bench_tm_result_t t[TEST_MAX_BENCH];
    } result;
} args_t;
 
static args_t *gbl_args;
 
static void sig_handler(int signo ODP_UNUSED)
{
    if (gbl_args == NULL)
        return;
    odp_atomic_store_u32(gbl_args->suite.exit, 1);
}
 
static void allocate_test_buffers(odp_buffer_t buf[], int num)
{
    int num_buf = 0;
 
    while (num_buf < num) {
        int ret;
 
        ret = odp_buffer_alloc_multi(gbl_args->pool, &buf[num_buf], num - num_buf);
        if (ret < 0)
            ODPH_ABORT("Allocating test buffers failed\n");
 
        num_buf += ret;
    }
}
 
static void create_buffers(void)
{
    allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT);
}
 
static void create_buffers_multi(void)
{
    allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size);
}
 
static void create_events(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
 
    allocate_test_buffers(gbl_args->buf_tbl, TEST_REPEAT_COUNT);
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        gbl_args->event_tbl[i] = odp_buffer_to_event(buf_tbl[i]);
}
 
static void create_events_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
 
    allocate_test_buffers(gbl_args->buf_tbl,
                  TEST_REPEAT_COUNT * gbl_args->appl.burst_size);
 
    for (int i = 0; i < TEST_REPEAT_COUNT * gbl_args->appl.burst_size; i++)
        gbl_args->event_tbl[i] = odp_buffer_to_event(buf_tbl[i]);
}
 
static void free_buffers(void)
{
    odp_buffer_free_multi(gbl_args->buf_tbl, TEST_REPEAT_COUNT);
}
 
static void free_buffers_multi(void)
{
    odp_buffer_free_multi(gbl_args->buf_tbl, TEST_REPEAT_COUNT * gbl_args->appl.burst_size);
}
 
static int check_uarea(void)
{
    return !!gbl_args->uarea_size;
}
 
static int check_flow_aware(void)
{
    return !!gbl_args->max_flow_id;
}
 
static int buffer_from_event(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        buf_tbl[i] = odp_buffer_from_event(event_tbl[i]);
 
    return i;
}
 
static int buffer_from_event_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_from_event()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        buf_tbl[i] = odp_buffer_from_event(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_from_event_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_buffer_from_event_multi(&buf_tbl[i * burst_size],
                        &event_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int buffer_from_event_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_from_event_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_buffer_from_event_multi(&buf_tbl[i * burst_size],
                        &event_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_to_event(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        event_tbl[i] = odp_buffer_to_event(buf_tbl[i]);
 
    return i;
}
 
static int buffer_to_event_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_to_event()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        event_tbl[i] = odp_buffer_to_event(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_to_event_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_buffer_to_event_multi(&buf_tbl[i * burst_size],
                      &event_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int buffer_to_event_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_to_event_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_buffer_to_event_multi(&buf_tbl[i * burst_size],
                      &event_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_addr(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        ptr_tbl[i] = odp_buffer_addr(buf_tbl[i]);
 
    return i;
}
 
static int buffer_addr_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_addr()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ptr_tbl[i] = odp_buffer_addr(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_size(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    uint32_t ret = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        ret += odp_buffer_size(buf_tbl[i]);
 
    return ret;
}
 
static int buffer_size_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    uint32_t ret = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_size()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ret += odp_buffer_size(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return ret;
}
 
static int buffer_user_area(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        ptr_tbl[i] = odp_buffer_user_area(buf_tbl[i]);
 
    return i;
}
 
static int buffer_user_area_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_user_area()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ptr_tbl[i] = odp_buffer_user_area(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_pool(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t *pool_tbl = gbl_args->pool_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        pool_tbl[i] = odp_buffer_pool(buf_tbl[i]);
 
    return i;
}
 
static int buffer_pool_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t *pool_tbl = gbl_args->pool_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_pool()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        pool_tbl[i] = odp_buffer_pool(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_alloc(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        buf_tbl[i] = odp_buffer_alloc(pool);
 
    return i;
}
 
static int buffer_alloc_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_alloc()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        buf_tbl[i] = odp_buffer_alloc(pool);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_alloc_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int burst_size = gbl_args->appl.burst_size;
    int num = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        num += odp_buffer_alloc_multi(pool, &buf_tbl[num], burst_size);
 
    return num;
}
 
static int buffer_alloc_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int burst_size = gbl_args->appl.burst_size;
    int num = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_alloc_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        num += odp_buffer_alloc_multi(pool, &buf_tbl[num], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return num;
}
 
static int buffer_free(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_buffer_free(buf_tbl[i]);
 
    return i;
}
 
static int buffer_free_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_free()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_buffer_free(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_free_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_buffer_free_multi(&buf_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int buffer_free_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_free_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_buffer_free_multi(&buf_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_alloc_free(void)
{
    odp_pool_t pool = gbl_args->pool;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++) {
        odp_buffer_t buf = odp_buffer_alloc(pool);
 
        if (odp_unlikely(buf == ODP_BUFFER_INVALID))
            return 0;
 
        odp_buffer_free(buf);
    }
    return i;
}
 
static int buffer_alloc_free_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_pool_t pool = gbl_args->pool;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_alloc()/_free()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_buffer_t buf = odp_buffer_alloc(pool);
 
        ODPH_ASSERT(buf != ODP_BUFFER_INVALID);
 
        odp_buffer_free(buf);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_alloc_free_multi(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++) {
        int num = odp_buffer_alloc_multi(pool, buf_tbl, burst_size);
 
        if (odp_unlikely(num < 1))
            return 0;
 
        odp_buffer_free_multi(buf_tbl, num);
    }
    return i;
}
 
static int buffer_alloc_free_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    odp_pool_t pool = gbl_args->pool;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_alloc_multi()/_free_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        int num = odp_buffer_alloc_multi(pool, buf_tbl, burst_size);
 
        ODPH_ASSERT(num >= 1);
 
        odp_buffer_free_multi(buf_tbl, num);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int buffer_is_valid(void)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    uint32_t ret = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        ret += odp_buffer_is_valid(buf_tbl[i]);
 
    return ret;
}
 
static int buffer_is_valid_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_buffer_t *buf_tbl = gbl_args->buf_tbl;
    uint32_t ret = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_buffer_is_valid()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ret += odp_buffer_is_valid(buf_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return ret;
}
 
static int event_type(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        event_type_tbl[i] = odp_event_type(event_tbl[i]);
 
    return i;
}
 
static int event_type_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_type()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        event_type_tbl[i] = odp_event_type(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_subtype(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        event_subtype_tbl[i] = odp_event_subtype(event_tbl[i]);
 
    return i;
}
 
static int event_subtype_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_subtype()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        event_subtype_tbl[i] = odp_event_subtype(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_types(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        event_type_tbl[i] = odp_event_types(event_tbl[i], &event_subtype_tbl[i]);
 
    return i;
}
 
static int event_types_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_types()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        event_type_tbl[i] = odp_event_types(event_tbl[i], &event_subtype_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_types_multi(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_types_multi(&event_tbl[i * burst_size],
                      &event_type_tbl[i * burst_size],
                      &event_subtype_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int event_types_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    odp_event_subtype_t *event_subtype_tbl = gbl_args->event_subtype_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_types_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_types_multi(&event_tbl[i * burst_size],
                      &event_type_tbl[i * burst_size],
                      &event_subtype_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_types_multi_no_sub(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_types_multi(&event_tbl[i * burst_size],
                      &event_type_tbl[i * burst_size], NULL, burst_size);
 
    return i;
}
 
static int event_types_multi_no_sub_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_types_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_types_multi(&event_tbl[i * burst_size],
                      &event_type_tbl[i * burst_size], NULL, burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_type_multi(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int burst_size = gbl_args->appl.burst_size;
    uint32_t ret = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        ret += odp_event_type_multi(&event_tbl[i * burst_size], burst_size,
                        &event_type_tbl[i]);
 
    return ret;
}
 
static int event_type_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_event_type_t *event_type_tbl = gbl_args->event_type_tbl;
    int burst_size = gbl_args->appl.burst_size;
    uint32_t ret = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_type_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ret += odp_event_type_multi(&event_tbl[i * burst_size], burst_size,
                        &event_type_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return ret;
}
 
static int event_pool(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_pool_t *pool_tbl = gbl_args->pool_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        pool_tbl[i] = odp_event_pool(event_tbl[i]);
 
    return i;
}
 
static int event_pool_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    odp_pool_t *pool_tbl = gbl_args->pool_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_pool()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        pool_tbl[i] = odp_event_pool(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_user_area(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        ptr_tbl[i] = odp_event_user_area(event_tbl[i]);
 
    return i;
}
 
static int event_user_area_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_user_area()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ptr_tbl[i] = odp_event_user_area(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_user_area_and_flag(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int ret = 0;
    int flag;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++) {
        ptr_tbl[i] = odp_event_user_area_and_flag(event_tbl[i], &flag);
        ret += flag;
    }
 
    return ret;
}
 
static int event_user_area_and_flag_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    void **ptr_tbl = gbl_args->ptr_tbl;
    int ret = 0;
    int flag;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_user_area_and_flag()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ptr_tbl[i] = odp_event_user_area_and_flag(event_tbl[i], &flag);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
        ret += flag;
    }
 
    return ret;
}
 
static int event_is_valid(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
 
    uint32_t ret = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        ret += odp_event_is_valid(event_tbl[i]);
 
    return ret;
}
 
static int event_is_valid_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    uint32_t ret = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_is_valid()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ret += odp_event_is_valid(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return ret;
}
 
static int event_free(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
 
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_free(event_tbl[i]);
 
    return i;
}
 
static int event_free_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_free()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_free(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_free_multi(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_free_multi(&event_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int event_free_multi_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_free_multi()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_free_multi(&event_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_free_sp(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_free_sp(&event_tbl[i * burst_size], burst_size);
 
    return i;
}
 
static int event_free_sp_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int burst_size = gbl_args->appl.burst_size;
    int i;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_free_sp()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_free_sp(&event_tbl[i * burst_size], burst_size);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static int event_flow_id(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    uint32_t ret = 0;
 
    for (int i = 0; i < TEST_REPEAT_COUNT; i++)
        ret += odp_event_flow_id(event_tbl[i]);
 
    return !ret;
}
 
static int event_flow_id_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    uint32_t ret = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_flow_id()");
    bench_tm_stamp_t s1, s2;
 
    for (int i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        ret += odp_event_flow_id(event_tbl[i]);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return !ret;
}
 
static int event_flow_id_set(void)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i = 0;
 
    for (i = 0; i < TEST_REPEAT_COUNT; i++)
        odp_event_flow_id_set(event_tbl[i], 0);
 
    return i;
}
 
static int event_flow_id_set_tm(bench_tm_result_t *res, int repeat_count)
{
    odp_event_t *event_tbl = gbl_args->event_tbl;
    int i = 0;
    const uint8_t id1 = bench_tm_func_register(res, "odp_event_flow_id_set()");
    bench_tm_stamp_t s1, s2;
 
    for (i = 0; i < repeat_count; i++) {
        bench_tm_now(res, &s1);
        odp_event_flow_id_set(event_tbl[i], 0);
        bench_tm_now(res, &s2);
        bench_tm_func_record(&s2, &s1, res, id1);
    }
 
    return i;
}
 
static void usage(char *progname)
{
    printf("\n"
           "OpenDataPlane Buffer/Event API microbenchmarks.\n"
           "\n"
           "Usage: %s OPTIONS\n"
           "  E.g. %s\n"
           "\n"
           "Optional OPTIONS:\n"
           "  -b, --burst <num>       Test burst size.\n"
           "  -c, --cache_size <num>  Pool cache size.\n"
           "  -i, --index <idx>       Benchmark index to run indefinitely.\n"
           "  -r, --rounds <num>      Run each test case 'num' times (default %u).\n"
           "  -t, --time <opt>        Time measurement. 0: measure CPU cycles (default), 1: measure time\n"
           "  -m, --mode <mode>       Measurement mode. 0: measure throughput, track average execution\n"
           "                          time (default), 1: measure latency, track function minimum and\n"
           "                          maximum in addition to average execution time.\n"
           "  -h, --help              Display help and exit.\n\n"
           "\n", NO_PATH(progname), NO_PATH(progname), TEST_ROUNDS);
}
 
static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
{
    int opt;
    static const struct option longopts[] = {
        {"burst", required_argument, NULL, 'b'},
        {"cache_size", required_argument, NULL, 'c'},
        {"index", required_argument, NULL, 'i'},
        {"rounds", required_argument, NULL, 'r'},
        {"time", required_argument, NULL, 't'},
        {"mode", required_argument, NULL, 'm'},
        {"help", no_argument, NULL, 'h'},
        {NULL, 0, NULL, 0}
    };
 
    static const char *shortopts =  "c:b:i:r:t:m:h";
 
    appl_args->bench_idx = 0; /* Run all benchmarks */
    appl_args->burst_size = TEST_DEF_BURST;
    appl_args->cache_size = -1;
    appl_args->rounds = TEST_ROUNDS;
    appl_args->time = 0;
    appl_args->mode = M_TPUT;
 
    while (1) {
        opt = getopt_long(argc, argv, shortopts, longopts, NULL);
 
        if (opt == -1)
            break;  /* No more options */
 
        switch (opt) {
        case 'c':
            appl_args->cache_size = atoi(optarg);
            break;
        case 'b':
            appl_args->burst_size = atoi(optarg);
            break;
        case 'h':
            usage(argv[0]);
            exit(EXIT_SUCCESS);
            break;
        case 'i':
            appl_args->bench_idx = atoi(optarg);
            break;
        case 'r':
            appl_args->rounds = atoi(optarg);
            break;
        case 't':
            appl_args->time = atoi(optarg);
            break;
        case 'm':
            appl_args->mode = atoi(optarg);
            break;
        default:
            break;
        }
    }
 
    if (appl_args->burst_size < 1 ||
        appl_args->burst_size > TEST_MAX_BURST) {
        printf("Invalid burst size (max %d)\n", TEST_MAX_BURST);
        exit(EXIT_FAILURE);
    }
 
    if (appl_args->rounds < 1) {
        printf("Invalid number test rounds: %d\n", appl_args->rounds);
        exit(EXIT_FAILURE);
    }
 
    if (appl_args->mode != M_TPUT && appl_args->mode != M_LATENCY) {
        printf("Invalid measurement mode: %d\n", appl_args->mode);
        exit(EXIT_FAILURE);
    }
 
    optind = 1; /* Reset 'extern optind' from the getopt lib */
}
 
static void print_info(void)
{
    odp_sys_info_print();
 
    printf("\n"
           "odp_bench_buffer options\n"
           "------------------------\n");
 
    printf("Burst size:       %d\n", gbl_args->appl.burst_size);
    printf("Buffer size:      %d\n", gbl_args->buf_size);
    printf("CPU mask:         %s\n", gbl_args->cpumask_str);
    if (gbl_args->appl.cache_size < 0)
        printf("Pool cache size:  default\n");
    else
        printf("Pool cache size:  %d\n", gbl_args->appl.cache_size);
    printf("Measurement unit: %s\n", gbl_args->appl.time ? "nsec" : "CPU cycles");
    printf("Test rounds:      %u\n", gbl_args->appl.rounds);
    printf("Measurement mode: %s\n", gbl_args->appl.mode == M_TPUT ? "throughput" : "latency");
    printf("\n");
}
 
static int bench_buffer_tm_export(void *data)
{
    args_t *gbl_args = data;
    bench_tm_result_t *res;
    uint64_t num;
    int ret = 0;
    const char *unit = gbl_args->appl.time ? "nsec" : "cpu cycles";
 
    if (test_common_write("function name,min %s per function call,"
                  "average %s per function call,"
                  "max %s per function call\n", unit, unit, unit)) {
        ret = -1;
        goto exit;
    }
 
    for (uint32_t i = 0; i < gbl_args->suite.t.num_bench; i++) {
        res = &gbl_args->suite.t.result[i];
 
        for (int j = 0; j < res->num; j++) {
            num = res->func[j].num ? res->func[j].num : 1;
            if (test_common_write("%s,%" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n",
                          res->func[j].name,
                          bench_tm_to_u64(res, &res->func[j].min),
                          bench_tm_to_u64(res, &res->func[j].tot) / num,
                          bench_tm_to_u64(res, &res->func[j].max))) {
                ret = -1;
                goto exit;
            }
        }
    }
 
exit:
    test_common_write_term();
 
    return ret;
}
 
static int bench_buffer_export(void *data)
{
    args_t *gbl_args = data;
    int ret = 0;
 
    if (test_common_write("%s", gbl_args->appl.time ?
                  "function name,average nsec per function call\n" :
                  "function name,average cpu cycles per function call\n")) {
        ret = -1;
        goto exit;
    }
 
    for (int i = 0; i < gbl_args->suite.b.num_bench; i++) {
        if (test_common_write("odp_%s,%f\n", gbl_args->suite.b.bench[i].desc != NULL ?
                    gbl_args->suite.b.bench[i].desc :
                    gbl_args->suite.b.bench[i].name,
                      gbl_args->suite.b.result[i])) {
            ret = -1;
            goto exit;
        }
    }
 
exit:
    test_common_write_term();
 
    return ret;
}
 
bench_info_t test_suite[] = {
    BENCH_INFO(buffer_from_event, create_events, free_buffers, NULL),
    BENCH_INFO(buffer_from_event_multi, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO(buffer_to_event, create_buffers, free_buffers, NULL),
    BENCH_INFO(buffer_to_event_multi, create_buffers_multi, free_buffers_multi, NULL),
    BENCH_INFO(buffer_addr, create_buffers, free_buffers, NULL),
    BENCH_INFO(buffer_size, create_buffers, free_buffers, NULL),
    BENCH_INFO_COND(buffer_user_area, create_buffers, free_buffers, NULL, check_uarea),
    BENCH_INFO(buffer_pool, create_buffers, free_buffers, NULL),
    BENCH_INFO(buffer_alloc, NULL, free_buffers, NULL),
    BENCH_INFO(buffer_alloc_multi, NULL, free_buffers_multi, NULL),
    BENCH_INFO(buffer_free, create_buffers, NULL, NULL),
    BENCH_INFO(buffer_free_multi, create_buffers_multi, NULL, NULL),
    BENCH_INFO(buffer_alloc_free, NULL, NULL, NULL),
    BENCH_INFO(buffer_alloc_free_multi, NULL, NULL, NULL),
    BENCH_INFO(buffer_is_valid, create_buffers, free_buffers, NULL),
    BENCH_INFO(event_type, create_events, free_buffers, NULL),
    BENCH_INFO(event_subtype, create_events, free_buffers, NULL),
    BENCH_INFO(event_types, create_events, free_buffers, NULL),
    BENCH_INFO(event_types_multi, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO(event_types_multi_no_sub, create_events_multi, free_buffers_multi,
           "event_types_multi (no sub)"),
    BENCH_INFO(event_type_multi, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO(event_pool, create_events, free_buffers, NULL),
    BENCH_INFO_COND(event_user_area, create_events, free_buffers, NULL, check_uarea),
    BENCH_INFO_COND(event_user_area_and_flag, create_events, free_buffers, NULL, check_uarea),
    BENCH_INFO(event_is_valid, create_events, free_buffers, NULL),
    BENCH_INFO(event_free, create_events, NULL, NULL),
    BENCH_INFO(event_free_multi, create_events_multi, NULL, NULL),
    BENCH_INFO(event_free_sp, create_events_multi, NULL, NULL),
    BENCH_INFO_COND(event_flow_id, create_events, free_buffers, NULL, check_flow_aware),
    BENCH_INFO_COND(event_flow_id_set, create_events, free_buffers, NULL, check_flow_aware),
};
 
ODP_STATIC_ASSERT(ODPH_ARRAY_SIZE(test_suite) < TEST_MAX_BENCH,
          "Result array is too small to hold all the results");
 
bench_tm_info_t test_suite_tm[] = {
    BENCH_INFO_TM(buffer_from_event_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(buffer_from_event_multi_tm, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO_TM(buffer_to_event_tm, create_buffers, free_buffers, NULL),
    BENCH_INFO_TM(buffer_to_event_multi_tm, create_buffers_multi, free_buffers_multi, NULL),
    BENCH_INFO_TM(buffer_addr_tm, create_buffers, free_buffers, NULL),
    BENCH_INFO_TM(buffer_size_tm, create_buffers, free_buffers, NULL),
    BENCH_INFO_TM(buffer_user_area_tm, create_buffers, free_buffers, check_uarea),
    BENCH_INFO_TM(buffer_pool_tm, create_buffers, free_buffers, NULL),
    BENCH_INFO_TM(buffer_alloc_tm, NULL, free_buffers, NULL),
    BENCH_INFO_TM(buffer_alloc_multi_tm, NULL, free_buffers_multi, NULL),
    BENCH_INFO_TM(buffer_free_tm, create_buffers, NULL, NULL),
    BENCH_INFO_TM(buffer_free_multi_tm, create_buffers_multi, NULL, NULL),
    BENCH_INFO_TM(buffer_alloc_free_tm, NULL, NULL, NULL),
    BENCH_INFO_TM(buffer_alloc_free_multi_tm, NULL, NULL, NULL),
    BENCH_INFO_TM(buffer_is_valid_tm, create_buffers, free_buffers, NULL),
    BENCH_INFO_TM(event_type_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(event_subtype_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(event_types_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(event_types_multi_tm, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO_TM(event_types_multi_no_sub_tm, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO_TM(event_type_multi_tm, create_events_multi, free_buffers_multi, NULL),
    BENCH_INFO_TM(event_pool_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(event_user_area_tm, create_events, free_buffers, check_uarea),
    BENCH_INFO_TM(event_user_area_and_flag_tm, create_events, free_buffers, check_uarea),
    BENCH_INFO_TM(event_is_valid_tm, create_events, free_buffers, NULL),
    BENCH_INFO_TM(event_free_tm, create_events, NULL, NULL),
    BENCH_INFO_TM(event_free_multi_tm, create_events_multi, NULL, NULL),
    BENCH_INFO_TM(event_free_sp_tm, create_events_multi, NULL, NULL),
    BENCH_INFO_TM(event_flow_id_tm, create_events, free_buffers, check_flow_aware),
    BENCH_INFO_TM(event_flow_id_set_tm, create_events, free_buffers, check_flow_aware),
};
 
ODP_STATIC_ASSERT(ODPH_ARRAY_SIZE(test_suite_tm) < TEST_MAX_BENCH,
          "Result array is too small to hold all the results");
 
static void init_suite(args_t *gbl_args, odp_bool_t is_export)
{
    if (gbl_args->appl.mode == M_LATENCY) {
        bench_tm_suite_init(&gbl_args->suite.t);
        gbl_args->suite.t.bench = test_suite_tm;
        gbl_args->suite.t.num_bench = ODPH_ARRAY_SIZE(test_suite_tm);
        gbl_args->suite.t.rounds = TEST_REPEAT_COUNT;
        gbl_args->suite.t.bench_idx = gbl_args->appl.bench_idx;
        gbl_args->suite.t.measure_time = !!gbl_args->appl.time;
        gbl_args->suite.exit = &gbl_args->suite.t.exit_worker;
        gbl_args->suite.retval = &gbl_args->suite.t.retval;
        gbl_args->suite.args = &gbl_args->suite.t;
        gbl_args->suite.suite_fn = bench_tm_run;
        gbl_args->suite.export_fn = bench_buffer_tm_export;
 
        if (is_export)
            gbl_args->suite.t.result = gbl_args->result.t;
    } else {
        bench_suite_init(&gbl_args->suite.b);
        gbl_args->suite.b.bench = test_suite;
        gbl_args->suite.b.num_bench = ODPH_ARRAY_SIZE(test_suite);
        gbl_args->suite.b.indef_idx = gbl_args->appl.bench_idx;
        gbl_args->suite.b.rounds = gbl_args->appl.rounds;
        gbl_args->suite.b.repeat_count = TEST_REPEAT_COUNT;
        gbl_args->suite.b.measure_time = !!gbl_args->appl.time;
        gbl_args->suite.exit = &gbl_args->suite.b.exit_worker;
        gbl_args->suite.retval = &gbl_args->suite.b.retval;
        gbl_args->suite.args = &gbl_args->suite.b;
        gbl_args->suite.suite_fn = bench_run;
        gbl_args->suite.export_fn = bench_buffer_export;
 
        if (is_export)
            gbl_args->suite.b.result = gbl_args->result.b;
    }
}
 
int main(int argc, char *argv[])
{
    odph_helper_options_t helper_options;
    test_common_options_t common_options;
    odph_thread_t worker_thread;
    odph_thread_common_param_t thr_common;
    odph_thread_param_t thr_param;
    int cpu;
    odp_shm_t shm;
    odp_cpumask_t cpumask, default_mask;
    odp_schedule_capability_t sched_capa;
    odp_pool_capability_t capa;
    odp_pool_param_t params;
    odp_instance_t instance;
    odp_init_t init_param;
    uint32_t buf_num;
    uint8_t ret;
 
    /* Let helper collect its own arguments (e.g. --odph_proc) */
    argc = odph_parse_options(argc, argv);
    if (odph_options(&helper_options)) {
        ODPH_ERR("Error: reading ODP helper options failed\n");
        exit(EXIT_FAILURE);
    }
 
    argc = test_common_parse_options(argc, argv);
    if (test_common_options(&common_options)) {
        ODPH_ERR("Error: reading test options failed\n");
        exit(EXIT_FAILURE);
    }
 
    odp_init_param_init(&init_param);
    init_param.mem_model = helper_options.mem_model;
 
    /* Init ODP before calling anything else */
    if (odp_init_global(&instance, &init_param, NULL)) {
        ODPH_ERR("Error: ODP global init failed\n");
        exit(EXIT_FAILURE);
    }
 
    /* Init this thread */
    if (odp_init_local(instance, ODP_THREAD_CONTROL)) {
        ODPH_ERR("Error: ODP local init failed\n");
        exit(EXIT_FAILURE);
    }
 
    /* Reserve memory for args from shared mem */
    shm = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE, 0);
    if (shm == ODP_SHM_INVALID) {
        ODPH_ERR("Error: shared mem reserve failed\n");
        exit(EXIT_FAILURE);
    }
 
    gbl_args = odp_shm_addr(shm);
    if (gbl_args == NULL) {
        ODPH_ERR("Error: shared mem alloc failed\n");
        exit(EXIT_FAILURE);
    }
 
    memset(gbl_args, 0, sizeof(args_t));
 
    /* Parse and store the application arguments */
    parse_args(argc, argv, &gbl_args->appl);
    init_suite(gbl_args, common_options.is_export);
 
    /* Get default worker cpumask */
    if (odp_cpumask_default_worker(&default_mask, 1) != 1) {
        ODPH_ERR("Error: unable to allocate worker thread\n");
        exit(EXIT_FAILURE);
    }
    (void)odp_cpumask_to_str(&default_mask, gbl_args->cpumask_str,
                 sizeof(gbl_args->cpumask_str));
 
    if (odp_schedule_capability(&sched_capa)) {
        ODPH_ERR("Error: schedule capability failed\n");
        exit(EXIT_FAILURE);
    }
 
    gbl_args->max_flow_id = 0;
    if (sched_capa.max_flow_id) {
        odp_schedule_config_t sched_config;
 
        odp_schedule_config_init(&sched_config);
        sched_config.max_flow_id = 1;
 
        if (odp_schedule_config(&sched_config)) {
            ODPH_ERR("Error: schedule config failed\n");
            exit(EXIT_FAILURE);
        }
        gbl_args->max_flow_id = 1;
    }
 
    if (odp_pool_capability(&capa)) {
        ODPH_ERR("Error: unable to query pool capability\n");
        exit(EXIT_FAILURE);
    }
 
    buf_num = gbl_args->appl.burst_size * TEST_REPEAT_COUNT;
 
    if (capa.buf.max_num && capa.buf.max_num < buf_num) {
        ODPH_ERR("Error: pool size not supported (max %" PRIu32 ")\n", capa.buf.max_num);
        exit(EXIT_FAILURE);
    } else if (gbl_args->appl.cache_size > (int)capa.buf.max_cache_size) {
        ODPH_ERR("Error: cache size not supported (max %" PRIu32 ")\n",
             capa.buf.max_cache_size);
        exit(EXIT_FAILURE);
    }
 
    gbl_args->buf_size = TEST_BUF_SIZE;
    if (capa.buf.max_size && capa.buf.max_size < TEST_BUF_SIZE)
        gbl_args->buf_size = capa.buf.max_size;
 
    gbl_args->uarea_size = TEST_UAREA_SIZE < capa.buf.max_uarea_size ?
                TEST_UAREA_SIZE : capa.buf.max_uarea_size;
 
    print_info();
 
    /* Create buffer pool */
    odp_pool_param_init(&params);
    params.buf.size = gbl_args->buf_size;
    params.buf.num = buf_num;
    params.buf.uarea_size = gbl_args->uarea_size;
    if (gbl_args->appl.cache_size >= 0)
        params.buf.cache_size = gbl_args->appl.cache_size;
    params.type = ODP_POOL_BUFFER;
 
    gbl_args->pool = odp_pool_create("microbench", &params);
    if (gbl_args->pool == ODP_POOL_INVALID) {
        ODPH_ERR("Error: pool create failed\n");
        exit(EXIT_FAILURE);
    }
 
    odp_pool_print(gbl_args->pool);
 
    memset(&worker_thread, 0, sizeof(odph_thread_t));
 
    signal(SIGINT, sig_handler);
 
    /* Create worker thread */
    cpu = odp_cpumask_first(&default_mask);
 
    odp_cpumask_zero(&cpumask);
    odp_cpumask_set(&cpumask, cpu);
 
    odph_thread_common_param_init(&thr_common);
    thr_common.instance = instance;
    thr_common.cpumask = &cpumask;
    thr_common.share_param = 1;
 
    odph_thread_param_init(&thr_param);
    thr_param.start = gbl_args->suite.suite_fn;
    thr_param.arg = gbl_args->suite.args;
    thr_param.thr_type = ODP_THREAD_WORKER;
 
    odph_thread_create(&worker_thread, &thr_common, &thr_param, 1);
 
    odph_thread_join(&worker_thread, 1);
 
    ret = *gbl_args->suite.retval;
 
    if (ret == 0 && common_options.is_export) {
        if (gbl_args->suite.export_fn(gbl_args)) {
            ODPH_ERR("Error: Export failed\n");
            ret = -1;
        }
    }
 
    if (odp_pool_destroy(gbl_args->pool)) {
        ODPH_ERR("Error: pool destroy\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_shm_free(shm)) {
        ODPH_ERR("Error: shm free\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_term_local()) {
        ODPH_ERR("Error: term local\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_term_global(instance)) {
        ODPH_ERR("Error: term global\n");
        exit(EXIT_FAILURE);
    }
 
    return ret;
}