odp_dma_perf.c

This tester application can be used to profile the performance of an ODP DMA implementation.

This tester application can be used to profile the performance of an ODP DMA implementation.Tester workflow is simple and consists of issuing as many back-to-back DMA transfers as the implementation allows and then recording key performance statistics (such as function overhead, latencies etc.).

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2021-2024 Nokia
 */
 
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
 
#include <inttypes.h>
#include <stdlib.h>
#include <signal.h>
#include <stdint.h>
#include <unistd.h>
 
#include <odp_api.h>
#include <odp/helper/odph_api.h>
 
#include <export_results.h>
 
#define EXIT_NOT_SUP 2
#define PROG_NAME "odp_dma_perf"
 
enum {
    SYNC_DMA = 0U,
    ASYNC_DMA,
    SW_COPY
};
 
enum {
    DENSE_PACKET = 0U,
    SPARSE_PACKET,
    DENSE_MEMORY,
    SPARSE_MEMORY
};
 
enum {
    POLL = 0U,
    EVENT
};
 
enum {
    SINGLE = 0U,
    MANY
};
 
#define DEF_TRS_TYPE SYNC_DMA
#define DEF_SEG_CNT 1U
#define DEF_LEN 1024U
#define DEF_SEG_TYPE DENSE_PACKET
#define DEF_MODE POLL
#define DEF_INFLIGHT 1U
#define DEF_TIME 10U
#define DEF_WORKERS 1U
#define DEF_POLICY SINGLE
 
#define MAX_SEGS 1024U
#define MAX_WORKERS 32
#define MAX_MEMORY (256U * 1024U * 1024U)
 
#define GIGAS 1000000000
#define MEGAS 1000000
#define KILOS 1000
 
#define DATA 0xAA
 
typedef enum {
    PRS_OK,
    PRS_NOK,
    PRS_TERM,
    PRS_NOT_SUP
} parse_result_t;
 
typedef struct {
    uint64_t completed;
    uint64_t start_errs;
    uint64_t poll_errs;
    uint64_t scheduler_timeouts;
    uint64_t transfer_errs;
    uint64_t data_errs;
    uint64_t tot_tm;
    uint64_t trs_tm;
    uint64_t max_trs_tm;
    uint64_t min_trs_tm;
    uint64_t start_cc;
    uint64_t max_start_cc;
    uint64_t min_start_cc;
    uint64_t wait_cc;
    uint64_t max_wait_cc;
    uint64_t min_wait_cc;
    uint64_t trs_cc;
    uint64_t max_trs_cc;
    uint64_t min_trs_cc;
    uint64_t start_cnt;
    uint64_t wait_cnt;
    uint64_t trs_poll_cnt;
    uint64_t trs_cnt;
} stats_t;
 
typedef struct {
    odp_dma_transfer_param_t trs_param;
    odp_dma_compl_param_t compl_param;
    odp_ticketlock_t lock;
    odp_time_t trs_start_tm;
    uint64_t trs_start_cc;
    uint64_t trs_poll_cnt;
    odp_bool_t is_running;
} trs_info_t;
 
typedef struct sd_s sd_t;
typedef void (*ver_fn_t)(trs_info_t *info, stats_t *stats);
 
typedef struct ODP_ALIGNED_CACHE sd_s {
    struct {
        trs_info_t infos[MAX_SEGS];
        odp_dma_seg_t src_seg[MAX_SEGS];
        odp_dma_seg_t dst_seg[MAX_SEGS];
        odp_dma_t handle;
        odp_pool_t pool;
        odp_queue_t compl_q;
        uint32_t num_in_segs;
        uint32_t num_out_segs;
        uint32_t src_seg_len;
        uint32_t dst_seg_len;
        uint32_t num_inflight;
        uint8_t trs_type;
        uint8_t compl_mode;
    } dma;
 
    struct {
        odp_packet_t src_pkt[MAX_SEGS];
        odp_packet_t dst_pkt[MAX_SEGS];
        odp_pool_t src_pool;
        odp_pool_t dst_pool;
        odp_shm_t src_shm;
        odp_shm_t dst_shm;
        void *src;
        void *dst;
        void *src_high;
        void *dst_high;
        void *cur_src;
        void *cur_dst;
        uint64_t shm_size;
        uint8_t seg_type;
    } seg;
 
    odp_schedule_group_t grp;
    /* Prepare single transfer. */
    void (*prep_trs_fn)(sd_t *sd, trs_info_t *info);
    /* Verify single transfer. */
    ver_fn_t ver_fn;
} sd_t;
 
typedef struct prog_config_s prog_config_t;
 
typedef struct ODP_ALIGNED_CACHE {
    stats_t stats;
    prog_config_t *prog_config;
    sd_t *sd;
} thread_config_t;
 
typedef struct {
    /* Configure DMA session specific resources. */
    odp_bool_t (*session_cfg_fn)(sd_t *sd);
    /* Setup transfer elements (memory/packet segments). */
    odp_bool_t (*setup_fn)(sd_t *sd);
    /* Configure DMA transfers (segment addresses etc.). */
    void (*trs_fn)(sd_t *sd);
    /* Configure transfer completion resources (transfer IDs, events etc.). */
    odp_bool_t (*compl_fn)(sd_t *sd);
    /* Initiate required initial transfers. */
    odp_bool_t (*bootstrap_fn)(sd_t *sd);
    /* Wait and handle finished transfer. */
    void (*wait_fn)(sd_t *sd, stats_t *stats);
    /* Handle all unfinished transfers after main test has been stopped. */
    void (*drain_fn)(sd_t *sd);
    /* Free any resources that might have been allocated during setup phase. */
    void (*free_fn)(const sd_t *sd);
} test_api_t;
 
typedef struct prog_config_s {
    odph_thread_t threads[MAX_WORKERS];
    thread_config_t thread_config[MAX_WORKERS];
    sd_t sds[MAX_WORKERS];
    test_api_t api;
    odp_atomic_u32_t is_running;
    odp_instance_t odp_instance;
    odp_barrier_t init_barrier;
    odp_barrier_t term_barrier;
    odp_dma_compl_mode_t compl_mode_mask;
    odp_pool_t src_pool;
    odp_pool_t dst_pool;
    uint64_t shm_size;
    uint32_t num_in_segs;
    uint32_t num_out_segs;
    uint32_t src_seg_len;
    uint32_t dst_seg_len;
    uint32_t num_inflight;
    double time_sec;
    uint32_t num_sessions;
    uint32_t src_cache_size;
    uint32_t dst_cache_size;
    int num_workers;
    odp_bool_t is_verify;
    uint8_t trs_type;
    uint8_t seg_type;
    uint8_t compl_mode;
    uint8_t policy;
    test_common_options_t common_options;
} prog_config_t;
 
static prog_config_t *prog_conf;
 
static const int mode_map[] = { ODP_DMA_COMPL_POLL, ODP_DMA_COMPL_EVENT };
 
static void terminate(int signal ODP_UNUSED)
{
    odp_atomic_store_u32(&prog_conf->is_running, 0U);
}
 
static void init_config(prog_config_t *config)
{
    sd_t *sd;
    trs_info_t *info;
    stats_t *stats;
 
    memset(config, 0, sizeof(*config));
    config->compl_mode_mask |= ODP_DMA_COMPL_SYNC;
    config->src_pool = ODP_POOL_INVALID;
    config->dst_pool = ODP_POOL_INVALID;
    config->num_in_segs = DEF_SEG_CNT;
    config->num_out_segs = DEF_SEG_CNT;
    config->src_seg_len = DEF_LEN;
    config->num_inflight = DEF_INFLIGHT;
    config->time_sec = DEF_TIME;
    config->num_workers = DEF_WORKERS;
    config->trs_type = DEF_TRS_TYPE;
    config->seg_type = DEF_SEG_TYPE;
    config->compl_mode = DEF_MODE;
    config->policy = DEF_POLICY;
 
    for (uint32_t i = 0U; i < MAX_WORKERS; ++i) {
        sd = &config->sds[i];
        stats = &config->thread_config[i].stats;
        memset(sd, 0, sizeof(*sd));
 
        for (uint32_t j = 0U; j < MAX_SEGS; ++j) {
            info = &sd->dma.infos[j];
            info->compl_param.transfer_id = ODP_DMA_TRANSFER_ID_INVALID;
            info->compl_param.event = ODP_EVENT_INVALID;
            info->compl_param.queue = ODP_QUEUE_INVALID;
            odp_ticketlock_init(&info->lock);
            sd->seg.src_pkt[j] = ODP_PACKET_INVALID;
            sd->seg.dst_pkt[j] = ODP_PACKET_INVALID;
        }
 
        sd->dma.handle = ODP_DMA_INVALID;
        sd->dma.pool = ODP_POOL_INVALID;
        sd->dma.compl_q = ODP_QUEUE_INVALID;
        sd->seg.src_shm = ODP_SHM_INVALID;
        sd->seg.dst_shm = ODP_SHM_INVALID;
        sd->grp = ODP_SCHED_GROUP_INVALID;
        stats->min_trs_tm = UINT64_MAX;
        stats->min_start_cc = UINT64_MAX;
        stats->min_wait_cc = UINT64_MAX;
        stats->min_trs_cc = UINT64_MAX;
    }
}
 
static void print_usage(void)
{
    printf("\n"
           "DMA performance test. Load DMA subsystem from several workers.\n"
           "\n"
           "Usage: " PROG_NAME " [OPTIONS]\n"
           "\n"
           "  E.g. " PROG_NAME "\n"
           "       " PROG_NAME " -s 10240\n"
           "       " PROG_NAME " -t 0 -i 1 -o 1 -s 51200 -S 2 -f 64 -T 10\n"
           "       " PROG_NAME " -t 1 -i 10 -o 10 -s 4096 -S 0 -m 1 -f 10 -c 4 -p 1\n"
           "       " PROG_NAME " -t 2 -i 10 -o 1 -s 1024 -S 3 -f 10 -c 4 -p 1\n"
           "\n"
           "Optional OPTIONS:\n"
           "\n"
           "  -t, --trs_type      Transfer type for test data. %u by default.\n"
           "                      Types:\n"
           "                          0: synchronous DMA\n"
           "                          1: asynchronous DMA\n"
           "                          2: SW memory copy\n"
           "  -i, --num_in_seg    Number of input segments to transfer. 0 means the maximum\n"
           "                      count supported by the implementation. %u by default.\n"
           "  -o, --num_out_seg   Number of output segments to transfer to. 0 means the\n"
           "                      maximum count supported by the implementation. %u by\n"
           "                      default.\n"
           "  -s, --in_seg_len    Input segment length in bytes. 0 length means the maximum\n"
           "                      segment length supported by the implementation. The actual\n"
           "                      maximum might be limited by what type of data is\n"
           "                      transferred (packet/memory). %u by default.\n"
           "  -S, --in_seg_type   Input segment data type. Dense types can load the DMA\n"
           "                      subsystem more heavily as transfer resources are\n"
           "                      pre-configured. Sparse types might on the other hand\n"
           "                      reflect application usage more precisely as transfer\n"
           "                      resources are configured in runtime. %u by default.\n"
           "                      Types:\n"
           "                          0: dense packet\n"
           "                          1: sparse packet\n"
           "                          2: dense memory\n"
           "                          3: sparse memory\n"
           "  -m, --compl_mode    Completion mode for transfers. %u by default.\n"
           "                      Modes:\n"
           "                          0: poll\n"
           "                          1: event\n"
           "  -f, --max_in_flight Maximum transfers in-flight per session. 0 means the\n"
           "                      maximum supported by the tester/implementation. %u by\n"
           "                      default.\n"
           "  -T, --time_sec      Time in seconds to run. 0 means infinite. %u by default.\n"
           "  -c, --worker_count  Amount of workers. %u by default.\n"
           "  -p, --policy        DMA session policy. %u by default.\n"
           "                      Policies:\n"
           "                          0: One session shared by workers\n"
           "                          1: One session per worker\n"
           "  -v, --verify        Verify transfers. Checks correctness of destination data\n"
           "                      after successful transfers.\n"
           "  -h, --help          This help.\n"
           "\n", DEF_TRS_TYPE, DEF_SEG_CNT, DEF_SEG_CNT, DEF_LEN, DEF_SEG_TYPE, DEF_MODE,
           DEF_INFLIGHT, DEF_TIME, DEF_WORKERS, DEF_POLICY);
}
 
static parse_result_t check_options(prog_config_t *config)
{
    int max_workers;
    odp_dma_capability_t dma_capa;
    uint32_t num_sessions, max_seg_len, max_trs, max_in, max_out, max_segs;
    odp_schedule_capability_t sched_capa;
    odp_pool_capability_t pool_capa;
    odp_shm_capability_t shm_capa;
    uint64_t shm_size = 0U;
 
    if (config->trs_type != SYNC_DMA && config->trs_type != ASYNC_DMA &&
        config->trs_type != SW_COPY) {
        ODPH_ERR("Invalid transfer type: %u\n", config->trs_type);
        return PRS_NOK;
    }
 
    if (config->seg_type != DENSE_PACKET && config->seg_type != SPARSE_PACKET &&
        config->seg_type != DENSE_MEMORY && config->seg_type != SPARSE_MEMORY) {
        ODPH_ERR("Invalid segment type: %u\n", config->seg_type);
        return PRS_NOK;
    }
 
    max_workers = ODPH_MIN(odp_thread_count_max() - 1, MAX_WORKERS);
 
    if (config->num_workers <= 0 || config->num_workers > max_workers) {
        ODPH_ERR("Invalid thread count: %d (min: 1, max: %d)\n", config->num_workers,
             max_workers);
        return PRS_NOK;
    }
 
    if (config->policy != SINGLE && config->policy != MANY) {
        ODPH_ERR("Invalid DMA session policy: %u\n", config->policy);
        return PRS_NOK;
    }
 
    if (odp_dma_capability(&dma_capa) < 0) {
        ODPH_ERR("Error querying DMA capabilities\n");
        return PRS_NOK;
    }
 
    num_sessions = config->policy == SINGLE ? 1 : config->num_workers;
 
    if (num_sessions > dma_capa.max_sessions) {
        ODPH_ERR("Not enough DMA sessions supported: %u (max: %u)\n", num_sessions,
             dma_capa.max_sessions);
        return PRS_NOT_SUP;
    }
 
    config->num_sessions = num_sessions;
 
    if (config->num_in_segs == 0U)
        config->num_in_segs = dma_capa.max_src_segs;
 
    if (config->num_out_segs == 0U)
        config->num_out_segs = dma_capa.max_dst_segs;
 
    if (config->num_in_segs > dma_capa.max_src_segs ||
        config->num_out_segs > dma_capa.max_dst_segs ||
        config->num_in_segs + config->num_out_segs > dma_capa.max_segs) {
        ODPH_ERR("Unsupported segment count configuration, in: %u, out: %u (max in: %u, "
             "max out: %u, max tot: %u)\n", config->num_in_segs, config->num_out_segs,
             dma_capa.max_src_segs, dma_capa.max_dst_segs, dma_capa.max_segs);
        return PRS_NOT_SUP;
    }
 
    if (config->src_seg_len == 0U)
        config->src_seg_len = dma_capa.max_seg_len;
 
    config->dst_seg_len = config->src_seg_len * config->num_in_segs /
                  config->num_out_segs + config->src_seg_len *
                  config->num_in_segs % config->num_out_segs;
 
    max_seg_len = ODPH_MAX(config->src_seg_len, config->dst_seg_len);
 
    if (max_seg_len > dma_capa.max_seg_len) {
        ODPH_ERR("Unsupported total DMA segment length: %u (max: %u)\n", max_seg_len,
             dma_capa.max_seg_len);
        return PRS_NOT_SUP;
    }
 
    if (config->trs_type == ASYNC_DMA) {
        if (config->compl_mode != POLL && config->compl_mode != EVENT) {
            ODPH_ERR("Invalid completion mode: %u\n", config->compl_mode);
            return PRS_NOK;
        }
 
        if (config->compl_mode == POLL && (dma_capa.compl_mode_mask & ODP_DMA_COMPL_POLL)
            == 0U) {
            ODPH_ERR("Unsupported DMA completion mode, poll\n");
            return PRS_NOT_SUP;
        }
 
        if (config->compl_mode == EVENT) {
            if (config->num_sessions > dma_capa.pool.max_pools) {
                ODPH_ERR("Unsupported amount of completion pools: %u (max: %u)\n",
                     config->num_sessions, dma_capa.pool.max_pools);
                return PRS_NOT_SUP;
            }
 
            if ((dma_capa.compl_mode_mask & ODP_DMA_COMPL_EVENT) == 0U) {
                ODPH_ERR("Unsupported DMA completion mode, event\n");
                return PRS_NOT_SUP;
            }
 
            if (dma_capa.queue_type_sched == 0) {
                ODPH_ERR("Unsupported DMA queueing type, scheduled\n");
                return PRS_NOT_SUP;
            }
 
            if (config->num_inflight > dma_capa.pool.max_num) {
                ODPH_ERR("Unsupported amount of completion events: %u (max: %u)\n",
                     config->num_inflight, dma_capa.pool.max_num);
                return PRS_NOT_SUP;
            }
 
            if (odp_schedule_capability(&sched_capa) < 0) {
                ODPH_ERR("Error querying scheduler capabilities\n");
                return PRS_NOK;
            }
 
            if (config->num_sessions > sched_capa.max_groups - 3U) {
                ODPH_ERR("Unsupported amount of scheduler groups: %u (max: %u)\n",
                     config->num_sessions, sched_capa.max_groups - 3U);
                return PRS_NOT_SUP;
            }
        }
 
        config->compl_mode_mask |= mode_map[config->compl_mode];
    }
 
    max_trs = ODPH_MIN(dma_capa.max_transfers, MAX_SEGS);
 
    if (config->num_inflight == 0U)
        config->num_inflight = max_trs;
 
    if (config->num_inflight > max_trs) {
        ODPH_ERR("Unsupported amount of in-flight DMA transfers: %u (max: %u)\n",
             config->num_inflight, max_trs);
        return PRS_NOT_SUP;
    }
 
    max_in = config->num_in_segs * config->num_inflight;
    max_out = config->num_out_segs * config->num_inflight;
    max_segs = ODPH_MAX(max_in, max_out);
 
    if (max_segs > MAX_SEGS) {
        ODPH_ERR("Unsupported input/output * inflight segment combination: %u (max: %u)\n",
             max_segs, MAX_SEGS);
        return PRS_NOT_SUP;
    }
 
    if (config->seg_type == DENSE_PACKET || config->seg_type == SPARSE_PACKET) {
        if (odp_pool_capability(&pool_capa) < 0) {
            ODPH_ERR("Error querying pool capabilities\n");
            return PRS_NOK;
        }
 
        if (pool_capa.pkt.max_pools < 2U) {
            ODPH_ERR("Unsupported amount of packet pools: 2 (max: %u)\n",
                 pool_capa.pkt.max_pools);
            return PRS_NOT_SUP;
        }
 
        if (pool_capa.pkt.max_len != 0U && max_seg_len > pool_capa.pkt.max_len) {
            ODPH_ERR("Unsupported packet size: %u (max: %u)\n", max_seg_len,
                 pool_capa.pkt.max_len);
            return PRS_NOT_SUP;
        }
 
        if (pool_capa.pkt.max_num != 0U &&
            max_segs * num_sessions > pool_capa.pkt.max_num) {
            ODPH_ERR("Unsupported amount of packet pool elements: %u (max: %u)\n",
                 max_segs * num_sessions, pool_capa.pkt.max_num);
            return PRS_NOT_SUP;
        }
 
        config->src_cache_size = ODPH_MIN(ODPH_MAX(max_in, pool_capa.pkt.min_cache_size),
                          pool_capa.pkt.max_cache_size);
        config->dst_cache_size = ODPH_MIN(ODPH_MAX(max_out, pool_capa.pkt.min_cache_size),
                          pool_capa.pkt.max_cache_size);
    } else {
        /* If SHM implementation capabilities are very puny, program will have already
         * failed when reserving memory for global program configuration. */
        if (odp_shm_capability(&shm_capa) < 0) {
            ODPH_ERR("Error querying SHM capabilities\n");
            return PRS_NOK;
        }
 
        /* One block for program configuration, one for source memory and one for
         * destination memory. */
        if (shm_capa.max_blocks < 3U) {
            ODPH_ERR("Unsupported amount of SHM blocks: 3 (max: %u)\n",
                 shm_capa.max_blocks);
            return PRS_NOT_SUP;
        }
 
        shm_size = (uint64_t)config->dst_seg_len * config->num_out_segs *
               config->num_inflight;
 
        if (shm_capa.max_size != 0U && shm_size > shm_capa.max_size) {
            ODPH_ERR("Unsupported total SHM block size: %" PRIu64 ""
                 " (max: %" PRIu64 ")\n", shm_size, shm_capa.max_size);
            return PRS_NOT_SUP;
        }
 
        if (config->seg_type == SPARSE_MEMORY && shm_size < MAX_MEMORY)
            shm_size = shm_capa.max_size != 0U ?
                    ODPH_MIN(shm_capa.max_size, MAX_MEMORY) : MAX_MEMORY;
 
        config->shm_size = shm_size;
    }
 
    return PRS_OK;
}
 
static parse_result_t parse_options(int argc, char **argv, prog_config_t *config)
{
    int opt;
    static const struct option longopts[] = {
        { "trs_type", required_argument, NULL, 't' },
        { "num_in_seg", required_argument, NULL, 'i' },
        { "num_out_seg", required_argument, NULL, 'o' },
        { "in_seg_len", required_argument, NULL, 's' },
        { "in_seg_type", required_argument, NULL, 'S' },
        { "compl_mode", required_argument, NULL, 'm' },
        { "max_in_flight", required_argument, NULL, 'f'},
        { "time_sec", required_argument, NULL, 'T' },
        { "worker_count", required_argument, NULL, 'c' },
        { "policy", required_argument, NULL, 'p' },
        { "verify", no_argument, NULL, 'v' },
        { "help", no_argument, NULL, 'h' },
        { NULL, 0, NULL, 0 }
    };
    static const char *shortopts = "t:i:o:s:S:m:f:T:c:p:vh";
 
    init_config(config);
 
    while (1) {
        opt = getopt_long(argc, argv, shortopts, longopts, NULL);
 
        if (opt == -1)
            break;
 
        switch (opt) {
        case 't':
            config->trs_type = atoi(optarg);
            break;
        case 'i':
            config->num_in_segs = atoi(optarg);
            break;
        case 'o':
            config->num_out_segs = atoi(optarg);
            break;
        case 's':
            config->src_seg_len = atoi(optarg);
            break;
        case 'S':
            config->seg_type = atoi(optarg);
            break;
        case 'm':
            config->compl_mode = atoi(optarg);
            break;
        case 'f':
            config->num_inflight = atoi(optarg);
            break;
        case 'T':
            config->time_sec = atof(optarg);
            break;
        case 'c':
            config->num_workers = atoi(optarg);
            break;
        case 'p':
            config->policy = atoi(optarg);
            break;
        case 'v':
            config->is_verify = true;
            break;
        case 'h':
            print_usage();
            return PRS_TERM;
        case '?':
        default:
            print_usage();
            return PRS_NOK;
        }
    }
 
    return check_options(config);
}
 
static parse_result_t setup_program(int argc, char **argv, prog_config_t *config)
{
    struct sigaction action = { .sa_handler = terminate };
 
    if (sigemptyset(&action.sa_mask) == -1 || sigaddset(&action.sa_mask, SIGINT) == -1 ||
        sigaddset(&action.sa_mask, SIGTERM) == -1 ||
        sigaddset(&action.sa_mask, SIGHUP) == -1 || sigaction(SIGINT, &action, NULL) == -1 ||
        sigaction(SIGTERM, &action, NULL) == -1 || sigaction(SIGHUP, &action, NULL) == -1) {
        ODPH_ERR("Error installing signal handler\n");
        return PRS_NOK;
    }
 
    return parse_options(argc, argv, config);
}
 
static odp_pool_t get_src_packet_pool(void)
{
    odp_pool_param_t param;
    uint32_t num_pkts_per_worker = ODPH_MAX(prog_conf->num_inflight * prog_conf->num_in_segs,
                        prog_conf->src_cache_size);
 
    if (prog_conf->src_pool != ODP_POOL_INVALID)
        return prog_conf->src_pool;
 
    odp_pool_param_init(&param);
    param.type = ODP_POOL_PACKET;
    param.pkt.num = num_pkts_per_worker * prog_conf->num_workers;
    param.pkt.len = prog_conf->src_seg_len;
    param.pkt.seg_len = prog_conf->src_seg_len;
    param.pkt.cache_size = prog_conf->src_cache_size;
    prog_conf->src_pool = odp_pool_create(PROG_NAME "_src_pkts", &param);
 
    return prog_conf->src_pool;
}
 
static odp_pool_t get_dst_packet_pool(void)
{
    odp_pool_param_t param;
    uint32_t num_pkts_per_worker = ODPH_MAX(prog_conf->num_inflight * prog_conf->num_out_segs,
                        prog_conf->dst_cache_size);
 
    if (prog_conf->dst_pool != ODP_POOL_INVALID)
        return prog_conf->dst_pool;
 
    odp_pool_param_init(&param);
    param.type = ODP_POOL_PACKET;
    param.pkt.num = num_pkts_per_worker * prog_conf->num_workers;
    param.pkt.len = prog_conf->dst_seg_len;
    param.pkt.seg_len = prog_conf->dst_seg_len;
    param.pkt.cache_size = prog_conf->dst_cache_size;
    prog_conf->dst_pool = odp_pool_create(PROG_NAME "_dst_pkts", &param);
 
    return prog_conf->dst_pool;
}
 
static odp_bool_t configure_packets(sd_t *sd)
{
    sd->seg.src_pool = get_src_packet_pool();
 
    if (sd->seg.src_pool == ODP_POOL_INVALID) {
        ODPH_ERR("Error creating source packet pool\n");
        return false;
    }
 
    sd->seg.dst_pool = get_dst_packet_pool();
 
    if (sd->seg.dst_pool == ODP_POOL_INVALID) {
        ODPH_ERR("Error creating destination packet pool\n");
        return false;
    }
 
    return true;
}
 
static odp_bool_t allocate_packets(sd_t *sd)
{
    for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) {
        sd->seg.src_pkt[i] = odp_packet_alloc(sd->seg.src_pool, sd->dma.src_seg_len);
 
        if (sd->seg.src_pkt[i] == ODP_PACKET_INVALID) {
            ODPH_ERR("Error allocating source segment packets\n");
            return false;
        }
    }
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) {
        sd->seg.dst_pkt[i] = odp_packet_alloc(sd->seg.dst_pool, sd->dma.dst_seg_len);
 
        if (sd->seg.dst_pkt[i] == ODP_PACKET_INVALID) {
            ODPH_ERR("Error allocating destination segment packets\n");
            return false;
        }
    }
 
    return true;
}
 
static odp_bool_t setup_packet_segments(sd_t *sd)
{
    return configure_packets(sd) &&
           (sd->seg.seg_type == DENSE_PACKET ? allocate_packets(sd) : true);
}
 
static inline void fill_data(uint8_t *data, uint32_t len)
{
    memset(data, DATA, len);
}
 
static void configure_packet_transfer(sd_t *sd)
{
    odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg;
    uint32_t k = 0U, z = 0U, len;
    odp_packet_t pkt;
    odp_dma_transfer_param_t *param;
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
        start_src_seg = &sd->dma.src_seg[k];
        start_dst_seg = &sd->dma.dst_seg[z];
 
        for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) {
            pkt = sd->seg.src_pkt[k];
            seg = &start_src_seg[j];
            seg->packet = pkt;
            seg->offset = 0U;
            seg->len = sd->dma.src_seg_len;
 
            if (seg->packet != ODP_PACKET_INVALID)
                fill_data(odp_packet_data(seg->packet), seg->len);
        }
 
        len = sd->dma.num_in_segs * sd->dma.src_seg_len;
 
        for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) {
            pkt = sd->seg.dst_pkt[z];
            seg = &start_dst_seg[j];
            seg->packet = pkt;
            seg->offset = 0U;
            seg->len = ODPH_MIN(len, sd->dma.dst_seg_len);
            len -= sd->dma.dst_seg_len;
        }
 
        param = &sd->dma.infos[i].trs_param;
        odp_dma_transfer_param_init(param);
        param->src_format = ODP_DMA_FORMAT_PACKET;
        param->dst_format = ODP_DMA_FORMAT_PACKET;
        param->num_src = sd->dma.num_in_segs;
        param->num_dst = sd->dma.num_out_segs;
        param->src_seg = start_src_seg;
        param->dst_seg = start_dst_seg;
    }
}
 
static void free_packets(const sd_t *sd)
{
    for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_in_segs; ++i) {
        if (sd->seg.src_pkt[i] != ODP_PACKET_INVALID)
            odp_packet_free(sd->seg.src_pkt[i]);
    }
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight * sd->dma.num_out_segs; ++i) {
        if (sd->seg.dst_pkt[i] != ODP_PACKET_INVALID)
            odp_packet_free(sd->seg.dst_pkt[i]);
    }
}
 
static odp_bool_t allocate_memory(sd_t *sd)
{
    sd->seg.src_shm = odp_shm_reserve(PROG_NAME "_src_shm", sd->seg.shm_size,
                      ODP_CACHE_LINE_SIZE, 0U);
    sd->seg.dst_shm = odp_shm_reserve(PROG_NAME "_dst_shm", sd->seg.shm_size,
                      ODP_CACHE_LINE_SIZE, 0U);
 
    if (sd->seg.src_shm == ODP_SHM_INVALID || sd->seg.dst_shm == ODP_SHM_INVALID) {
        ODPH_ERR("Error allocating SHM block\n");
        return false;
    }
 
    sd->seg.src = odp_shm_addr(sd->seg.src_shm);
    sd->seg.dst = odp_shm_addr(sd->seg.dst_shm);
 
    if (sd->seg.src == NULL || sd->seg.dst == NULL) {
        ODPH_ERR("Error resolving SHM block address\n");
        return false;
    }
 
    sd->seg.src_high = (uint8_t *)sd->seg.src + sd->seg.shm_size - sd->dma.src_seg_len;
    sd->seg.dst_high = (uint8_t *)sd->seg.dst + sd->seg.shm_size - sd->dma.dst_seg_len;
    sd->seg.cur_src = sd->seg.src;
    sd->seg.cur_dst = sd->seg.dst;
 
    return true;
}
 
static odp_bool_t setup_memory_segments(sd_t *sd)
{
    return allocate_memory(sd);
}
 
static void configure_address_transfer(sd_t *sd)
{
    odp_dma_seg_t *start_src_seg, *start_dst_seg, *seg;
    uint32_t k = 0U, z = 0U, len;
    odp_dma_transfer_param_t *param;
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
        start_src_seg = &sd->dma.src_seg[k];
        start_dst_seg = &sd->dma.dst_seg[z];
 
        for (uint32_t j = 0U; j < sd->dma.num_in_segs; ++j, ++k) {
            seg = &start_src_seg[j];
            seg->addr = sd->seg.seg_type == SPARSE_MEMORY ?
                    NULL : (uint8_t *)sd->seg.src + k * sd->dma.src_seg_len;
            seg->len = sd->dma.src_seg_len;
 
            if (seg->addr != NULL)
                fill_data(seg->addr, seg->len);
        }
 
        len = sd->dma.num_in_segs * sd->dma.src_seg_len;
 
        for (uint32_t j = 0U; j < sd->dma.num_out_segs; ++j, ++z) {
            seg = &start_dst_seg[j];
            seg->addr = sd->seg.seg_type == SPARSE_MEMORY ?
                    NULL : (uint8_t *)sd->seg.dst + z * sd->dma.dst_seg_len;
            seg->len = ODPH_MIN(len, sd->dma.dst_seg_len);
            len -= sd->dma.dst_seg_len;
        }
 
        param = &sd->dma.infos[i].trs_param;
        odp_dma_transfer_param_init(param);
        param->src_format = ODP_DMA_FORMAT_ADDR;
        param->dst_format = ODP_DMA_FORMAT_ADDR;
        param->num_src = sd->dma.num_in_segs;
        param->num_dst = sd->dma.num_out_segs;
        param->src_seg = start_src_seg;
        param->dst_seg = start_dst_seg;
    }
}
 
static void free_memory(const sd_t *sd)
{
    if (sd->seg.src_shm != ODP_SHM_INVALID)
        (void)odp_shm_free(sd->seg.src_shm);
 
    if (sd->seg.dst_shm != ODP_SHM_INVALID)
        (void)odp_shm_free(sd->seg.dst_shm);
}
 
static void run_transfer(odp_dma_t handle, trs_info_t *info, stats_t *stats, ver_fn_t ver_fn)
{
    odp_time_t start_tm, end_tm;
    uint64_t start_cc, end_cc, trs_tm, trs_cc;
    odp_dma_result_t res;
    int ret;
 
    start_tm = odp_time_local_strict();
    start_cc = odp_cpu_cycles();
    ret = odp_dma_transfer(handle, &info->trs_param, &res);
    end_cc = odp_cpu_cycles();
    end_tm = odp_time_local_strict();
 
    if (odp_unlikely(ret <= 0)) {
        ++stats->start_errs;
    } else {
        trs_tm = odp_time_diff_ns(end_tm, start_tm);
        stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
        stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
        stats->trs_tm += trs_tm;
        trs_cc = odp_cpu_cycles_diff(end_cc, start_cc);
        stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
        stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
        stats->trs_cc += trs_cc;
        ++stats->trs_cnt;
        stats->max_start_cc = stats->max_trs_cc;
        stats->min_start_cc = stats->min_trs_cc;
        stats->start_cc += trs_cc;
        ++stats->start_cnt;
 
        if (odp_unlikely(!res.success)) {
            ++stats->transfer_errs;
        } else {
            ++stats->completed;
 
            if (ver_fn != NULL)
                ver_fn(info, stats);
        }
    }
}
 
static void run_transfers_mt_unsafe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    odp_dma_t handle = sd->dma.handle;
    trs_info_t *infos = sd->dma.infos, *info;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (sd->prep_trs_fn != NULL)
            sd->prep_trs_fn(sd, info);
 
        run_transfer(handle, info, stats, sd->ver_fn);
    }
}
 
static void run_transfers_mt_safe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    odp_dma_t handle = sd->dma.handle;
    trs_info_t *infos = sd->dma.infos, *info;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (odp_ticketlock_trylock(&info->lock)) {
            if (sd->prep_trs_fn != NULL)
                sd->prep_trs_fn(sd, info);
 
            run_transfer(handle, info, stats, sd->ver_fn);
            odp_ticketlock_unlock(&info->lock);
        }
    }
}
 
static odp_bool_t configure_poll_compl(sd_t *sd)
{
    odp_dma_compl_param_t *param;
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
        param = &sd->dma.infos[i].compl_param;
 
        odp_dma_compl_param_init(param);
        param->compl_mode = mode_map[sd->dma.compl_mode];
        param->transfer_id = odp_dma_transfer_id_alloc(sd->dma.handle);
 
        if (param->transfer_id == ODP_DMA_TRANSFER_ID_INVALID) {
            ODPH_ERR("Error allocating transfer ID\n");
            return false;
        }
    }
 
    return true;
}
 
static void poll_transfer(sd_t *sd, trs_info_t *info, stats_t *stats)
{
    uint64_t start_cc, end_cc, trs_tm, trs_cc, wait_cc, start_cc_diff;
    odp_time_t start_tm;
    odp_dma_t handle = sd->dma.handle;
    odp_dma_result_t res;
    int ret;
 
    if (info->is_running) {
        start_cc = odp_cpu_cycles();
        ret = odp_dma_transfer_done(handle, info->compl_param.transfer_id, &res);
        end_cc = odp_cpu_cycles();
 
        if (odp_unlikely(ret < 0)) {
            ++stats->poll_errs;
            return;
        }
 
        ++info->trs_poll_cnt;
        wait_cc = odp_cpu_cycles_diff(end_cc, start_cc);
        stats->max_wait_cc = ODPH_MAX(wait_cc, stats->max_wait_cc);
        stats->min_wait_cc = ODPH_MIN(wait_cc, stats->min_wait_cc);
        stats->wait_cc += wait_cc;
        ++stats->wait_cnt;
 
        if (ret == 0)
            return;
 
        trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm);
        stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
        stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
        stats->trs_tm += trs_tm;
        trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc);
        stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
        stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
        stats->trs_cc += trs_cc;
        stats->trs_poll_cnt += info->trs_poll_cnt;
        ++stats->trs_cnt;
 
        if (odp_unlikely(!res.success)) {
            ++stats->transfer_errs;
        } else {
            ++stats->completed;
 
            if (sd->ver_fn != NULL)
                sd->ver_fn(info, stats);
        }
 
        info->is_running = false;
    } else {
        if (sd->prep_trs_fn != NULL)
            sd->prep_trs_fn(sd, info);
 
        start_tm = odp_time_global_strict();
        start_cc = odp_cpu_cycles();
        ret = odp_dma_transfer_start(handle, &info->trs_param, &info->compl_param);
        end_cc = odp_cpu_cycles();
 
        if (odp_unlikely(ret <= 0)) {
            ++stats->start_errs;
        } else {
            info->trs_start_tm = start_tm;
            info->trs_start_cc = start_cc;
            info->trs_poll_cnt = 0U;
            start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
            stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
            stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
            stats->start_cc += start_cc_diff;
            ++stats->start_cnt;
            info->is_running = true;
        }
    }
}
 
static void poll_transfers_mt_unsafe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    trs_info_t *infos = sd->dma.infos;
 
    for (uint32_t i = 0U; i < count; ++i)
        poll_transfer(sd, &infos[i], stats);
}
 
static void poll_transfers_mt_safe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    trs_info_t *infos = sd->dma.infos, *info;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (odp_ticketlock_trylock(&info->lock)) {
            poll_transfer(sd, info, stats);
            odp_ticketlock_unlock(&info->lock);
        }
    }
}
 
static void drain_poll_transfers(sd_t *sd)
{
    const uint32_t count = sd->dma.num_inflight;
    trs_info_t *infos = sd->dma.infos, *info;
    odp_dma_t handle = sd->dma.handle;
    int rc;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (info->is_running) {
            do {
                rc = odp_dma_transfer_done(handle, info->compl_param.transfer_id,
                               NULL);
            } while (rc == 0);
        }
    }
}
 
static odp_bool_t configure_event_compl_session(sd_t *sd)
{
    odp_thrmask_t zero;
    odp_dma_pool_param_t pool_param;
    odp_queue_param_t queue_param;
 
    odp_thrmask_zero(&zero);
    sd->grp = odp_schedule_group_create(PROG_NAME "_scd_grp", &zero);
 
    if (sd->grp == ODP_SCHED_GROUP_INVALID) {
        ODPH_ERR("Error creating scheduler group for DMA session\n");
        return false;
    }
 
    odp_dma_pool_param_init(&pool_param);
    pool_param.num = sd->dma.num_inflight;
    sd->dma.pool = odp_dma_pool_create(PROG_NAME "_dma_evs", &pool_param);
 
    if (sd->dma.pool == ODP_POOL_INVALID) {
        ODPH_ERR("Error creating DMA event completion pool\n");
        return false;
    }
 
    odp_queue_param_init(&queue_param);
    queue_param.type = ODP_QUEUE_TYPE_SCHED;
    queue_param.sched.sync = ODP_SCHED_SYNC_PARALLEL;
    queue_param.sched.prio = odp_schedule_default_prio();
    queue_param.sched.group = sd->grp;
    sd->dma.compl_q = odp_queue_create(PROG_NAME, &queue_param);
 
    if (sd->dma.compl_q == ODP_QUEUE_INVALID) {
        ODPH_ERR("Error creating DMA completion queue\n");
        return false;
    }
 
    return true;
}
 
static odp_bool_t configure_event_compl(sd_t *sd)
{
    odp_dma_compl_param_t *param;
    odp_dma_compl_t c_ev;
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
        param = &sd->dma.infos[i].compl_param;
 
        odp_dma_compl_param_init(param);
        param->compl_mode = mode_map[sd->dma.compl_mode];
        c_ev = odp_dma_compl_alloc(sd->dma.pool);
 
        if (c_ev == ODP_DMA_COMPL_INVALID) {
            ODPH_ERR("Error allocating completion event\n");
            return false;
        }
 
        param->event = odp_dma_compl_to_event(c_ev);
        param->queue = sd->dma.compl_q;
        param->user_ptr = &sd->dma.infos[i];
    }
 
    return true;
}
 
static odp_bool_t start_initial_transfers(sd_t *sd)
{
    odp_time_t start_tm;
    uint64_t start_cc;
    trs_info_t *info;
    int ret;
 
    for (uint32_t i = 0U; i < sd->dma.num_inflight; ++i) {
        info = &sd->dma.infos[i];
 
        if (sd->prep_trs_fn != NULL)
            sd->prep_trs_fn(sd, info);
 
        start_tm = odp_time_global_strict();
        start_cc = odp_cpu_cycles();
        ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param);
 
        if (ret <= 0) {
            ODPH_ERR("Error starting DMA transfer\n");
            return false;
        }
 
        info->trs_start_tm = start_tm;
        info->trs_start_cc = start_cc;
    }
 
    return true;
}
 
static void wait_compl_event(sd_t *sd, stats_t *stats)
{
    uint64_t start_cc, end_cc, wait_cc, trs_tm, trs_cc, start_cc_diff;
    odp_time_t start_tm;
    odp_event_t ev;
    odp_dma_result_t res;
    trs_info_t *info;
    int ret;
 
    start_cc = odp_cpu_cycles();
    ev = odp_schedule(NULL, odp_schedule_wait_time(ODP_TIME_SEC_IN_NS));
    end_cc = odp_cpu_cycles();
 
    if (odp_unlikely(ev == ODP_EVENT_INVALID)) {
        ++stats->scheduler_timeouts;
        return;
    }
 
    odp_dma_compl_result(odp_dma_compl_from_event(ev), &res);
    info = res.user_ptr;
    trs_tm = odp_time_diff_ns(odp_time_global_strict(), info->trs_start_tm);
    stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
    stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
    stats->trs_tm += trs_tm;
    trs_cc = odp_cpu_cycles_diff(odp_cpu_cycles(), info->trs_start_cc);
    stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
    stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
    stats->trs_cc += trs_cc;
    ++stats->trs_cnt;
    wait_cc = odp_cpu_cycles_diff(end_cc, start_cc);
    stats->max_wait_cc = ODPH_MAX(wait_cc, stats->max_wait_cc);
    stats->min_wait_cc = ODPH_MIN(wait_cc, stats->min_wait_cc);
    stats->wait_cc += wait_cc;
    ++stats->wait_cnt;
 
    if (odp_unlikely(!res.success)) {
        ++stats->transfer_errs;
    } else {
        ++stats->completed;
 
        if (sd->ver_fn != NULL)
            sd->ver_fn(info, stats);
    }
 
    if (sd->prep_trs_fn != NULL)
        sd->prep_trs_fn(sd, info);
 
    start_tm = odp_time_global_strict();
    start_cc = odp_cpu_cycles();
    ret = odp_dma_transfer_start(sd->dma.handle, &info->trs_param, &info->compl_param);
    end_cc = odp_cpu_cycles();
 
    if (odp_unlikely(ret <= 0)) {
        ++stats->start_errs;
    } else {
        info->trs_start_tm = start_tm;
        info->trs_start_cc = start_cc;
        start_cc_diff = odp_cpu_cycles_diff(end_cc, start_cc);
        stats->max_start_cc = ODPH_MAX(start_cc_diff, stats->max_start_cc);
        stats->min_start_cc = ODPH_MIN(start_cc_diff, stats->min_start_cc);
        stats->start_cc += start_cc_diff;
        ++stats->start_cnt;
    }
}
 
static void drain_compl_events(ODP_UNUSED sd_t *sd)
{
    odp_event_t ev;
 
    while (true) {
        ev = odp_schedule(NULL, odp_schedule_wait_time(100 * ODP_TIME_MSEC_IN_NS));
 
        if (ev == ODP_EVENT_INVALID)
            break;
    }
}
 
static void run_memcpy(trs_info_t *info, stats_t *stats, ver_fn_t ver_fn)
{
    odp_time_t start_tm;
    uint64_t start_cc, end_cc, trs_tm, trs_cc;
    const odp_dma_transfer_param_t *param = &info->trs_param;
    uint32_t tot_len, src_len, dst_len, min_len, len, i = 0U, j = 0U, src_off = 0U,
    dst_off = 0U, src_rem, dst_rem;
    const odp_bool_t is_addr = param->src_format == ODP_DMA_FORMAT_ADDR;
    uint8_t *src_data, *dst_data;
 
    /* Test data is configured so that total source and total destination sizes always match,
     * all source and all destination segments have the same size and in case of packets,
     * there's always just a single segment. */
    tot_len = param->num_src * param->src_seg->len;
    src_len = param->src_seg->len;
    dst_len = param->dst_seg->len;
    min_len = ODPH_MIN(src_len, dst_len);
    len = min_len;
    start_tm = odp_time_local_strict();
    start_cc = odp_cpu_cycles();
 
    while (tot_len > 0U) {
        if (is_addr) {
            src_data = param->src_seg[i].addr;
            dst_data = param->dst_seg[j].addr;
        } else {
            src_data = odp_packet_data(param->src_seg[i].packet);
            dst_data = odp_packet_data(param->dst_seg[j].packet);
        }
 
        memcpy(dst_data + dst_off, src_data + src_off, len);
        dst_off += len;
        src_off += len;
        src_rem = src_len - src_off;
        dst_rem = dst_len - dst_off;
        tot_len -= len;
        len = ODPH_MIN(ODPH_MAX(src_rem, dst_rem), min_len);
 
        if (dst_rem > 0U) {
            ++i;
            src_off = 0U;
        } else {
            ++j;
            dst_off = 0U;
        }
    }
 
    end_cc = odp_cpu_cycles();
    trs_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm);
    stats->max_trs_tm = ODPH_MAX(trs_tm, stats->max_trs_tm);
    stats->min_trs_tm = ODPH_MIN(trs_tm, stats->min_trs_tm);
    stats->trs_tm += trs_tm;
    trs_cc = odp_cpu_cycles_diff(end_cc, start_cc);
    stats->max_trs_cc = ODPH_MAX(trs_cc, stats->max_trs_cc);
    stats->min_trs_cc = ODPH_MIN(trs_cc, stats->min_trs_cc);
    stats->trs_cc += trs_cc;
    ++stats->trs_cnt;
    stats->max_start_cc = stats->max_trs_cc;
    stats->min_start_cc = stats->min_trs_cc;
    stats->start_cc += trs_cc;
    ++stats->start_cnt;
    ++stats->completed;
 
    if (ver_fn != NULL)
        ver_fn(info, stats);
}
 
static void run_memcpy_mt_unsafe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    trs_info_t *infos = sd->dma.infos, *info;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (sd->prep_trs_fn != NULL)
            sd->prep_trs_fn(sd, info);
 
        run_memcpy(info, stats, sd->ver_fn);
    }
}
 
static void run_memcpy_mt_safe(sd_t *sd, stats_t *stats)
{
    const uint32_t count = sd->dma.num_inflight;
    trs_info_t *infos = sd->dma.infos, *info;
 
    for (uint32_t i = 0U; i < count; ++i) {
        info = &infos[i];
 
        if (odp_ticketlock_trylock(&info->lock)) {
            if (sd->prep_trs_fn != NULL)
                sd->prep_trs_fn(sd, info);
 
            run_memcpy(info, stats, sd->ver_fn);
            odp_ticketlock_unlock(&info->lock);
        }
    }
}
 
static void setup_api(prog_config_t *config)
{
    if (config->seg_type == DENSE_PACKET || config->seg_type == SPARSE_PACKET) {
        config->api.setup_fn = setup_packet_segments;
        config->api.trs_fn = configure_packet_transfer;
        config->api.free_fn = free_packets;
    } else {
        config->api.setup_fn = setup_memory_segments;
        config->api.trs_fn = configure_address_transfer;
        config->api.free_fn = free_memory;
    }
 
    if (config->trs_type == SYNC_DMA) {
        config->api.session_cfg_fn = NULL;
        config->api.compl_fn = NULL;
        config->api.bootstrap_fn = NULL;
        config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
                    run_transfers_mt_unsafe : run_transfers_mt_safe;
        config->api.drain_fn = NULL;
    } else if (config->trs_type == ASYNC_DMA) {
        if (config->compl_mode == POLL) {
            config->api.session_cfg_fn = NULL;
            config->api.compl_fn = configure_poll_compl;
            config->api.bootstrap_fn = NULL;
            config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
                        poll_transfers_mt_unsafe : poll_transfers_mt_safe;
            config->api.drain_fn = drain_poll_transfers;
        } else {
            config->api.session_cfg_fn = configure_event_compl_session;
            config->api.compl_fn = configure_event_compl;
            config->api.bootstrap_fn = start_initial_transfers;
            config->api.wait_fn = wait_compl_event;
            config->api.drain_fn = drain_compl_events;
        }
    } else {
        config->api.session_cfg_fn = NULL;
        config->api.compl_fn = NULL;
        config->api.bootstrap_fn = NULL;
        config->api.wait_fn = config->num_workers == 1 || config->policy == MANY ?
                    run_memcpy_mt_unsafe : run_memcpy_mt_safe;
        config->api.drain_fn = NULL;
    }
}
 
static void prepare_packet_transfer(sd_t *sd, trs_info_t *info)
{
    odp_dma_transfer_param_t *param = &info->trs_param;
    odp_dma_seg_t *seg;
 
    for (uint32_t i = 0U; i < param->num_src; ++i) {
        seg = &param->src_seg[i];
 
        if (odp_likely(seg->packet != ODP_PACKET_INVALID))
            odp_packet_free(seg->packet);
 
        seg->packet = odp_packet_alloc(sd->seg.src_pool, seg->len);
 
        if (odp_unlikely(seg->packet == ODP_PACKET_INVALID))
            /* There should always be enough packets. */
            ODPH_ABORT("Failed to allocate packet, aborting\n");
 
        fill_data(odp_packet_data(seg->packet), seg->len);
    }
 
    for (uint32_t i = 0U; i < param->num_dst; ++i) {
        seg = &param->dst_seg[i];
 
        if (odp_likely(seg->packet != ODP_PACKET_INVALID))
            odp_packet_free(seg->packet);
 
        seg->packet = odp_packet_alloc(sd->seg.dst_pool, seg->len);
 
        if (odp_unlikely(seg->packet == ODP_PACKET_INVALID))
            /* There should always be enough packets. */
            ODPH_ABORT("Failed to allocate packet, aborting\n");
    }
}
 
static void prepare_address_transfer(sd_t *sd, trs_info_t *info)
{
    odp_dma_transfer_param_t *param = &info->trs_param;
    uint8_t *addr = sd->seg.cur_src;
    odp_dma_seg_t *seg;
 
    for (uint32_t i = 0U; i < param->num_src; ++i) {
        seg = &param->src_seg[i];
 
        if (odp_unlikely(addr > (uint8_t *)sd->seg.src_high))
            addr = sd->seg.src;
 
        seg->addr = addr;
        addr += sd->dma.src_seg_len;
        fill_data(seg->addr, seg->len);
    }
 
    sd->seg.cur_src = addr + ODP_CACHE_LINE_SIZE;
    addr = sd->seg.cur_dst;
 
    for (uint32_t i = 0U; i < param->num_dst; ++i) {
        if (odp_unlikely(addr > (uint8_t *)sd->seg.dst_high))
            addr = sd->seg.dst;
 
        param->dst_seg[i].addr = addr;
        addr += sd->dma.dst_seg_len;
    }
 
    sd->seg.cur_dst = addr + ODP_CACHE_LINE_SIZE;
}
 
static void verify_transfer(trs_info_t *info, stats_t *stats)
{
    odp_dma_transfer_param_t *param = &info->trs_param;
    odp_dma_seg_t *seg;
    const odp_bool_t is_addr = param->dst_format == ODP_DMA_FORMAT_ADDR;
    uint8_t *data;
 
    for (uint32_t i = 0U; i < param->num_dst; ++i) {
        seg = &param->dst_seg[i];
        data = is_addr ? seg->addr : odp_packet_data(seg->packet);
 
        for (uint32_t j = 0U; j < seg->len; ++j)
            if (odp_unlikely(data[j] != DATA)) {
                ++stats->data_errs;
                return;
            }
    }
}
 
static odp_bool_t setup_session_descriptors(prog_config_t *config)
{
    sd_t *sd;
    const odp_dma_param_t dma_params = {
        .direction = ODP_DMA_MAIN_TO_MAIN,
        .type = ODP_DMA_TYPE_COPY,
        .compl_mode_mask = config->compl_mode_mask,
        .mt_mode = config->num_workers == 1 || config->policy == MANY ?
                ODP_DMA_MT_SERIAL : ODP_DMA_MT_SAFE,
        .order = ODP_DMA_ORDER_NONE };
 
    for (uint32_t i = 0U; i < config->num_sessions; ++i) {
        char name[ODP_DMA_NAME_LEN];
 
        sd = &config->sds[i];
        sd->dma.num_in_segs = config->num_in_segs;
        sd->dma.num_out_segs = config->num_out_segs;
        sd->dma.src_seg_len = config->src_seg_len;
        sd->dma.dst_seg_len = config->dst_seg_len;
        sd->dma.num_inflight = config->num_inflight;
        sd->dma.trs_type = config->trs_type;
        sd->dma.compl_mode = config->compl_mode;
        snprintf(name, sizeof(name), PROG_NAME "_dma_%u", i);
        sd->dma.handle = odp_dma_create(name, &dma_params);
 
        if (sd->dma.handle == ODP_DMA_INVALID) {
            ODPH_ERR("Error creating DMA session\n");
            return false;
        }
 
        if (config->api.session_cfg_fn != NULL && !config->api.session_cfg_fn(sd))
            return false;
 
        sd->seg.shm_size = config->shm_size;
        sd->seg.seg_type = config->seg_type;
        sd->prep_trs_fn = config->seg_type == SPARSE_PACKET ? prepare_packet_transfer :
                    config->seg_type == SPARSE_MEMORY ?
                        prepare_address_transfer : NULL;
        sd->ver_fn = config->is_verify ? verify_transfer : NULL;
    }
 
    return true;
}
 
static odp_bool_t setup_data(prog_config_t *config)
{
    sd_t *sd;
 
    for (uint32_t i = 0U; i < config->num_sessions; ++i) {
        sd = &config->sds[i];
 
        if (!config->api.setup_fn(sd))
            return false;
 
        config->api.trs_fn(sd);
 
        if (config->api.compl_fn != NULL && !config->api.compl_fn(sd))
            return false;
    }
 
    return true;
}
 
static int transfer(void *args)
{
    thread_config_t *thr_config = args;
    prog_config_t *prog_config = thr_config->prog_config;
    sd_t *sd = thr_config->sd;
    stats_t *stats = &thr_config->stats;
    test_api_t *api = &prog_conf->api;
    odp_thrmask_t mask;
    odp_time_t start_tm;
 
    odp_barrier_wait(&prog_config->init_barrier);
 
    if (sd->grp != ODP_SCHED_GROUP_INVALID) {
        odp_thrmask_zero(&mask);
        odp_thrmask_set(&mask, odp_thread_id());
 
        if (odp_schedule_group_join(sd->grp, &mask) < 0) {
            ODPH_ERR("Error joining scheduler group\n");
            goto out;
        }
    }
 
    start_tm = odp_time_local_strict();
 
    while (odp_atomic_load_u32(&prog_config->is_running))
        api->wait_fn(sd, stats);
 
    thr_config->stats.tot_tm = odp_time_diff_ns(odp_time_local_strict(), start_tm);
 
    if (api->drain_fn != NULL)
        api->drain_fn(sd);
 
out:
    odp_barrier_wait(&prog_config->term_barrier);
 
    return 0;
}
 
static odp_bool_t setup_workers(prog_config_t *config)
{
    odp_cpumask_t cpumask;
    int num_workers;
    odph_thread_common_param_t thr_common;
    odph_thread_param_t thr_params[config->num_workers], *thr_param;
    thread_config_t *thr_config;
    sd_t *sd;
 
    /* Barrier init count for control and worker. */
    odp_barrier_init(&config->init_barrier, config->num_workers + 1);
    odp_barrier_init(&config->term_barrier, config->num_workers);
    num_workers = odp_cpumask_default_worker(&cpumask, config->num_workers);
    odph_thread_common_param_init(&thr_common);
    thr_common.instance = config->odp_instance;
    thr_common.cpumask = &cpumask;
 
    for (int i = 0; i < config->num_workers; ++i) {
        thr_param = &thr_params[i];
        thr_config = &config->thread_config[i];
        sd = config->policy == SINGLE ? &config->sds[0U] : &config->sds[i];
 
        odph_thread_param_init(thr_param);
        thr_param->start = transfer;
        thr_param->thr_type = ODP_THREAD_WORKER;
        thr_config->prog_config = config;
        thr_config->sd = sd;
        thr_param->arg = thr_config;
    }
 
    num_workers = odph_thread_create(config->threads, &thr_common, thr_params, num_workers);
 
    if (num_workers != config->num_workers) {
        ODPH_ERR("Error configuring worker threads\n");
        return false;
    }
 
    for (uint32_t i = 0U; i < config->num_sessions; ++i) {
        if (config->api.bootstrap_fn != NULL && !config->api.bootstrap_fn(&config->sds[i]))
            return false;
    }
 
    odp_barrier_wait(&config->init_barrier);
 
    return true;
}
 
static odp_bool_t setup_test(prog_config_t *config)
{
    setup_api(config);
 
    return setup_session_descriptors(config) && setup_data(config) && setup_workers(config);
}
 
static void stop_test(prog_config_t *config)
{
    (void)odph_thread_join(config->threads, config->num_workers);
}
 
static void teardown_data(const sd_t *sd, void (*free_fn)(const sd_t *sd))
{
    const odp_dma_compl_param_t *compl_param;
 
    for (uint32_t i = 0U; i < MAX_SEGS; ++i) {
        compl_param = &sd->dma.infos[i].compl_param;
 
        if (compl_param->transfer_id != ODP_DMA_TRANSFER_ID_INVALID)
            odp_dma_transfer_id_free(sd->dma.handle, compl_param->transfer_id);
 
        if (compl_param->event != ODP_EVENT_INVALID)
            odp_event_free(compl_param->event);
    }
 
    free_fn(sd);
}
 
static void teardown_test(prog_config_t *config)
{
    sd_t *sd;
 
    for (uint32_t i = 0U; i < config->num_sessions; ++i) {
        sd = &config->sds[i];
        teardown_data(sd, config->api.free_fn);
 
        if (sd->dma.compl_q != ODP_QUEUE_INVALID)
            (void)odp_queue_destroy(sd->dma.compl_q);
 
        if (sd->dma.pool != ODP_POOL_INVALID)
            (void)odp_pool_destroy(sd->dma.pool);
 
        if (sd->grp != ODP_SCHED_GROUP_INVALID)
            (void)odp_schedule_group_destroy(sd->grp);
 
        if (sd->dma.handle != ODP_DMA_INVALID)
            (void)odp_dma_destroy(sd->dma.handle);
    }
 
    if (config->src_pool != ODP_POOL_INVALID)
        (void)odp_pool_destroy(config->src_pool);
 
    if (config->dst_pool != ODP_POOL_INVALID)
        (void)odp_pool_destroy(config->dst_pool);
}
 
static void print_humanised(uint64_t value, const char *type)
{
    if (value > GIGAS)
        printf("%.2f G%s\n", (double)value / GIGAS, type);
    else if (value > MEGAS)
        printf("%.2f M%s\n", (double)value / MEGAS, type);
    else if (value > KILOS)
        printf("%.2f k%s\n", (double)value / KILOS, type);
    else
        printf("%" PRIu64 " %s\n", value, type);
}
 
static int output_results(const prog_config_t *config)
{
    const stats_t *stats;
    uint64_t data_cnt = config->num_in_segs * config->src_seg_len, tot_completed = 0U,
    tot_tm = 0U, tot_trs_tm = 0U, tot_trs_cc = 0U, tot_trs_cnt = 0U, tot_min_tm = UINT64_MAX,
    tot_max_tm = 0U, tot_min_cc = UINT64_MAX, tot_max_cc = 0U, avg_start_cc,
    avg_start_cc_tot = 0U, min_start = UINT64_MAX, max_start = 0U, avg_wait_cc,
    avg_wait_cc_tot = 0U, min_wait = UINT64_MAX, max_wait = 0U, start_cnt_sum = 0U,
    wait_cnt_sum = 0U;
    double avg_tot_tm;
 
    printf("\n======================\n\n"
           "DMA performance test done\n\n"
           "    mode:                 %s\n"
           "    input segment count:  %u\n"
           "    output segment count: %u\n"
           "    segment length:       %u\n"
           "    segment type:         %s\n"
           "    inflight count:       %u\n"
           "    session policy:       %s\n\n",
           config->trs_type == SYNC_DMA ? "DMA synchronous" :
        config->trs_type == ASYNC_DMA && config->compl_mode == POLL ?
            "DMA asynchronous-poll" :
                config->trs_type == ASYNC_DMA && config->compl_mode == EVENT ?
                    "DMA asynchronous-event" : "SW", config->num_in_segs,
           config->num_out_segs, config->src_seg_len,
           config->seg_type == DENSE_PACKET ? "dense packet" :
        config->seg_type == SPARSE_PACKET ? "sparse packet" :
            config->seg_type == DENSE_MEMORY ? "dense memory" : "sparse memory",
           config->num_inflight, config->policy == SINGLE ? "shared" : "per-worker");
 
    for (int i = 0; i < config->num_workers; ++i) {
        stats = &config->thread_config[i].stats;
        tot_completed += stats->completed;
        tot_tm += stats->tot_tm;
        tot_trs_tm += stats->trs_tm;
        tot_trs_cc += stats->trs_cc;
        tot_trs_cnt += stats->trs_cnt;
        tot_min_tm = ODPH_MIN(tot_min_tm, stats->min_trs_tm);
        tot_max_tm = ODPH_MAX(tot_max_tm, stats->max_trs_tm);
        tot_min_cc = ODPH_MIN(tot_min_cc, stats->min_trs_cc);
        tot_max_cc = ODPH_MAX(tot_max_cc, stats->max_trs_cc);
        avg_start_cc = 0U;
        avg_wait_cc = 0U;
 
        printf("    worker %d:\n", i);
        printf("        successful transfers: %" PRIu64 "\n"
               "        start errors:         %" PRIu64 "\n",
               stats->completed, stats->start_errs);
 
        if (config->trs_type == ASYNC_DMA) {
            if (config->compl_mode == POLL)
                printf("        poll errors:          %" PRIu64 "\n",
                       stats->poll_errs);
            else
                printf("        scheduler timeouts:   %" PRIu64 "\n",
                       stats->scheduler_timeouts);
        }
 
        printf("        transfer errors:      %" PRIu64 "\n", stats->transfer_errs);
 
        if (config->is_verify)
            printf("        data errors:          %" PRIu64 "\n", stats->data_errs);
 
        printf("        run time:             %" PRIu64 " ns\n", stats->tot_tm);
 
        if (config->policy == MANY) {
            printf("        session:\n"
                   "            average time per transfer:   %" PRIu64 " "
                   "(min: %" PRIu64 ", max: %" PRIu64 ") ns\n"
                   "            average cycles per transfer: %" PRIu64 " "
                   "(min: %" PRIu64 ", max: %" PRIu64 ")\n"
                   "            ops:                         ",
                   stats->trs_cnt > 0U ? stats->trs_tm / stats->trs_cnt : 0U,
                   stats->trs_cnt > 0U ? stats->min_trs_tm : 0U,
                   stats->trs_cnt > 0U ? stats->max_trs_tm : 0U,
                   stats->trs_cnt > 0U ? stats->trs_cc / stats->trs_cnt : 0U,
                   stats->trs_cnt > 0U ? stats->min_trs_cc : 0U,
                   stats->trs_cnt > 0U ? stats->max_trs_cc : 0U);
            print_humanised(stats->completed /
                    ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS),
                    "OPS");
            printf("            speed:                       ");
            print_humanised(stats->completed * data_cnt /
                    ((double)stats->tot_tm / ODP_TIME_SEC_IN_NS), "B/s");
        }
 
        if (stats->start_cnt > 0U) {
            avg_start_cc = stats->start_cc / stats->start_cnt;
            start_cnt_sum += stats->start_cnt;
            avg_start_cc_tot += stats->start_cc;
            min_start = stats->min_start_cc < min_start ?
                stats->min_start_cc : min_start;
            max_start = stats->max_start_cc > max_start ?
                stats->max_start_cc : max_start;
        }
 
        printf("        average cycles breakdown:\n");
 
        if (config->trs_type == SYNC_DMA) {
            printf("            odp_dma_transfer(): %" PRIu64 " "
                   "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
                   avg_start_cc > 0U ? stats->min_start_cc : 0U,
                   avg_start_cc > 0U ? stats->max_start_cc : 0U);
        } else if (config->trs_type == SW_COPY) {
            printf("            memcpy(): %" PRIu64 " "
                   "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
                   avg_start_cc > 0U ? stats->min_start_cc : 0U,
                   avg_start_cc > 0U ? stats->max_start_cc : 0U);
        } else {
            printf("            odp_dma_transfer_start(): %" PRIu64 " "
                   "(min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_start_cc,
                   avg_start_cc > 0U ? stats->min_start_cc : 0U,
                   avg_start_cc > 0U ? stats->max_start_cc : 0U);
 
            if (stats->wait_cnt > 0U) {
                avg_wait_cc = stats->wait_cc / stats->wait_cnt;
                wait_cnt_sum += stats->wait_cnt;
                avg_wait_cc_tot += stats->wait_cc;
                min_wait = stats->min_wait_cc < min_wait ?
                    stats->min_wait_cc : min_wait;
                max_wait = stats->max_wait_cc > max_wait ?
                    stats->max_wait_cc : max_wait;
            }
 
            if (config->compl_mode == POLL) {
                printf("            odp_dma_transfer_done():  %" PRIu64 ""
                       " (min: %" PRIu64 ", max: %" PRIu64 ", x%" PRIu64 ""
                       " per transfer)\n", avg_wait_cc,
                       avg_wait_cc > 0U ? stats->min_wait_cc : 0U,
                       avg_wait_cc > 0U ? stats->max_wait_cc : 0U,
                       stats->trs_cnt > 0U ?
                        stats->trs_poll_cnt / stats->trs_cnt : 0U);
            } else {
                printf("            odp_schedule():           %" PRIu64 " "
                       " (min: %" PRIu64 ", max: %" PRIu64 ")\n", avg_wait_cc,
                       avg_wait_cc > 0U ? stats->min_wait_cc : 0U,
                       avg_wait_cc > 0U ? stats->max_wait_cc : 0U);
            }
        }
 
        printf("\n");
    }
    avg_start_cc_tot = start_cnt_sum > 0U ? avg_start_cc_tot / start_cnt_sum : 0U;
    avg_wait_cc_tot = wait_cnt_sum > 0U ? avg_wait_cc_tot / wait_cnt_sum : 0U;
 
    avg_tot_tm = (double)tot_tm / config->num_workers / ODP_TIME_SEC_IN_NS;
    printf("    total:\n"
           "        average time per transfer:   %" PRIu64 " (min: %" PRIu64
           ", max: %" PRIu64 ") ns\n"
           "        average cycles per transfer: %" PRIu64 " (min: %" PRIu64
           ", max: %" PRIu64 ")\n"
           "        ops:                         ",
           tot_trs_cnt > 0U ? tot_trs_tm / tot_trs_cnt : 0U,
           tot_trs_cnt > 0U ? tot_min_tm : 0U,
           tot_trs_cnt > 0U ? tot_max_tm : 0U,
           tot_trs_cnt > 0U ? tot_trs_cc / tot_trs_cnt : 0U,
           tot_trs_cnt > 0U ? tot_min_cc : 0U,
           tot_trs_cnt > 0U ? tot_max_cc : 0U);
    print_humanised(avg_tot_tm > 0U ? tot_completed / avg_tot_tm : 0U, "OPS");
    printf("        speed:                       ");
    print_humanised(avg_tot_tm > 0U ? tot_completed * data_cnt / avg_tot_tm : 0U, "B/s");
    printf("\n");
    printf("======================\n");
 
    if (config->common_options.is_export) {
        /* Write header */
        if (test_common_write("time per transfer avg (ns),time per transfer min (ns),"
                      "time per transfer max (ns),cycles per transfer avg,"
                      "cycles per transfer min,cycles per transfer max,"
                      "ops (OPS),speed (B/s),dma_transfer avg,"
                      "dma_transfer min,dma_transfer max,memcpy avg,memcpy min,"
                      "memcpy max,dma_transfer_start avg,dma_transfer_start min,"
                      "dma_transfer_start max,dma_transfer_done avg,"
                      "dma_transfer_done min,dma_transfer_done max,schedule avg,"
                      "schedule min,schedule max\n"))
            goto exit;
        /* Write the values always present, disregarding parameters */
        if (test_common_write("%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ","
                      "%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",",
                      tot_trs_cnt > 0U ? tot_trs_tm / tot_trs_cnt : 0U,
                      tot_trs_cnt > 0U ? tot_min_tm : 0U,
                      tot_trs_cnt > 0U ? tot_max_tm : 0U,
                      tot_trs_cnt > 0U ? tot_trs_cc / tot_trs_cnt : 0U,
                      tot_trs_cnt > 0U ? tot_min_cc : 0U,
                      tot_trs_cnt > 0U ? tot_max_cc : 0U,
                      avg_tot_tm > 0U ? (uint64_t)(tot_completed / avg_tot_tm) : 0U,
                      avg_tot_tm > 0U ?
                      (uint64_t)(tot_completed * data_cnt / avg_tot_tm) : 0U))
            goto exit;
        /* Write the function specific values */
        if (config->trs_type == SYNC_DMA) {
            if (test_common_write("%" PRIu64 ",%" PRIu64 ",%" PRIu64 ","
                          "0,0,0,0,0,0,0,0,0,0,0,0\n",
                          avg_start_cc_tot,
                          avg_start_cc_tot > 0U ? min_start : 0U,
                          avg_start_cc_tot > 0U ? max_start : 0U))
                goto exit;
        } else if (config->trs_type == SW_COPY) {
            if (test_common_write("0,0,0 %" PRIu64 ",%" PRIu64 ",%" PRIu64 ","
                          "0,0,0,0,0,0,0,0,0\n",
                          avg_start_cc_tot,
                          avg_start_cc_tot > 0U ? min_start : 0U,
                          avg_start_cc_tot > 0U ? max_start : 0U))
                goto exit;
        } else if (config->trs_type == ASYNC_DMA) {
            if (test_common_write("0,0,0,0,0,0, %" PRIu64 ",%" PRIu64 ",%" PRIu64 ",",
                          avg_start_cc_tot,
                          avg_start_cc_tot > 0U ? min_start : 0U,
                          avg_start_cc_tot > 0U ? max_start : 0U))
                goto exit;
 
            if (config->compl_mode == POLL) {
                if (test_common_write("%" PRIu64 ",%" PRIu64 ",%" PRIu64 ","
                              "0,0,0\n",
                              avg_wait_cc_tot,
                              avg_wait_cc_tot > 0U ? min_wait : 0U,
                              avg_wait_cc_tot > 0U ? max_wait : 0U))
                    goto exit;
            } else if (config->compl_mode == EVENT) {
                if (test_common_write("0,0,0 %" PRIu64 ",%" PRIu64 ",%" PRIu64 "\n",
                              avg_wait_cc_tot,
                              avg_wait_cc_tot > 0U ? min_wait : 0U,
                              avg_wait_cc_tot > 0U ? max_wait : 0U))
                    goto exit;
            }
        }
        test_common_write_term();
    }
 
    return 0;
 
exit:
    ODPH_ERR("Export failed\n");
    test_common_write_term();
    return -1;
}
 
int main(int argc, char **argv)
{
    odph_helper_options_t odph_opts;
    odp_init_t init_param;
    odp_instance_t odp_instance;
    odp_shm_t shm_cfg = ODP_SHM_INVALID;
    parse_result_t parse_res;
    int ret = EXIT_SUCCESS;
    test_common_options_t common_options;
 
    argc = odph_parse_options(argc, argv);
 
    if (odph_options(&odph_opts)) {
        ODPH_ERR("Error while reading ODP helper options, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    argc = test_common_parse_options(argc, argv);
    if (test_common_options(&common_options)) {
        ODPH_ERR("Error while reading test options, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    odp_init_param_init(&init_param);
    init_param.mem_model = odph_opts.mem_model;
 
    if (odp_init_global(&odp_instance, &init_param, NULL)) {
        ODPH_ERR("ODP global init failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_init_local(odp_instance, ODP_THREAD_CONTROL)) {
        ODPH_ERR("ODP local init failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE,
                  0U);
 
    if (shm_cfg == ODP_SHM_INVALID) {
        ODPH_ERR("Error reserving shared memory\n");
        ret = EXIT_FAILURE;
        goto out;
    }
 
    prog_conf = odp_shm_addr(shm_cfg);
 
    if (prog_conf == NULL) {
        ODPH_ERR("Error resolving shared memory address\n");
        ret = EXIT_FAILURE;
        goto out;
    }
 
    parse_res = setup_program(argc, argv, prog_conf);
 
    if (parse_res == PRS_NOK) {
        ret = EXIT_FAILURE;
        goto out;
    }
 
    if (parse_res == PRS_TERM) {
        ret = EXIT_SUCCESS;
        goto out;
    }
 
    if (parse_res == PRS_NOT_SUP) {
        ret = EXIT_NOT_SUP;
        goto out;
    }
 
    if (odp_schedule_config(NULL) < 0) {
        ODPH_ERR("Error configuring scheduler\n");
        ret = EXIT_FAILURE;
        goto out;
    }
 
    prog_conf->odp_instance = odp_instance;
    odp_atomic_init_u32(&prog_conf->is_running, 1U);
 
    if (!setup_test(prog_conf)) {
        ret = EXIT_FAILURE;
        goto out_test;
    }
 
    if (prog_conf->time_sec > 0.001) {
        struct timespec ts;
 
        ts.tv_sec = prog_conf->time_sec;
        ts.tv_nsec = (prog_conf->time_sec - ts.tv_sec) * ODP_TIME_SEC_IN_NS;
        nanosleep(&ts, NULL);
        odp_atomic_store_u32(&prog_conf->is_running, 0U);
    }
 
    stop_test(prog_conf);
 
    prog_conf->common_options = common_options;
 
    output_results(prog_conf);
 
out_test:
    /* Release all resources that have been allocated during 'setup_test()'. */
    teardown_test(prog_conf);
 
out:
    if (shm_cfg != ODP_SHM_INVALID)
        (void)odp_shm_free(shm_cfg);
 
    if (odp_term_local()) {
        ODPH_ERR("ODP local terminate failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_term_global(odp_instance)) {
        ODPH_ERR("ODP global terminate failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    return ret;
}