odp_ipsecfwd.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2022-2023 Nokia
 */
 
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
 
#include <stdlib.h>
#include <signal.h>
#include <stdio.h>
#include <inttypes.h>
#include <string.h>
 
#include <errno.h>
 
#include <odp_api.h>
#include <odp/helper/odph_api.h>
#include <libconfig.h>
 
#define PROG_NAME "odp_ipsecfwd"
#define SHORT_PROG_NAME "ipsfwd"
#define DELIMITER ","
 
#define MAX_IFS 2U
#define MAX_SAS 4000U
#define MAX_FWDS 64U
#define MAX_SPIS (UINT16_MAX + 1U)
#define MAX_WORKERS (ODP_THREAD_COUNT_MAX - 1)
#define MAX_QUEUES 64U
#define MAX_SA_QUEUES 1024U
#define PKT_SIZE 1024U
#define PKT_CNT 32768U
#define MAX_BURST 32U
#define ORDERED 0U
#define IP_ADDR_LEN 32U
 
#define ALG_ENTRY(_alg_name, _type) \
    { \
        .idx = (_alg_name), \
        .type = (_type), \
        .name = #_alg_name \
    }
 
enum {
    CIPHER_TYPE,
    COMB_CIPHER_TYPE,
    AUTH_TYPE,
    COMB_AUTH_TYPE
};
 
typedef enum {
    PRS_OK,
    PRS_NOK,
    PRS_TERM
} parse_result_t;
 
enum {
    DIR_IN = 0,
    DIR_OUT
};
 
typedef struct pktio_s pktio_t;
 
typedef struct pktio_s {
    union {
        odp_pktout_queue_t out_dir_qs[MAX_QUEUES];
        odp_queue_t out_ev_qs[MAX_QUEUES];
    };
 
    odp_pktin_queue_t in_dir_qs[MAX_QUEUES];
    odph_ethaddr_t src_mac;
    char *name;
    odp_pktio_t handle;
    uint32_t (*send_fn)(const pktio_t *pktio, uint8_t index, odp_packet_t pkts[], int num);
    uint32_t num_tx_qs;
    uint8_t idx;
} pktio_t;
 
typedef struct {
    uint32_t prefix;
    uint32_t mask;
    odph_ethaddr_t dst_mac;
    const pktio_t *pktio;
} fwd_entry_t;
 
typedef struct {
    fwd_entry_t entries[MAX_FWDS];
    uint32_t num;
} lookup_table_t;
 
typedef struct {
    uint64_t ipsec_in_pkts;
    uint64_t ipsec_out_pkts;
    uint64_t ipsec_in_errs;
    uint64_t ipsec_out_errs;
    uint64_t status_errs;
    uint64_t fwd_pkts;
    uint64_t discards;
} stats_t;
 
typedef struct prog_config_s prog_config_t;
 
typedef struct ODP_ALIGNED_CACHE {
    stats_t stats;
    prog_config_t *prog_config;
    int thr_idx;
    uint8_t pktio;
} thread_config_t;
 
typedef struct {
    odp_ipsec_sa_param_t sa_param;
    char cipher_key[65U];
    char cipher_key_extra[5U];
    char auth_key[65U];
    char auth_key_extra[5U];
    odp_u32be_t lkp_dst_ip;
    odp_u32be_t src_ip;
    odp_u32be_t dst_ip;
} sa_config_t;
 
typedef uint32_t (*rx_fn_t)(thread_config_t *config, odp_event_t evs[], int num);
typedef void (*ipsec_fn_t)(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl, stats_t *stats);
typedef void (*drain_fn_t)(prog_config_t *config);
 
typedef struct {
    rx_fn_t rx;
    ipsec_fn_t proc;
    ipsec_fn_t compl;
    drain_fn_t drain;
} ops_t;
 
typedef struct prog_config_s {
    odph_thread_t thread_tbl[MAX_WORKERS];
    thread_config_t thread_config[MAX_WORKERS];
    odp_ipsec_sa_t sas[MAX_SAS];
    odp_queue_t sa_qs[MAX_SA_QUEUES];
    pktio_t pktios[MAX_IFS];
    lookup_table_t fwd_tbl;
    odp_atomic_u32_t is_running;
    sa_config_t default_cfg;
    ops_t ops;
    char *conf_file;
    odp_instance_t odp_instance;
    odp_queue_t compl_q;
    odp_pool_t pktio_pool;
    odp_barrier_t init_barrier;
    odp_barrier_t term_barrier;
    uint32_t num_input_qs;
    uint32_t num_sa_qs;
    uint32_t num_output_qs;
    uint32_t num_pkts;
    uint32_t pkt_len;
    uint32_t num_ifs;
    uint32_t num_sas;
    int num_thrs;
    odp_bool_t is_dir_rx;
    odp_bool_t is_hashed_tx;
    uint8_t mode;
} prog_config_t;
 
typedef struct {
    const char *name;
    int idx;
    int type;
} exposed_alg_t;
 
typedef struct {
    odp_packet_t pkts[MAX_BURST];
    const pktio_t *pktio;
    uint32_t num;
} pkt_vec_t;
 
typedef struct {
    pkt_vec_t vecs[MAX_QUEUES];
    uint8_t num_qs;
} pkt_out_t;
 
typedef struct {
    pkt_out_t ifs[MAX_IFS];
    odp_bool_t is_hashed_tx;
    uint8_t q_idx;
} pkt_ifs_t;
 
static const exposed_alg_t exposed_algs[] = {
    ALG_ENTRY(ODP_CIPHER_ALG_NULL, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_DES, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_3DES_CBC, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_AES_CBC, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_AES_CTR, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_AES_ECB, CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_AES_GCM, COMB_CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_AES_CCM, COMB_CIPHER_TYPE),
    ALG_ENTRY(ODP_CIPHER_ALG_CHACHA20_POLY1305, COMB_CIPHER_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_NULL, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_MD5_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_SHA1_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_SHA224_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_SHA256_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_SHA384_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_SHA512_HMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_AES_GCM, COMB_AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_AES_GMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_AES_CCM, COMB_AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_AES_CMAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_AES_XCBC_MAC, AUTH_TYPE),
    ALG_ENTRY(ODP_AUTH_ALG_CHACHA20_POLY1305, COMB_AUTH_TYPE)
};
 
/* SPIs for in and out directions */
static odp_ipsec_sa_t *spi_to_sa_map[2U][MAX_SPIS];
static const int ipsec_out_mark;
static __thread pkt_ifs_t ifs;
static prog_config_t *prog_conf;
 
static void init_config(prog_config_t *config)
{
    memset(config, 0, sizeof(*config));
    odp_ipsec_sa_param_init(&config->default_cfg.sa_param);
    config->compl_q = ODP_QUEUE_INVALID;
    config->pktio_pool = ODP_POOL_INVALID;
    config->num_input_qs = 1;
    config->num_sa_qs = 1;
    config->num_output_qs = 1;
    config->num_thrs = 1;
}
 
static void terminate(int signal ODP_UNUSED)
{
    odp_atomic_store_u32(&prog_conf->is_running, 0U);
}
 
static void parse_interfaces(prog_config_t *config, const char *optarg)
{
    char *tmp_str = strdup(optarg), *tmp;
 
    if (tmp_str == NULL)
        return;
 
    tmp = strtok(tmp_str, DELIMITER);
 
    while (tmp && config->num_ifs < MAX_IFS) {
        config->pktios[config->num_ifs].name = strdup(tmp);
 
        if (config->pktios[config->num_ifs].name != NULL)
            ++config->num_ifs;
 
        tmp = strtok(NULL, DELIMITER);
    }
 
    free(tmp_str);
}
 
static void print_supported_algos(const odp_ipsec_capability_t *ipsec_capa)
{
    int c_cnt, a_cnt;
    const size_t len = ODPH_ARRAY_SIZE(exposed_algs);
 
    printf("                          Cipher algorithms:\n");
 
    for (size_t i = 0U; i < len; ++i) {
        if ((exposed_algs[i].type == CIPHER_TYPE ||
             exposed_algs[i].type == COMB_CIPHER_TYPE) &&
            (ipsec_capa->ciphers.all_bits & (1 << exposed_algs[i].idx)) > 0U) {
            c_cnt = odp_ipsec_cipher_capability(exposed_algs[i].idx, NULL, 0);
 
            if (c_cnt < 0)
                continue;
 
            printf("                              %d: %s",
                   exposed_algs[i].idx, exposed_algs[i].name);
            printf(exposed_algs[i].type == COMB_CIPHER_TYPE ? " (combined)" : "");
 
            odp_ipsec_cipher_capability_t capa[c_cnt];
 
            (void)odp_ipsec_cipher_capability(exposed_algs[i].idx, capa, c_cnt);
 
            for (int j = 0; j < c_cnt; ++j)
                printf(j == 0 ? " (key lengths: %u" : ", %u", capa[j].key_len);
 
            printf(")\n");
        }
    }
 
    printf("                          Authentication algorithms:\n");
 
    for (size_t i = 0U; i < len; ++i) {
        if ((exposed_algs[i].type == AUTH_TYPE ||
             exposed_algs[i].type == COMB_AUTH_TYPE) &&
            (ipsec_capa->auths.all_bits & (1 << exposed_algs[i].idx)) > 0U) {
            a_cnt = odp_ipsec_auth_capability(exposed_algs[i].idx, NULL, 0);
 
            if (a_cnt < 0)
                continue;
 
            printf("                              %d: %s",
                   exposed_algs[i].idx, exposed_algs[i].name);
            printf(exposed_algs[i].type == COMB_AUTH_TYPE ? " (combined)" : "");
 
            odp_ipsec_auth_capability_t capa[a_cnt];
 
            (void)odp_ipsec_auth_capability(exposed_algs[i].idx, capa, a_cnt);
 
            for (int j = 0; j < a_cnt; ++j)
                printf(j == 0 ? " (key/icv lengths: %u/%u" : ", %u/%u",
                       capa[j].key_len, capa[j].icv_len);
 
            printf(")\n");
        }
    }
}
 
static void print_usage(void)
{
    odp_pool_capability_t pool_capa;
    odp_ipsec_capability_t ipsec_capa;
 
    if (odp_pool_capability(&pool_capa) < 0) {
        ODPH_ERR("Error querying pool capabilities\n");
        return;
    }
 
    if (odp_ipsec_capability(&ipsec_capa) < 0) {
        ODPH_ERR("Error querying IPsec capabilities\n");
        return;
    }
 
    printf("\n"
           "Simple IPsec performance tester. Forward and process plain and IPsec packets.\n"
           "\n"
           "Usage: %s OPTIONS\n"
           "\n"
           "  E.g. %s -i ens9f1 -C /etc/odp/ipsecfwd.conf\n"
           "\n"
           "  With ipsecfwd.conf containing, for example:\n"
           "      default: {\n"
           "          dir = 1\n"
           "          proto = 0\n"
           "          mode = 0\n"
           "          crypto: {\n"
           "              cipher_alg = 4\n"
           "              cipher_key = \"jWnZr4t7w!zwC*F-\"\n"
           "              auth_alg = 2\n"
           "              auth_key = \"n2r5u7x!A%%D*\"\n"
           "              icv_len = 12\n"
           "          };\n"
           "      };\n"
           "\n"
           "      sa: (\n"
           "          {\n"
           "              spi = 1337\n"
           "              outbound: {\n"
           "                  tunnel: {\n"
           "                      src_addr = \"192.168.1.10\"\n"
           "                      dst_addr = \"192.168.1.16\"\n"
           "                  };\n"
           "              };\n"
           "          },\n"
           "          {\n"
           "              spi = 1338\n"
           "              outbound: {\n"
           "                  tunnel: {\n"
           "                      src_addr = \"192.168.3.110\"\n"
           "                      dst_addr = \"192.168.3.116\"\n"
           "                  };\n"
           "              };\n"
           "          }\n"
           "      );\n"
           "\n"
           "      fwd: (\n"
           "          {\n"
           "              prefix: \"192.168.1.0/24\"\n"
           "              if: \"ens9f1\"\n"
           "              dst_mac: \"00:00:05:00:07:00\"\n"
           "          },\n"
           "          {\n"
           "              prefix: \"192.1.0.0/16\"\n"
           "              if: \"ens9f0\"\n"
           "              dst_mac: \"00:00:05:00:08:00\"\n"
           "          }\n"
           "      );\n"
           "\n"
           "Mandatory OPTIONS:\n"
           "\n"
           "  -i, --interfaces    Ethernet interfaces for packet I/O, comma-separated,\n"
           "                      no spaces.\n"
           "  -C, --conf          Configuration file. 'libconfig' syntax is expected.\n"
           "                      SA configuration supports default fallback, i.e.\n"
           "                      individual SA configuration blocks may omit some\n"
           "                      parameters and instead set these once in default block\n"
           "                      which then are used to fill missing parameters. The only\n"
           "                      required argument for an SA is the 'spi' parameter.\n"
           "                      Individual SA parameter blocks are expected to be in\n"
           "                      'sa'-named list. Parameter naming follows API\n"
           "                      specification, see 'odp_ipsec_sa_param_t' for parameter\n"
           "                      names and hierarchy. Traffic is mapped to SAs based on UDP\n"
           "                      port: the port is used as the SPI. For forwarding entries,\n"
           "                      individual parameter blocks are similarly expected to be\n"
           "                      in 'fwd'-named list. With forwarding entries, every\n"
           "                      parameter is always required and interfaces present in\n"
           "                      forwarding entries should be one of the interfaces passed\n"
           "                      with '--interfaces' option. The entries are looked up\n"
           "                      in the order they are in the list. See example above for\n"
           "                      potential SA and forwarding configuration.\n"
           "\n"
           "                      Supported cipher and authentication algorithms for SAs:\n",
           PROG_NAME, PROG_NAME);
    print_supported_algos(&ipsec_capa);
    printf("\n"
           "Optional OPTIONS:\n"
           "\n"
           "  -n, --num_pkts      Number of packet buffers allocated for packet I/O pool.\n"
           "                      %u by default.\n"
           "  -l, --pkt_len       Maximum size of packet buffers in packet I/O pool. %u by\n"
           "                      default.\n"
           "  -c, --count         Worker thread count. 1 by default.\n"
           "  -m, --mode          Queueing mode.\n"
           "                          0: ordered (default)\n"
           "                          1: parallel\n"
           "  -I, --num_input_qs  Input queue count. 1 by default.\n"
           "  -S, --num_sa_qs     SA queue count. 1 by default.\n"
           "  -O, --num_output_qs Output queue count. 1 by default.\n"
           "  -d, --direct_rx     Use direct RX. Interfaces will be polled by workers\n"
           "                      directly. '--mode', '--num_input_qs' and '--num_output_qs'\n"
           "                      options are ignored, input and output queue counts will\n"
           "                      match worker count.\n"
           "  -h, --help          This help.\n"
           "\n", pool_capa.pkt.max_num > 0U ? ODPH_MIN(pool_capa.pkt.max_num, PKT_CNT) :
           PKT_CNT, pool_capa.pkt.max_len > 0U ? ODPH_MIN(pool_capa.pkt.max_len, PKT_SIZE) :
           PKT_SIZE);
}
 
static inline odp_ipsec_sa_t *get_in_sa(odp_packet_t pkt)
{
    odph_esphdr_t esp;
    uint32_t spi;
 
    if (!odp_packet_has_ipsec(pkt))
        return NULL;
 
    if (odp_packet_copy_to_mem(pkt, odp_packet_l4_offset(pkt), ODPH_ESPHDR_LEN, &esp) < 0)
        return NULL;
 
    spi = odp_be_to_cpu_32(esp.spi);
 
    return spi <= UINT16_MAX ? spi_to_sa_map[DIR_IN][spi] : NULL;
}
 
static inline int process_ipsec_in_enq(odp_packet_t pkts[], const odp_ipsec_sa_t sas[], int num)
{
    odp_ipsec_in_param_t param;
    int left, sent = 0, ret;
 
    memset(&param, 0, sizeof(param));
    /* IPsec in/out need to be identified somehow, so use user_ptr for this. */
    for (int i = 0; i < num; ++i)
        odp_packet_user_ptr_set(pkts[i], NULL);
 
    while (sent < num) {
        left = num - sent;
        param.num_sa = left;
        param.sa = &sas[sent];
        ret = odp_ipsec_in_enq(&pkts[sent], left, &param);
 
        if (odp_unlikely(ret <= 0))
            break;
 
        sent += ret;
    }
 
    return sent;
}
 
static inline odp_ipsec_sa_t *get_out_sa(odp_packet_t pkt)
{
    odph_udphdr_t udp;
    uint16_t dst_port;
 
    if (!odp_packet_has_udp(pkt))
        return NULL;
 
    if (odp_packet_copy_to_mem(pkt, odp_packet_l4_offset(pkt), ODPH_UDPHDR_LEN, &udp) < 0)
        return NULL;
 
    dst_port = odp_be_to_cpu_16(udp.dst_port);
 
    return dst_port ? spi_to_sa_map[DIR_OUT][dst_port] : NULL;
}
 
static inline int process_ipsec_out_enq(odp_packet_t pkts[], const odp_ipsec_sa_t sas[], int num)
{
    odp_ipsec_out_param_t param;
    int left, sent = 0, ret;
 
    memset(&param, 0, sizeof(param));
    /* IPsec in/out need to be identified somehow, so use user_ptr for this. */
    for (int i = 0; i < num; ++i)
        odp_packet_user_ptr_set(pkts[i], &ipsec_out_mark);
 
    while (sent < num) {
        left = num - sent;
        param.num_sa = left;
        param.sa = &sas[sent];
        ret = odp_ipsec_out_enq(&pkts[sent], left, &param);
 
        if (odp_unlikely(ret <= 0))
            break;
 
        sent += ret;
    }
 
    return sent;
}
 
static inline const fwd_entry_t *get_fwd_entry(lookup_table_t *table, uint32_t ip)
{
    fwd_entry_t *entry;
 
    for (uint32_t i = 0U; i < table->num; ++i) {
        entry = &table->entries[i];
 
        if ((ip & entry->mask) == entry->prefix)
            return entry;
    }
 
    return NULL;
}
 
static inline const pktio_t *lookup_and_apply(odp_packet_t pkt, lookup_table_t *fwd_tbl,
                          uint8_t *q_idx)
{
    const uint32_t l3_off = odp_packet_l3_offset(pkt);
    odph_ipv4hdr_t ipv4;
    uint32_t dst_ip, src_ip;
    const fwd_entry_t *fwd;
    odph_ethhdr_t eth;
 
    if (odp_packet_copy_to_mem(pkt, l3_off, ODPH_IPV4HDR_LEN, &ipv4) < 0)
        return NULL;
 
    dst_ip = odp_be_to_cpu_32(ipv4.dst_addr);
    fwd = get_fwd_entry(fwd_tbl, dst_ip);
 
    if (fwd == NULL)
        return NULL;
 
    if (l3_off != ODPH_ETHHDR_LEN) {
        if (l3_off > ODPH_ETHHDR_LEN) {
            if (odp_packet_pull_head(pkt, l3_off - ODPH_ETHHDR_LEN) == NULL)
                return NULL;
        } else {
            if (odp_packet_push_head(pkt, ODPH_ETHHDR_LEN - l3_off) == NULL)
                return NULL;
        }
    }
 
    eth.dst = fwd->dst_mac;
    eth.src = fwd->pktio->src_mac;
    eth.type = odp_cpu_to_be_16(ODPH_ETHTYPE_IPV4);
 
    if (odp_packet_copy_from_mem(pkt, 0U, ODPH_ETHHDR_LEN, &eth) < 0)
        return NULL;
 
    if (q_idx != NULL) {
        src_ip = odp_be_to_cpu_32(ipv4.src_addr);
        *q_idx = (src_ip ^ dst_ip) % fwd->pktio->num_tx_qs;
    }
 
    return fwd->pktio;
}
 
static inline uint32_t forward_packets(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl)
{
    odp_packet_t pkt;
    odp_bool_t is_hashed_tx = ifs.is_hashed_tx;
    uint8_t q_idx = is_hashed_tx ? 0U : ifs.q_idx, qs_done;
    uint8_t *q_idx_ptr = is_hashed_tx ? &q_idx : NULL;
    const pktio_t *pktio;
    pkt_out_t *out;
    pkt_vec_t *vec;
    uint32_t num_procd = 0U, ret;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
        pktio = lookup_and_apply(pkt, fwd_tbl, q_idx_ptr);
 
        if (pktio == NULL) {
            odp_packet_free(pkt);
            continue;
        }
 
        out = &ifs.ifs[pktio->idx];
        vec = &out->vecs[q_idx];
 
        if (vec->num == 0U)
            out->num_qs++;
 
        vec->pkts[vec->num++] = pkt;
        vec->pktio = pktio;
    }
 
    for (uint32_t i = 0U; i < MAX_IFS; ++i) {
        qs_done = 0U;
        out = &ifs.ifs[i];
 
        for (uint32_t j = 0U; j < MAX_QUEUES && qs_done < out->num_qs; ++j) {
            if (out->vecs[j].num == 0U)
                continue;
 
            vec = &out->vecs[j];
            pktio = vec->pktio;
            ret = pktio->send_fn(pktio, j, vec->pkts, vec->num);
 
            if (odp_unlikely(ret < vec->num))
                odp_packet_free_multi(&vec->pkts[ret], vec->num - ret);
 
            ++qs_done;
            vec->num = 0U;
            num_procd += ret;
        }
 
        out->num_qs = 0U;
    }
 
    return num_procd;
}
 
static inline void process_packets_out_enq(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl,
                       stats_t *stats)
{
    odp_packet_t pkt, pkts_ips[MAX_BURST], pkts_fwd[MAX_BURST];
    odp_ipsec_sa_t *sa, sas[MAX_BURST];
    int num_pkts_ips = 0, num_pkts_fwd = 0, num_procd;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
        sa = get_out_sa(pkt);
 
        if (sa != NULL) {
            sas[num_pkts_ips] = *sa;
            pkts_ips[num_pkts_ips] = pkt;
            ++num_pkts_ips;
        } else {
            pkts_fwd[num_pkts_fwd++] = pkt;
        }
    }
 
    if (num_pkts_ips > 0) {
        num_procd = process_ipsec_out_enq(pkts_ips, sas, num_pkts_ips);
 
        if (odp_unlikely(num_procd < num_pkts_ips)) {
            stats->ipsec_out_errs += num_pkts_ips - num_procd;
            odp_packet_free_multi(&pkts_ips[num_procd], num_pkts_ips - num_procd);
        }
    }
 
    if (num_pkts_fwd > 0) {
        num_procd = forward_packets(pkts_fwd, num_pkts_fwd, fwd_tbl);
        stats->discards += num_pkts_fwd - num_procd;
        stats->fwd_pkts += num_procd;
    }
}
 
static void process_packets_in_enq(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl,
                   stats_t *stats)
{
    odp_packet_t pkt, pkts_ips[MAX_BURST], pkts_out[MAX_BURST];
    odp_ipsec_sa_t *sa, sas[MAX_BURST];
    int num_pkts_ips = 0, num_pkts_out = 0, num_procd;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
 
        if (odp_unlikely(odp_packet_has_error(pkt))) {
            ++stats->discards;
            odp_packet_free(pkt);
            continue;
        }
 
        sa = get_in_sa(pkt);
 
        if (sa != NULL) {
            sas[num_pkts_ips] = *sa;
            pkts_ips[num_pkts_ips] = pkt;
            ++num_pkts_ips;
        } else {
            pkts_out[num_pkts_out++] = pkt;
        }
    }
 
    if (num_pkts_ips > 0) {
        num_procd = process_ipsec_in_enq(pkts_ips, sas, num_pkts_ips);
 
        if (odp_unlikely(num_procd < num_pkts_ips)) {
            stats->ipsec_in_errs += num_pkts_ips - num_procd;
            odp_packet_free_multi(&pkts_ips[num_procd], num_pkts_ips - num_procd);
        }
    }
 
    if (num_pkts_out > 0)
        process_packets_out_enq(pkts_out, num_pkts_out, fwd_tbl, stats);
}
 
static inline odp_bool_t is_ipsec_in(odp_packet_t pkt)
{
    return odp_packet_user_ptr(pkt) == NULL;
}
 
static void complete_ipsec_ops(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl,
                   stats_t *stats)
{
    odp_packet_t pkt, pkts_out[MAX_BURST], pkts_fwd[MAX_BURST];
    odp_bool_t is_in;
    odp_ipsec_packet_result_t result;
    int num_pkts_out = 0, num_pkts_fwd = 0, num_procd;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
        is_in = is_ipsec_in(pkt);
 
        if (odp_unlikely(odp_ipsec_result(&result, pkt) < 0)) {
            is_in ? ++stats->ipsec_in_errs : ++stats->ipsec_out_errs;
            odp_packet_free(pkt);
            continue;
        }
 
        if (odp_unlikely(result.status.all != ODP_IPSEC_OK)) {
            is_in ? ++stats->ipsec_in_errs : ++stats->ipsec_out_errs;
            odp_packet_free(pkt);
            continue;
        }
 
        if (is_in) {
            ++stats->ipsec_in_pkts;
            pkts_out[num_pkts_out++] = pkt;
        } else {
            ++stats->ipsec_out_pkts;
            pkts_fwd[num_pkts_fwd++] = pkt;
        }
    }
 
    if (num_pkts_out > 0)
        process_packets_out_enq(pkts_out, num_pkts_out, fwd_tbl, stats);
 
    if (num_pkts_fwd > 0) {
        num_procd = forward_packets(pkts_fwd, num_pkts_fwd, fwd_tbl);
        stats->discards += num_pkts_fwd - num_procd;
        stats->fwd_pkts += num_procd;
    }
}
 
static void drain_scheduler(prog_config_t *config ODP_UNUSED)
{
    odp_event_t ev;
 
    while (true) {
        ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT);
 
        if (ev == ODP_EVENT_INVALID)
            break;
 
        odp_event_free(ev);
    }
}
 
static inline int process_ipsec_in(odp_packet_t pkts[], const odp_ipsec_sa_t sas[], int num,
                   odp_packet_t pkts_out[])
{
    odp_ipsec_in_param_t param;
    int left, sent = 0, num_out, ret;
 
    memset(&param, 0, sizeof(param));
 
    while (sent < num) {
        left = num - sent;
        num_out = left;
        param.num_sa = left;
        param.sa = &sas[sent];
        ret = odp_ipsec_in(&pkts[sent], left, &pkts_out[sent], &num_out, &param);
 
        if (odp_unlikely(ret <= 0))
            break;
 
        sent += ret;
    }
 
    return sent;
}
 
static inline int process_ipsec_out(odp_packet_t pkts[], const odp_ipsec_sa_t sas[], int num,
                    odp_packet_t pkts_out[])
{
    odp_ipsec_out_param_t param;
    int left, sent = 0, num_out, ret;
 
    memset(&param, 0, sizeof(param));
 
    while (sent < num) {
        left = num - sent;
        num_out = left;
        param.num_sa = left;
        param.sa = &sas[sent];
        ret = odp_ipsec_out(&pkts[sent], left, &pkts_out[sent], &num_out, &param);
 
        if (odp_unlikely(ret <= 0))
            break;
 
        sent += ret;
    }
 
    return sent;
}
 
static inline void process_packets_out(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl,
                       stats_t *stats)
{
    odp_packet_t pkt, pkts_ips[MAX_BURST], pkts_fwd[MAX_BURST], pkts_ips_out[MAX_BURST];
    odp_ipsec_sa_t *sa, sas[MAX_BURST];
    int num_pkts_ips = 0, num_pkts_fwd = 0, num_procd;
    odp_ipsec_packet_result_t result;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
        sa = get_out_sa(pkt);
 
        if (sa != NULL) {
            sas[num_pkts_ips] = *sa;
            pkts_ips[num_pkts_ips] = pkt;
            ++num_pkts_ips;
        } else {
            pkts_fwd[num_pkts_fwd++] = pkt;
        }
    }
 
    if (num_pkts_ips > 0) {
        num_procd = process_ipsec_out(pkts_ips, sas, num_pkts_ips, pkts_ips_out);
 
        if (odp_unlikely(num_procd < num_pkts_ips)) {
            stats->ipsec_out_errs += num_pkts_ips - num_procd;
            odp_packet_free_multi(&pkts_ips[num_procd], num_pkts_ips - num_procd);
        }
 
        for (int i = 0; i < num_procd; ++i) {
            pkt = pkts_ips_out[i];
 
            if (odp_unlikely(odp_ipsec_result(&result, pkt) < 0)) {
                ++stats->ipsec_out_errs;
                odp_packet_free(pkt);
                continue;
            }
 
            if (odp_unlikely(result.status.all != ODP_IPSEC_OK)) {
                ++stats->ipsec_out_errs;
                odp_packet_free(pkt);
                continue;
            }
 
            ++stats->ipsec_out_pkts;
            pkts_fwd[num_pkts_fwd++] = pkt;
        }
    }
 
    if (num_pkts_fwd > 0) {
        num_procd = forward_packets(pkts_fwd, num_pkts_fwd, fwd_tbl);
        stats->discards += num_pkts_fwd - num_procd;
        stats->fwd_pkts += num_procd;
    }
}
 
static void process_packets_in(odp_packet_t pkts[], int num, lookup_table_t *fwd_tbl,
                   stats_t *stats)
{
    odp_packet_t pkt, pkts_ips[MAX_BURST], pkts_out[MAX_BURST], pkts_ips_out[MAX_BURST];
    odp_ipsec_sa_t *sa, sas[MAX_BURST];
    int num_pkts_ips = 0, num_pkts_out = 0, num_procd;
    odp_ipsec_packet_result_t result;
 
    for (int i = 0; i < num; ++i) {
        pkt = pkts[i];
 
        if (odp_unlikely(odp_packet_has_error(pkt))) {
            ++stats->discards;
            odp_packet_free(pkt);
            continue;
        }
 
        sa = get_in_sa(pkt);
 
        if (sa != NULL) {
            sas[num_pkts_ips] = *sa;
            pkts_ips[num_pkts_ips] = pkt;
            ++num_pkts_ips;
        } else {
            pkts_out[num_pkts_out++] = pkt;
        }
    }
 
    if (num_pkts_ips > 0) {
        num_procd = process_ipsec_in(pkts_ips, sas, num_pkts_ips, pkts_ips_out);
 
        if (odp_unlikely(num_procd < num_pkts_ips)) {
            stats->ipsec_in_errs += num_pkts_ips - num_procd;
            odp_packet_free_multi(&pkts_ips[num_procd], num_pkts_ips - num_procd);
        }
 
        for (int i = 0; i < num_procd; ++i) {
            pkt = pkts_ips_out[i];
 
            if (odp_unlikely(odp_ipsec_result(&result, pkt) < 0)) {
                ++stats->ipsec_in_errs;
                odp_packet_free(pkt);
                continue;
            }
 
            if (odp_unlikely(result.status.all != ODP_IPSEC_OK)) {
                ++stats->ipsec_in_errs;
                odp_packet_free(pkt);
                continue;
            }
 
            ++stats->ipsec_in_pkts;
            pkts_out[num_pkts_out++] = pkt;
        }
    }
 
    if (num_pkts_out > 0)
        process_packets_out(pkts_out, num_pkts_out, fwd_tbl, stats);
}
 
static void drain_direct_inputs(prog_config_t *config)
{
    odp_packet_t pkt;
 
    for (uint32_t i = 0U; i < config->num_ifs; ++i) {
        for (uint32_t j = 0U; j < config->num_input_qs; ++j) {
            while (odp_pktin_recv(config->pktios[i].in_dir_qs[j], &pkt, 1) == 1)
                odp_packet_free(pkt);
        }
    }
}
 
static odp_bool_t setup_ipsec(prog_config_t *config)
{
    odp_queue_param_t q_param;
    odp_ipsec_config_t ipsec_config;
    char q_name[ODP_QUEUE_NAME_LEN];
 
    if (!config->is_dir_rx) {
        snprintf(q_name, sizeof(q_name), SHORT_PROG_NAME "_sa_status");
        odp_queue_param_init(&q_param);
        q_param.type = ODP_QUEUE_TYPE_SCHED;
        q_param.sched.prio = odp_schedule_default_prio();
        q_param.sched.sync = ODP_SCHED_SYNC_PARALLEL;
        q_param.sched.group = ODP_SCHED_GROUP_ALL;
        config->compl_q = odp_queue_create(q_name, &q_param);
 
        if (config->compl_q == ODP_QUEUE_INVALID) {
            ODPH_ERR("Error creating IPsec completion queue\n");
            return false;
        }
    }
 
    odp_ipsec_config_init(&ipsec_config);
 
    if (!config->is_dir_rx) {
        ipsec_config.inbound_mode = ODP_IPSEC_OP_MODE_ASYNC;
        ipsec_config.outbound_mode = ODP_IPSEC_OP_MODE_ASYNC;
        config->ops.proc = process_packets_in_enq;
        config->ops.compl = complete_ipsec_ops;
        config->ops.drain = drain_scheduler;
    } else {
        ipsec_config.inbound_mode = ODP_IPSEC_OP_MODE_SYNC;
        ipsec_config.outbound_mode = ODP_IPSEC_OP_MODE_SYNC;
        config->ops.proc = process_packets_in;
        config->ops.compl = NULL;
        config->ops.drain = drain_direct_inputs;
    }
 
    ipsec_config.inbound.default_queue = config->compl_q;
    /* For tunnel to tunnel, we need to parse up to this to check the UDP port for SA. */
    ipsec_config.inbound.parse_level = ODP_PROTO_LAYER_L4;
 
    if (odp_ipsec_config(&ipsec_config) < 0) {
        ODPH_ERR("Error configuring IPsec\n");
        return false;
    }
 
    return true;
}
 
static odp_bool_t create_sa_dest_queues(odp_ipsec_capability_t *ipsec_capa,
                    prog_config_t *config)
{
    odp_queue_param_t q_param;
    const uint32_t max_sa_qs = ODPH_MIN(MAX_SA_QUEUES, ipsec_capa->max_queues);
 
    if (config->num_sa_qs == 0U || config->num_sa_qs > max_sa_qs) {
        ODPH_ERR("Invalid number of SA queues: %u (min: 1, max: %u)\n", config->num_sa_qs,
             max_sa_qs);
        config->num_sa_qs = 0U;
        return false;
    }
 
    for (uint32_t i = 0U; i < config->num_sa_qs; ++i) {
        char q_name[ODP_QUEUE_NAME_LEN];
 
        snprintf(q_name, sizeof(q_name), SHORT_PROG_NAME "_sa_compl_%u", i);
        odp_queue_param_init(&q_param);
        q_param.type = ODP_QUEUE_TYPE_SCHED;
        q_param.sched.prio = odp_schedule_max_prio();
        q_param.sched.sync = config->mode == ORDERED ? ODP_SCHED_SYNC_ORDERED :
                                   ODP_SCHED_SYNC_PARALLEL;
        q_param.sched.group = ODP_SCHED_GROUP_ALL;
        config->sa_qs[i] = odp_queue_create(q_name, &q_param);
 
        if (config->sa_qs[i] == ODP_QUEUE_INVALID) {
            ODPH_ERR("Error creating SA destination queue (created count: %u)\n", i);
            config->num_sa_qs = i;
            return false;
        }
    }
 
    return true;
}
 
static void parse_crypto(config_setting_t *cfg, sa_config_t *config)
{
    int val;
    const char *val_str;
    config_setting_t *cs = config_setting_lookup(cfg, "crypto");
 
    if (cs == NULL)
        return;
 
    if (config_setting_lookup_int(cs, "cipher_alg", &val) == CONFIG_TRUE)
        config->sa_param.crypto.cipher_alg = val;
 
    if (config_setting_lookup_string(cs, "cipher_key", &val_str) == CONFIG_TRUE) {
        strcpy(config->cipher_key, val_str);
        config->sa_param.crypto.cipher_key.data = (uint8_t *)config->cipher_key;
        config->sa_param.crypto.cipher_key.length =
            strlen((const char *)config->cipher_key);
    }
 
    if (config_setting_lookup_string(cs, "cipher_key_extra", &val_str) == CONFIG_TRUE) {
        strcpy(config->cipher_key_extra, val_str);
        config->sa_param.crypto.cipher_key_extra.data =
            (uint8_t *)config->cipher_key_extra;
        config->sa_param.crypto.cipher_key_extra.length =
            strlen((const char *)config->cipher_key_extra);
    }
 
    if (config_setting_lookup_int(cs, "auth_alg", &val) == CONFIG_TRUE)
        config->sa_param.crypto.auth_alg = val;
 
    if (config_setting_lookup_string(cs, "auth_key", &val_str) == CONFIG_TRUE) {
        strcpy(config->auth_key, val_str);
        config->sa_param.crypto.auth_key.data = (uint8_t *)config->auth_key;
        config->sa_param.crypto.auth_key.length = strlen((const char *)config->auth_key);
    }
 
    if (config_setting_lookup_string(cs, "auth_key_extra", &val_str) == CONFIG_TRUE) {
        strcpy(config->auth_key_extra, val_str);
        config->sa_param.crypto.auth_key_extra.data = (uint8_t *)config->auth_key_extra;
        config->sa_param.crypto.auth_key_extra.length =
            strlen((const char *)config->auth_key_extra);
    }
 
    if (config_setting_lookup_int(cs, "icv_len", &val) == CONFIG_TRUE)
        config->sa_param.crypto.icv_len = val;
}
 
static void parse_opt(config_setting_t *cfg, sa_config_t *config)
{
    int val;
    config_setting_t *cs = config_setting_lookup(cfg, "opt");
 
    if (cs == NULL)
        return;
 
    if (config_setting_lookup_int(cs, "esn", &val) == CONFIG_TRUE)
        config->sa_param.opt.esn = val;
 
    if (config_setting_lookup_int(cs, "udp_encap", &val) == CONFIG_TRUE)
        config->sa_param.opt.udp_encap = val;
 
    if (config_setting_lookup_int(cs, "copy_dscp", &val) == CONFIG_TRUE)
        config->sa_param.opt.copy_dscp = val;
 
    if (config_setting_lookup_int(cs, "copy_flabel", &val) == CONFIG_TRUE)
        config->sa_param.opt.copy_flabel = val;
 
    if (config_setting_lookup_int(cs, "copy_df", &val) == CONFIG_TRUE)
        config->sa_param.opt.copy_df = val;
 
    if (config_setting_lookup_int(cs, "dec_ttl", &val) == CONFIG_TRUE)
        config->sa_param.opt.dec_ttl = val;
}
 
static void parse_limits(config_setting_t *cfg, sa_config_t *config)
{
    config_setting_t *cs = config_setting_lookup(cfg, "lifetime"), *soft, *hard;
    long long val;
 
    if (cs == NULL)
        return;
 
    soft = config_setting_lookup(cs, "soft_limit");
    hard = config_setting_lookup(cs, "hard_limit");
 
    if (soft != NULL) {
        if (config_setting_lookup_int64(soft, "bytes", &val) == CONFIG_TRUE)
            config->sa_param.lifetime.soft_limit.bytes = val;
 
        if (config_setting_lookup_int64(soft, "packets", &val) == CONFIG_TRUE)
            config->sa_param.lifetime.soft_limit.packets = val;
    }
 
    if (hard != NULL) {
        if (config_setting_lookup_int64(hard, "bytes", &val) == CONFIG_TRUE)
            config->sa_param.lifetime.hard_limit.bytes = val;
 
        if (config_setting_lookup_int64(hard, "packets", &val) == CONFIG_TRUE)
            config->sa_param.lifetime.hard_limit.packets = val;
    }
}
 
static void parse_inbound(config_setting_t *cfg, sa_config_t *config)
{
    config_setting_t *cs = config_setting_lookup(cfg, "inbound");
    int val;
    const char *val_str;
 
    if (cs == NULL)
        return;
 
    if (config_setting_lookup_int(cs, "lookup_mode", &val) == CONFIG_TRUE)
        config->sa_param.inbound.lookup_mode = val;
 
    if (config_setting_lookup_string(cs, "lookup_dst_addr", &val_str) == CONFIG_TRUE) {
        if (odph_ipv4_addr_parse(&config->lkp_dst_ip, val_str) == 0) {
            config->lkp_dst_ip = odp_cpu_to_be_32(config->lkp_dst_ip);
            config->sa_param.inbound.lookup_param.dst_addr = &config->lkp_dst_ip;
        }
    }
 
    if (config_setting_lookup_int(cs, "antireplay_ws", &val) == CONFIG_TRUE)
        config->sa_param.inbound.antireplay_ws = val;
 
    if (config_setting_lookup_int(cs, "reassembly_en", &val) == CONFIG_TRUE)
        config->sa_param.inbound.reassembly_en = val;
}
 
static void parse_outbound(config_setting_t *cfg, sa_config_t *config)
{
    config_setting_t *cs = config_setting_lookup(cfg, "outbound"), *tunnel;
    const char *val_str;
    int val;
 
    if (cs == NULL)
        return;
 
    tunnel = config_setting_lookup(cs, "tunnel");
 
    if (tunnel != NULL) {
        if (config_setting_lookup_string(tunnel, "src_addr", &val_str) == CONFIG_TRUE) {
            if (odph_ipv4_addr_parse(&config->src_ip, val_str) == 0) {
                config->src_ip = odp_cpu_to_be_32(config->src_ip);
                config->sa_param.outbound.tunnel.ipv4.src_addr = &config->src_ip;
            }
        }
 
        if (config_setting_lookup_string(tunnel, "dst_addr", &val_str) == CONFIG_TRUE) {
            if (odph_ipv4_addr_parse(&config->dst_ip, val_str) == 0) {
                config->dst_ip = odp_cpu_to_be_32(config->dst_ip);
                config->sa_param.outbound.tunnel.ipv4.dst_addr = &config->dst_ip;
            }
        }
 
        if (config_setting_lookup_int(tunnel, "dscp", &val) == CONFIG_TRUE)
            config->sa_param.outbound.tunnel.ipv4.dscp = val;
 
        if (config_setting_lookup_int(tunnel, "df", &val) == CONFIG_TRUE)
            config->sa_param.outbound.tunnel.ipv4.df = val;
 
        if (config_setting_lookup_int(tunnel, "ttl", &val) == CONFIG_TRUE)
            config->sa_param.outbound.tunnel.ipv4.ttl = val;
    }
 
    if (config_setting_lookup_int(cs, "frag_mode", &val) == CONFIG_TRUE)
        config->sa_param.outbound.frag_mode = val;
 
    if (config_setting_lookup_int(cs, "mtu", &val) == CONFIG_TRUE)
        config->sa_param.outbound.mtu = val;
}
 
static void parse_sa_entry(config_setting_t *cfg, sa_config_t *config)
{
    int val;
 
    if (config_setting_lookup_int(cfg, "dir", &val) == CONFIG_TRUE)
        config->sa_param.dir = val;
 
    if (config_setting_lookup_int(cfg, "proto", &val) == CONFIG_TRUE)
        config->sa_param.proto = val;
 
    if (config_setting_lookup_int(cfg, "mode", &val) == CONFIG_TRUE)
        config->sa_param.mode = val;
 
    if (config_setting_lookup_int(cfg, "spi", &val) == CONFIG_TRUE)
        config->sa_param.spi = val;
 
    parse_crypto(cfg, config);
    parse_opt(cfg, config);
    parse_limits(cfg, config);
    parse_inbound(cfg, config);
    parse_outbound(cfg, config);
}
 
static void create_sa_entry(odp_ipsec_sa_param_t *sa_param, prog_config_t *config,
                uint32_t max_num_sa)
{
    uint32_t dir = sa_param->dir;
    uint32_t spi = sa_param->spi;
    odp_ipsec_sa_t sa;
 
    if (config->num_sas == max_num_sa) {
        ODPH_ERR("Maximum number of SAs parsed (%u), ignoring rest\n", max_num_sa);
        return;
    }
 
    if (spi > UINT16_MAX) {
        ODPH_ERR("Unsupported SPI value for SA %u (> %u)\n", spi, UINT16_MAX);
        return;
    }
 
    if (spi_to_sa_map[dir][spi] != NULL) {
        ODPH_ERR("Non-unique SPIs not supported for SA %u\n", spi);
        return;
    }
 
    sa_param->dest_queue = config->sa_qs[config->num_sas % config->num_sa_qs];
    sa = odp_ipsec_sa_create(sa_param);
 
    if (sa == ODP_IPSEC_SA_INVALID) {
        ODPH_ERR("Error creating SA handle for SA %u\n", spi);
        return;
    }
 
    config->sas[config->num_sas] = sa;
    spi_to_sa_map[dir][spi] = &config->sas[config->num_sas];
    ++config->num_sas;
}
 
static void parse_and_create_sa_entries(config_t *cfg, prog_config_t *config, uint32_t max_num_sa)
{
    config_setting_t *cs;
    int count;
 
    cs = config_lookup(cfg, "default");
 
    if (cs != NULL)
        parse_sa_entry(cs, &config->default_cfg);
 
    cs = config_lookup(cfg, "sa");
 
    if (cs == NULL)
        return;
 
    count = config_setting_length(cs);
 
    for (int i = 0; i < count; i++) {
        sa_config_t sa_cfg;
        config_setting_t *sa;
        int val;
 
        sa_cfg = config->default_cfg;
        sa = config_setting_get_elem(cs, i);
 
        if (sa == NULL)
            continue;
 
        if (config_setting_lookup_int(sa, "spi", &val) == CONFIG_TRUE) {
            parse_sa_entry(sa, &sa_cfg);
            create_sa_entry(&sa_cfg.sa_param, config, max_num_sa);
        }
    }
}
 
static void parse_sas(config_t *cfg, prog_config_t *config)
{
    odp_ipsec_capability_t ipsec_capa;
    uint32_t max_num_sa;
 
    if (odp_ipsec_capability(&ipsec_capa) < 0) {
        ODPH_ERR("Error querying IPsec capabilities\n");
        return;
    }
 
    if (!setup_ipsec(config))
        return;
 
    if (!config->is_dir_rx && !create_sa_dest_queues(&ipsec_capa, config))
        return;
 
    max_num_sa = ODPH_MIN(MAX_SAS, ipsec_capa.max_num_sa);
    parse_and_create_sa_entries(cfg, config, max_num_sa);
}
 
static const pktio_t *get_pktio(const char *iface, const prog_config_t *config)
{
    for (uint32_t i = 0U; i < config->num_ifs; ++i) {
        if (strcmp(iface, config->pktios[i].name) == 0)
            return &config->pktios[i];
    }
 
    return NULL;
}
 
static void create_fwd_table_entry(config_setting_t *cfg, prog_config_t *config)
{
    const char *val_str;
    char dst_ip_str[16U] = { 0 };
    uint32_t mask, dst_ip;
    odph_ethaddr_t dst_mac;
    const pktio_t *pktio = NULL;
    fwd_entry_t *entry;
 
    if (config->fwd_tbl.num == MAX_FWDS) {
        ODPH_ERR("Maximum number of forwarding entries parsed (%u), ignoring rest\n",
             MAX_FWDS);
        return;
    }
 
    if (config_setting_lookup_string(cfg, "prefix", &val_str) == CONFIG_TRUE) {
        if (sscanf(val_str, "%[^/]/%u", dst_ip_str, &mask) != 2) {
            ODPH_ERR("Error parsing IP and subnet mask for forwarding entry\n");
            return;
        }
 
        if (odph_ipv4_addr_parse(&dst_ip, dst_ip_str) < 0) {
            ODPH_ERR("Syntax error in IP address for forwarding entry\n");
            return;
        }
 
        if (mask > IP_ADDR_LEN) {
            ODPH_ERR("Invalid subnet mask for forwarding entry: %u\n", mask);
            return;
        }
    } else {
        return;
    }
 
    if (config_setting_lookup_string(cfg, "if", &val_str) == CONFIG_TRUE) {
        pktio = get_pktio(val_str, config);
 
        if (pktio == NULL) {
            ODPH_ERR("Error parsing next interface for forwarding entry\n");
            return;
        }
    } else {
        return;
    }
 
    if (config_setting_lookup_string(cfg, "dst_mac", &val_str) == CONFIG_TRUE) {
        if (odph_eth_addr_parse(&dst_mac, val_str) < 0) {
            ODPH_ERR("Syntax error in destination MAC for forwarding entry\n");
            return;
        }
    } else {
        return;
    }
 
    mask = mask > 0U ? 0xFFFFFFFF << (IP_ADDR_LEN - mask) : 0U;
    entry = &config->fwd_tbl.entries[config->fwd_tbl.num];
    entry->prefix = dst_ip & mask;
    entry->mask = mask;
    entry->dst_mac = dst_mac;
    entry->pktio = pktio;
    ++config->fwd_tbl.num;
}
 
static void parse_fwd_table(config_t *cfg, prog_config_t *config)
{
    config_setting_t *cs;
    int count;
 
    cs = config_lookup(cfg, "fwd");
 
    if (cs == NULL)
        return;
 
    count = config_setting_length(cs);
 
    for (int i = 0; i < count; i++) {
        config_setting_t *fwd = config_setting_get_elem(cs, i);
 
        if (fwd == NULL)
            continue;
 
        create_fwd_table_entry(fwd, config);
    }
}
 
static parse_result_t check_options(prog_config_t *config)
{
    odp_pool_capability_t pool_capa;
 
    if (odp_pool_capability(&pool_capa) < 0) {
        ODPH_ERR("Error querying pool capabilities\n");
        return PRS_NOK;
    }
 
    if (config->num_ifs == 0U) {
        ODPH_ERR("Invalid number of interfaces: %u (min: 1, max: %u)\n", config->num_ifs,
             MAX_IFS);
        return PRS_NOK;
    }
 
    if (config->fwd_tbl.num == 0U) {
        ODPH_ERR("Invalid number of forwarding entries: %u (min: 1, max: %u)\n",
             config->fwd_tbl.num, MAX_FWDS);
        return PRS_NOK;
    }
 
    if (pool_capa.pkt.max_num > 0U && config->num_pkts > pool_capa.pkt.max_num) {
        ODPH_ERR("Invalid pool packet count: %u (max: %u)\n", config->num_pkts,
             pool_capa.pkt.max_num);
        return PRS_NOK;
    }
 
    if (config->num_pkts == 0U)
        config->num_pkts = pool_capa.pkt.max_num > 0U ?
                    ODPH_MIN(pool_capa.pkt.max_num, PKT_CNT) : PKT_CNT;
 
    if (pool_capa.pkt.max_len > 0U && config->pkt_len > pool_capa.pkt.max_len) {
        ODPH_ERR("Invalid pool packet length: %u (max: %u)\n", config->pkt_len,
             pool_capa.pkt.max_len);
        return PRS_NOK;
    }
 
    if (config->pkt_len == 0U)
        config->pkt_len = pool_capa.pkt.max_len > 0U ?
                    ODPH_MIN(pool_capa.pkt.max_len, PKT_SIZE) : PKT_SIZE;
 
    if (config->num_thrs <= 0 || config->num_thrs > MAX_WORKERS) {
        ODPH_ERR("Invalid thread count: %d (min: 1, max: %d)\n", config->num_thrs,
             MAX_WORKERS);
        return PRS_NOK;
    }
 
    if (config->is_dir_rx) {
        config->num_input_qs = config->num_thrs;
        config->num_output_qs = config->num_thrs;
    }
 
    return PRS_OK;
}
 
static parse_result_t parse_options(int argc, char **argv, prog_config_t *config)
{
    int opt;
    config_t cfg;
 
    static const struct option longopts[] = {
        { "interfaces", required_argument, NULL, 'i' },
        { "num_pkts", required_argument, NULL, 'n' },
        { "pkt_len", required_argument, NULL, 'l' },
        { "count", required_argument, NULL, 'c' },
        { "mode", required_argument, NULL, 'm' },
        { "conf", required_argument, NULL, 'C' },
        { "num_input_qs", required_argument, NULL, 'I' },
        { "num_sa_qs", required_argument, NULL, 'S' },
        { "num_output_qs", required_argument, NULL, 'O' },
        { "direct_rx", no_argument, NULL, 'd' },
        { "help", no_argument, NULL, 'h' },
        { NULL, 0, NULL, 0 }
    };
 
    static const char *shortopts = "i:n:l:c:m:C:I:S:O:dh";
 
    while (true) {
        opt = getopt_long(argc, argv, shortopts, longopts, NULL);
 
        if (opt == -1)
            break;
 
        switch (opt) {
        case 'i':
            parse_interfaces(config, optarg);
            break;
        case 'n':
            config->num_pkts = atoi(optarg);
            break;
        case 'l':
            config->pkt_len = atoi(optarg);
            break;
        case 'c':
            config->num_thrs = atoi(optarg);
            break;
        case 'm':
            config->mode = !!atoi(optarg);
            break;
        case 'C':
            config->conf_file = strdup(optarg);
            break;
        case 'I':
            config->num_input_qs = atoi(optarg);
            break;
        case 'S':
            config->num_sa_qs = atoi(optarg);
            break;
        case 'O':
            config->num_output_qs = atoi(optarg);
            break;
        case 'd':
            config->is_dir_rx = true;
            break;
        case 'h':
            print_usage();
            return PRS_TERM;
        case '?':
        default:
            print_usage();
            return PRS_NOK;
        }
    }
 
    config_init(&cfg);
 
    if (config_read_file(&cfg, config->conf_file) == CONFIG_FALSE) {
        ODPH_ERR("Error opening SA configuration file: %s\n",  config_error_text(&cfg));
        config_destroy(&cfg);
        return PRS_NOK;
    }
 
    parse_sas(&cfg, config);
    parse_fwd_table(&cfg, config);
    config_destroy(&cfg);
 
    return check_options(config);
}
 
static parse_result_t setup_program(int argc, char **argv, prog_config_t *config)
{
    struct sigaction action = { .sa_handler = terminate };
 
    if (sigemptyset(&action.sa_mask) == -1 || sigaddset(&action.sa_mask, SIGINT) == -1 ||
        sigaddset(&action.sa_mask, SIGTERM) == -1 ||
        sigaddset(&action.sa_mask, SIGHUP) == -1 || sigaction(SIGINT, &action, NULL) == -1 ||
        sigaction(SIGTERM, &action, NULL) == -1 || sigaction(SIGHUP, &action, NULL) == -1) {
        ODPH_ERR("Error installing signal handler\n");
        return PRS_NOK;
    }
 
    return parse_options(argc, argv, config);
}
 
static uint32_t schedule(thread_config_t *config ODP_UNUSED, odp_event_t evs[], int num)
{
    return odp_schedule_multi_no_wait(NULL, evs, num);
}
 
static uint32_t recv(thread_config_t *config, odp_event_t evs[], int num)
{
    prog_config_t *prog_config = config->prog_config;
    pktio_t *pktio = &prog_config->pktios[config->pktio++ % prog_config->num_ifs];
    odp_pktin_queue_t in_q = pktio->in_dir_qs[config->thr_idx % prog_config->num_input_qs];
    odp_packet_t pkts[num];
    int ret;
 
    ret = odp_pktin_recv(in_q, pkts, num);
 
    if (odp_unlikely(ret <= 0))
        return 0U;
 
    odp_packet_to_event_multi(pkts, evs, ret);
 
    return ret;
}
 
static uint32_t send(const pktio_t *pktio, uint8_t index, odp_packet_t pkts[], int num)
{
    int ret = odp_pktout_send(pktio->out_dir_qs[index], pkts, num);
 
    return ret < 0 ? 0U : (uint32_t)ret;
}
 
static uint32_t enqueue(const pktio_t *pktio, uint8_t index, odp_packet_t pkts[], int num)
{
    odp_event_t evs[MAX_BURST];
    int ret;
 
    odp_packet_to_event_multi(pkts, evs, num);
 
    ret = odp_queue_enq_multi(pktio->out_ev_qs[index], evs, num);
 
    return ret < 0 ? 0U : (uint32_t)ret;
}
 
static odp_bool_t setup_pktios(prog_config_t *config)
{
    odp_pool_param_t pool_param;
    pktio_t *pktio;
    odp_pktio_param_t pktio_param;
    odp_pktin_queue_param_t pktin_param;
    odp_pktio_capability_t capa;
    odp_pktout_queue_param_t pktout_param;
    odp_pktio_config_t pktio_config;
    uint32_t max_output_qs;
 
    odp_pool_param_init(&pool_param);
    pool_param.pkt.seg_len = config->pkt_len;
    pool_param.pkt.len = config->pkt_len;
    pool_param.pkt.num = config->num_pkts;
    pool_param.type = ODP_POOL_PACKET;
    config->pktio_pool = odp_pool_create(PROG_NAME, &pool_param);
 
    if (config->pktio_pool == ODP_POOL_INVALID) {
        ODPH_ERR("Error creating packet I/O pool\n");
        return false;
    }
 
    config->ops.rx = !config->is_dir_rx ? schedule : recv;
    config->is_hashed_tx = !config->is_dir_rx && config->mode == ORDERED;
 
    for (uint32_t i = 0U; i < config->num_ifs; ++i) {
        pktio = &config->pktios[i];
        pktio->idx = i;
        odp_pktio_param_init(&pktio_param);
        pktio_param.in_mode = !config->is_dir_rx ?
            ODP_PKTIN_MODE_SCHED : ODP_PKTIN_MODE_DIRECT;
        pktio_param.out_mode = config->is_hashed_tx ?
            ODP_PKTOUT_MODE_QUEUE : ODP_PKTOUT_MODE_DIRECT;
        pktio->handle = odp_pktio_open(pktio->name, config->pktio_pool, &pktio_param);
 
        if (pktio->handle == ODP_PKTIO_INVALID) {
            ODPH_ERR("Error opening packet I/O (%s)\n", pktio->name);
            return false;
        }
 
        if (odp_pktio_capability(pktio->handle, &capa) < 0) {
            ODPH_ERR("Error querying packet I/O capabilities (%s)\n", pktio->name);
            return false;
        }
 
        if (config->num_input_qs == 0U || config->num_input_qs > capa.max_input_queues) {
            ODPH_ERR("Invalid number of input queues for packet I/O: %u (min: 1, max: "
                 "%u) (%s)\n", config->num_input_qs, capa.max_input_queues,
                 pktio->name);
            return false;
        }
 
        max_output_qs = ODPH_MIN(MAX_QUEUES, capa.max_output_queues);
 
        if (config->num_output_qs == 0U || config->num_output_qs > max_output_qs) {
            ODPH_ERR("Invalid number of output queues for packet I/O: %u (min: 1, "
                 "max: %u) (%s)\n", config->num_output_qs, max_output_qs,
                 pktio->name);
            return false;
        }
 
        odp_pktin_queue_param_init(&pktin_param);
 
        if (config->is_hashed_tx)
            pktin_param.queue_param.sched.sync = ODP_SCHED_SYNC_ORDERED;
 
        if (config->num_input_qs > 1U) {
            pktin_param.hash_enable = true;
            pktin_param.hash_proto.proto.ipv4_udp = 1U;
            pktin_param.num_queues = config->num_input_qs;
        }
 
        pktin_param.op_mode = (config->is_dir_rx &&
                       config->num_thrs > (int)config->num_input_qs) ?
                        ODP_PKTIO_OP_MT : ODP_PKTIO_OP_MT_UNSAFE;
 
        if (odp_pktin_queue_config(pktio->handle, &pktin_param) < 0) {
            ODPH_ERR("Error configuring packet I/O input queues (%s)\n", pktio->name);
            return false;
        }
 
        if (config->is_dir_rx) {
            if (odp_pktin_queue(pktio->handle, pktio->in_dir_qs, config->num_input_qs)
                != (int)config->num_input_qs) {
                ODPH_ERR("Error querying packet I/O input queue (%s)\n",
                     pktio->name);
                return false;
            }
        }
 
        pktio->send_fn = config->is_hashed_tx ? enqueue : send;
        pktio->num_tx_qs = config->num_output_qs;
        odp_pktout_queue_param_init(&pktout_param);
        pktout_param.num_queues = pktio->num_tx_qs;
 
        if (!config->is_hashed_tx) {
            pktout_param.op_mode = config->num_thrs > (int)pktio->num_tx_qs ?
                ODP_PKTIO_OP_MT : ODP_PKTIO_OP_MT_UNSAFE;
        }
 
        if (odp_pktout_queue_config(pktio->handle, &pktout_param) < 0) {
            ODPH_ERR("Error configuring packet I/O output queues (%s)\n", pktio->name);
            return false;
        }
 
        if (config->is_hashed_tx) {
            if (odp_pktout_event_queue(pktio->handle, pktio->out_ev_qs,
                           pktio->num_tx_qs) != (int)pktio->num_tx_qs) {
                ODPH_ERR("Error querying packet I/O output event queue (%s)\n",
                     pktio->name);
                return false;
            }
        } else {
            if (odp_pktout_queue(pktio->handle, pktio->out_dir_qs, pktio->num_tx_qs)
                != (int)pktio->num_tx_qs) {
                ODPH_ERR("Error querying packet I/O output queue (%s)\n",
                     pktio->name);
                return false;
            }
        }
 
        odp_pktio_config_init(&pktio_config);
 
        if (odp_pktio_config(pktio->handle, &pktio_config) < 0) {
            ODPH_ERR("Error configuring packet I/O extra options (%s)\n", pktio->name);
            return false;
        }
 
        if (odp_pktio_mac_addr(pktio->handle, &pktio->src_mac, sizeof(pktio->src_mac))
            != sizeof(pktio->src_mac)) {
            ODPH_ERR("Error getting packet I/O MAC address (%s)\n", pktio->name);
            return false;
        }
 
        if (odp_pktio_start(pktio->handle) < 0) {
            ODPH_ERR("Error starting packet I/O (%s)\n", pktio->name);
            return false;
        }
    }
 
    return true;
}
 
static inline void check_ipsec_status_ev(odp_event_t ev, stats_t *stats)
{
    odp_ipsec_status_t status;
 
    if (odp_unlikely(odp_ipsec_status(&status, ev) < 0 || status.result < 0))
        ++stats->status_errs;
 
    odp_event_free(ev);
}
 
static int process_packets(void *args)
{
    thread_config_t *config = args;
    int thr_idx = odp_thread_id();
    odp_event_t evs[MAX_BURST], ev;
    ops_t ops = config->prog_config->ops;
    odp_atomic_u32_t *is_running = &config->prog_config->is_running;
    uint32_t cnt;
    odp_event_type_t type;
    odp_event_subtype_t subtype;
    odp_packet_t pkt, pkts_in[MAX_BURST], pkts_ips[MAX_BURST];
    lookup_table_t *fwd_tbl = &config->prog_config->fwd_tbl;
    stats_t *stats = &config->stats;
 
    ifs.is_hashed_tx = config->prog_config->is_hashed_tx;
    ifs.q_idx = thr_idx % config->prog_config->num_output_qs;
    config->thr_idx = thr_idx;
    odp_barrier_wait(&config->prog_config->init_barrier);
 
    while (odp_atomic_load_u32(is_running)) {
        int num_pkts_in = 0, num_pkts_ips = 0;
        /* TODO: Add possibility to configure scheduler and ipsec enq/deq burst sizes. */
        cnt = ops.rx(config, evs, MAX_BURST);
 
        if (cnt == 0U)
            continue;
 
        for (uint32_t i = 0U; i < cnt; ++i) {
            ev = evs[i];
            type = odp_event_types(ev, &subtype);
            pkt = odp_packet_from_event(ev);
 
            if (type == ODP_EVENT_PACKET) {
                if (subtype == ODP_EVENT_PACKET_BASIC) {
                    pkts_in[num_pkts_in++] = pkt;
                } else if (subtype == ODP_EVENT_PACKET_IPSEC) {
                    pkts_ips[num_pkts_ips++] = pkt;
                } else {
                    ++stats->discards;
                    odp_event_free(ev);
                }
            } else if (type == ODP_EVENT_IPSEC_STATUS) {
                check_ipsec_status_ev(ev, stats);
            } else {
                ++stats->discards;
                odp_event_free(ev);
            }
        }
 
        if (num_pkts_in > 0)
            ops.proc(pkts_in, num_pkts_in, fwd_tbl, stats);
 
        if (ops.compl && num_pkts_ips > 0)
            ops.compl(pkts_ips, num_pkts_ips, fwd_tbl, stats);
    }
 
    odp_barrier_wait(&config->prog_config->term_barrier);
    ops.drain(config->prog_config);
 
    return 0;
}
 
static odp_bool_t setup_workers(prog_config_t *config)
{
    odph_thread_common_param_t thr_common;
    odph_thread_param_t thr_param[config->num_thrs];
    odp_cpumask_t cpumask;
    int num_workers;
 
    num_workers = odp_cpumask_default_worker(&cpumask, config->num_thrs);
    odph_thread_common_param_init(&thr_common);
    thr_common.instance = config->odp_instance;
    thr_common.cpumask = &cpumask;
 
    for (int i = 0; i < config->num_thrs; ++i) {
        odph_thread_param_init(&thr_param[i]);
        thr_param[i].start = process_packets;
        thr_param[i].thr_type = ODP_THREAD_WORKER;
        config->thread_config[i].prog_config = config;
        thr_param[i].arg = &config->thread_config[i];
    }
 
    num_workers = odph_thread_create(config->thread_tbl, &thr_common, thr_param, num_workers);
 
    if (num_workers != config->num_thrs) {
        ODPH_ERR("Error configuring worker threads\n");
        return false;
    }
 
    return true;
}
 
static odp_bool_t setup_test(prog_config_t *config)
{
    odp_barrier_init(&config->init_barrier, config->num_thrs + 1);
    odp_barrier_init(&config->term_barrier, config->num_thrs + 1);
 
    if (!setup_pktios(config))
        return false;
 
    if (!setup_workers(config))
        return false;
 
    odp_barrier_wait(&config->init_barrier);
 
    return true;
}
 
static void stop_test(prog_config_t *config)
{
    for (uint32_t i = 0U; i < config->num_ifs; ++i)
        if (config->pktios[i].handle != ODP_PKTIO_INVALID)
            (void)odp_pktio_stop(config->pktios[i].handle);
 
    odp_barrier_wait(&config->term_barrier);
    (void)odph_thread_join(config->thread_tbl, config->num_thrs);
}
 
static void print_stats(const prog_config_t *config)
{
    const stats_t *stats;
 
    printf("\n====================\n\n"
           "IPsec forwarder done\n\n"
           "    configuration file: %s\n"
           "    queuing mode:       %s\n"
           "    input queue count:  %u\n"
           "    SA queue count:     %u\n"
           "    output queue count: %u\n"
           "    RX mode:            %s\n", config->conf_file,
           config->mode == ORDERED ? "ordered" : "parallel", config->num_input_qs,
           config->num_sa_qs, config->num_output_qs,
           config->is_dir_rx ? "direct" : "scheduled");
 
    for (int i = 0; i < config->num_thrs; ++i) {
        stats = &config->thread_config[i].stats;
 
        printf("\n    worker %d:\n"
        "        IPsec in packets:        %" PRIu64 "\n"
        "        IPsec out packets:       %" PRIu64 "\n"
        "        IPsec in packet errors:  %" PRIu64 "\n"
        "        IPsec out packet errors: %" PRIu64 "\n"
        "        IPsec status errors:     %" PRIu64 "\n"
        "        packets forwarded:       %" PRIu64 "\n"
        "        packets dropped:         %" PRIu64 "\n", i, stats->ipsec_in_pkts,
        stats->ipsec_out_pkts, stats->ipsec_in_errs, stats->ipsec_out_errs,
        stats->status_errs, stats->fwd_pkts, stats->discards);
    }
 
    printf("\n====================\n");
}
 
static void wait_sas_disabled(uint32_t num_sas)
{
    uint32_t num_sas_dis = 0U;
    odp_event_t ev;
    odp_ipsec_status_t status;
 
    while (num_sas_dis < num_sas) {
        ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT);
 
        if (ev == ODP_EVENT_INVALID)
            continue;
 
        if (odp_event_type(ev) != ODP_EVENT_IPSEC_STATUS) {
            odp_event_free(ev);
            continue;
        }
 
        if (odp_ipsec_status(&status, ev) < 0) {
            odp_event_free(ev);
            continue;
        }
 
        if (status.id == ODP_IPSEC_STATUS_SA_DISABLE)
            ++num_sas_dis;
 
        odp_event_free(ev);
    }
}
 
static void teardown_test(const prog_config_t *config)
{
    for (uint32_t i = 0U; i < config->num_ifs; ++i) {
        free(config->pktios[i].name);
 
        if (config->pktios[i].handle != ODP_PKTIO_INVALID)
            (void)odp_pktio_close(config->pktios[i].handle);
    }
 
    if (config->pktio_pool != ODP_POOL_INVALID)
        (void)odp_pool_destroy(config->pktio_pool);
 
    for (uint32_t i = 0U; i < config->num_sas; ++i)
        (void)odp_ipsec_sa_disable(config->sas[i]);
 
    if (!config->is_dir_rx)
        /* Drain SA status events. */
        wait_sas_disabled(config->num_sas);
 
    for (uint32_t i = 0U; i < config->num_sas; ++i)
        (void)odp_ipsec_sa_destroy(config->sas[i]);
 
    for (uint32_t i = 0U; i < config->num_sa_qs; ++i)
        (void)odp_queue_destroy(config->sa_qs[i]);
 
    if (config->compl_q != ODP_QUEUE_INVALID)
        (void)odp_queue_destroy(config->compl_q);
 
    free(config->conf_file);
}
 
int main(int argc, char **argv)
{
    odph_helper_options_t odph_opts;
    odp_init_t init_param;
    odp_instance_t odp_instance;
    odp_shm_t shm_cfg = ODP_SHM_INVALID;
    parse_result_t parse_res;
    int ret = EXIT_SUCCESS;
 
    argc = odph_parse_options(argc, argv);
 
    if (odph_options(&odph_opts) == -1) {
        ODPH_ERR("Error while reading ODP helper options, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    odp_init_param_init(&init_param);
    init_param.mem_model = odph_opts.mem_model;
 
    if (odp_init_global(&odp_instance, &init_param, NULL) < 0) {
        ODPH_ERR("ODP global init failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_init_local(odp_instance, ODP_THREAD_CONTROL) < 0) {
        ODPH_ERR("ODP local init failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    shm_cfg = odp_shm_reserve(PROG_NAME "_cfg", sizeof(prog_config_t), ODP_CACHE_LINE_SIZE,
                  0U);
 
    if (shm_cfg == ODP_SHM_INVALID) {
        ODPH_ERR("Error reserving shared memory\n");
        ret = EXIT_FAILURE;
        goto out;
    }
 
    prog_conf = odp_shm_addr(shm_cfg);
 
    if (prog_conf == NULL) {
        ODPH_ERR("Error resolving shared memory address\n");
        ret = EXIT_FAILURE;
        goto out;
    }
 
    init_config(prog_conf);
 
    if (!prog_conf->is_dir_rx && odp_schedule_config(NULL) < 0) {
        ODPH_ERR("Error configuring scheduler\n");
        ret = EXIT_FAILURE;
        goto out_test;
    }
 
    parse_res = setup_program(argc, argv, prog_conf);
 
    if (parse_res == PRS_NOK) {
        ret = EXIT_FAILURE;
        goto out_test;
    }
 
    if (parse_res == PRS_TERM) {
        ret = EXIT_SUCCESS;
        goto out_test;
    }
 
    prog_conf->odp_instance = odp_instance;
    odp_atomic_init_u32(&prog_conf->is_running, 1U);
 
    if (!setup_test(prog_conf)) {
        ret = EXIT_FAILURE;
        goto out_test;
    }
 
    while (odp_atomic_load_u32(&prog_conf->is_running))
        odp_cpu_pause();
 
    stop_test(prog_conf);
    print_stats(prog_conf);
 
out_test:
    teardown_test(prog_conf);
 
out:
    if (shm_cfg != ODP_SHM_INVALID)
        (void)odp_shm_free(shm_cfg);
 
    if (odp_term_local() < 0) {
        ODPH_ERR("ODP local terminate failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    if (odp_term_global(odp_instance) < 0) {
        ODPH_ERR("ODP global terminate failed, exiting\n");
        exit(EXIT_FAILURE);
    }
 
    return ret;
}