cmhi
/
f-stack
mirror of https://github.com/F-Stack/f-stack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
f-stack/dpdk/drivers/ml/cnxk/cn10k_ml_ops.c

1539 lines
47 KiB
C
Raw Normal View History

DPDK: Upgrade to 23.11.3.
2025-01-10 11:50:43 +00:00
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 Marvell.
*/
#include <rte_mldev.h>
#include <rte_mldev_pmd.h>
#include <mldev_utils.h>
#include "cnxk_ml_dev.h"
#include "cnxk_ml_model.h"
#include "cnxk_ml_ops.h"
#include "cnxk_ml_xstats.h"
/* ML model macros */
#define CN10K_ML_MODEL_MEMZONE_NAME "ml_cn10k_model_mz"
/* ML layer macros */
#define CN10K_ML_LAYER_MEMZONE_NAME "ml_cn10k_layer_mz"
/* ML Job descriptor flags */
#define ML_FLAGS_POLL_COMPL BIT(0)
#define ML_FLAGS_SSO_COMPL BIT(1)
/* Hardware non-fatal error subtype database */
static struct cnxk_ml_error_db ml_stype_db_hw_nf[] = {
{ML_CN10K_FW_ERR_NOERR, "NO ERROR"},
{ML_CN10K_FW_ERR_UNLOAD_ID_NOT_FOUND, "UNLOAD MODEL ID NOT FOUND"},
{ML_CN10K_FW_ERR_LOAD_LUT_OVERFLOW, "LOAD LUT OVERFLOW"},
{ML_CN10K_FW_ERR_ID_IN_USE, "MODEL ID IN USE"},
{ML_CN10K_FW_ERR_INVALID_TILEMASK, "INVALID TILEMASK"},
{ML_CN10K_FW_ERR_RUN_LUT_OVERFLOW, "RUN LUT OVERFLOW"},
{ML_CN10K_FW_ERR_RUN_ID_NOT_FOUND, "RUN MODEL ID NOT FOUND"},
{ML_CN10K_FW_ERR_COMMAND_NOTSUP, "COMMAND NOT SUPPORTED"},
{ML_CN10K_FW_ERR_DDR_ADDR_RANGE, "DDR ADDRESS OUT OF RANGE"},
{ML_CN10K_FW_ERR_NUM_BATCHES_INVALID, "INVALID BATCHES"},
{ML_CN10K_FW_ERR_INSSYNC_TIMEOUT, "INSSYNC TIMEOUT"},
};
/* Driver error subtype database */
static struct cnxk_ml_error_db ml_stype_db_driver[] = {
{ML_CN10K_DRIVER_ERR_NOERR, "NO ERROR"},
{ML_CN10K_DRIVER_ERR_UNKNOWN, "UNKNOWN ERROR"},
{ML_CN10K_DRIVER_ERR_EXCEPTION, "FW EXCEPTION"},
{ML_CN10K_DRIVER_ERR_FW_ERROR, "UNKNOWN FIRMWARE ERROR"},
};
__rte_hot void
cn10k_ml_set_poll_addr(struct cnxk_ml_req *req)
{
req->status = &req->cn10k_req.status;
}
void
cn10k_ml_qp_initialize(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_qp *qp)
{
uint64_t i;
RTE_SET_USED(cnxk_mldev);
/* Initialize job command */
for (i = 0; i < qp->nb_desc; i++) {
memset(&qp->queue.reqs[i].cn10k_req.jd, 0, sizeof(struct cn10k_ml_jd));
qp->queue.reqs[i].cn10k_req.jcmd.w1.s.jobptr =
PLT_U64_CAST(&qp->queue.reqs[i].cn10k_req.jd);
}
}
static void
cn10k_ml_prep_sp_job_descriptor(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer,
struct cnxk_ml_req *req, enum cn10k_ml_job_type job_type)
{
struct cn10k_ml_model_metadata *metadata;
struct cn10k_ml_layer_addr *addr;
struct cn10k_ml_dev *cn10k_mldev;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
metadata = &layer->glow.metadata;
addr = &layer->glow.addr;
memset(&req->cn10k_req.jd, 0, sizeof(struct cn10k_ml_jd));
req->cn10k_req.jd.hdr.jce.w0.u64 = 0;
req->cn10k_req.jd.hdr.jce.w1.u64 = PLT_U64_CAST(&req->cn10k_req.status);
req->cn10k_req.jd.hdr.model_id = layer->index;
req->cn10k_req.jd.hdr.job_type = job_type;
req->cn10k_req.jd.hdr.fp_flags = 0x0;
req->cn10k_req.jd.hdr.result =
roc_ml_addr_ap2mlip(&cn10k_mldev->roc, &req->cn10k_req.result);
if (job_type == ML_CN10K_JOB_TYPE_MODEL_START) {
if (!layer->glow.metadata.model.ocm_relocatable)
req->cn10k_req.jd.hdr.sp_flags = ML_CN10K_SP_FLAGS_OCM_NONRELOCATABLE;
else
req->cn10k_req.jd.hdr.sp_flags = 0x0;
req->cn10k_req.jd.hdr.sp_flags |= ML_CN10K_SP_FLAGS_EXTENDED_LOAD_JD;
req->cn10k_req.jd.model_start.extended_args = PLT_U64_CAST(
roc_ml_addr_ap2mlip(&cn10k_mldev->roc, &req->cn10k_req.extended_args));
req->cn10k_req.jd.model_start.model_dst_ddr_addr =
PLT_U64_CAST(roc_ml_addr_ap2mlip(&cn10k_mldev->roc, addr->init_load_addr));
req->cn10k_req.jd.model_start.model_init_offset = 0x0;
req->cn10k_req.jd.model_start.model_main_offset = metadata->init_model.file_size;
req->cn10k_req.jd.model_start.model_finish_offset =
metadata->init_model.file_size + metadata->main_model.file_size;
req->cn10k_req.jd.model_start.model_init_size = metadata->init_model.file_size;
req->cn10k_req.jd.model_start.model_main_size = metadata->main_model.file_size;
req->cn10k_req.jd.model_start.model_finish_size = metadata->finish_model.file_size;
req->cn10k_req.jd.model_start.model_wb_offset = metadata->init_model.file_size +
metadata->main_model.file_size +
metadata->finish_model.file_size;
req->cn10k_req.jd.model_start.num_layers = metadata->model.num_layers;
req->cn10k_req.jd.model_start.num_gather_entries = 0;
req->cn10k_req.jd.model_start.num_scatter_entries = 0;
req->cn10k_req.jd.model_start.tilemask = 0; /* Updated after reserving pages */
req->cn10k_req.jd.model_start.batch_size = layer->batch_size;
req->cn10k_req.jd.model_start.ocm_wb_base_address =
0; /* Updated after reserving pages */
req->cn10k_req.jd.model_start.ocm_wb_range_start =
metadata->model.ocm_wb_range_start;
req->cn10k_req.jd.model_start.ocm_wb_range_end = metadata->model.ocm_wb_range_end;
req->cn10k_req.jd.model_start.ddr_wb_base_address =
PLT_U64_CAST(roc_ml_addr_ap2mlip(
&cn10k_mldev->roc, PLT_PTR_ADD(addr->finish_load_addr,
metadata->finish_model.file_size)));
req->cn10k_req.jd.model_start.ddr_wb_range_start =
metadata->model.ddr_wb_range_start;
req->cn10k_req.jd.model_start.ddr_wb_range_end = metadata->model.ddr_wb_range_end;
req->cn10k_req.jd.model_start.input.s.ddr_range_start =
metadata->model.ddr_input_range_start;
req->cn10k_req.jd.model_start.input.s.ddr_range_end =
metadata->model.ddr_input_range_end;
req->cn10k_req.jd.model_start.output.s.ddr_range_start =
metadata->model.ddr_output_range_start;
req->cn10k_req.jd.model_start.output.s.ddr_range_end =
metadata->model.ddr_output_range_end;
req->cn10k_req.extended_args.start.ddr_scratch_base_address = PLT_U64_CAST(
roc_ml_addr_ap2mlip(&cn10k_mldev->roc, addr->scratch_base_addr));
req->cn10k_req.extended_args.start.ddr_scratch_range_start =
metadata->model.ddr_scratch_range_start;
req->cn10k_req.extended_args.start.ddr_scratch_range_end =
metadata->model.ddr_scratch_range_end;
}
}
static __rte_always_inline void
cn10k_ml_prep_fp_job_descriptor(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_req *req,
uint16_t index, void *input, void *output, uint16_t nb_batches)
{
struct cn10k_ml_dev *cn10k_mldev;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
req->cn10k_req.jd.hdr.jce.w0.u64 = 0;
req->cn10k_req.jd.hdr.jce.w1.u64 = PLT_U64_CAST(req->status);
req->cn10k_req.jd.hdr.model_id = index;
req->cn10k_req.jd.hdr.job_type = ML_CN10K_JOB_TYPE_MODEL_RUN;
req->cn10k_req.jd.hdr.fp_flags = ML_FLAGS_POLL_COMPL;
req->cn10k_req.jd.hdr.sp_flags = 0x0;
req->cn10k_req.jd.hdr.result =
roc_ml_addr_ap2mlip(&cn10k_mldev->roc, &req->cn10k_req.result);
req->cn10k_req.jd.model_run.input_ddr_addr =
PLT_U64_CAST(roc_ml_addr_ap2mlip(&cn10k_mldev->roc, input));
req->cn10k_req.jd.model_run.output_ddr_addr =
PLT_U64_CAST(roc_ml_addr_ap2mlip(&cn10k_mldev->roc, output));
req->cn10k_req.jd.model_run.num_batches = nb_batches;
}
static void
cn10k_ml_xstats_layer_name_update(struct cnxk_ml_dev *cnxk_mldev, uint16_t model_id,
uint16_t layer_id)
{
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
uint16_t rclk_freq;
uint16_t sclk_freq;
uint16_t stat_id;
char suffix[8];
uint16_t i;
model = cnxk_mldev->mldev->data->models[model_id];
layer = &model->layer[layer_id];
stat_id = cnxk_mldev->xstats.offset_for_layer[model_id][layer_id];
roc_clk_freq_get(&rclk_freq, &sclk_freq);
if (sclk_freq == 0)
strcpy(suffix, "cycles");
else
strcpy(suffix, "ns");
/* Update xstat name based on layer name and sclk availability */
for (i = 0; i < RTE_DIM(layer_xstats); i++) {
snprintf(cnxk_mldev->xstats.entries[stat_id].map.name,
sizeof(cnxk_mldev->xstats.entries[stat_id].map.name), "%s-%s-%s",
layer->glow.metadata.model.name, layer_xstats[i].name, suffix);
stat_id++;
}
}
void
cn10k_ml_xstat_model_name_set(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model,
uint16_t stat_id, uint16_t entry, char *suffix)
{
snprintf(cnxk_mldev->xstats.entries[stat_id].map.name,
sizeof(cnxk_mldev->xstats.entries[stat_id].map.name), "%s-%s-%s",
model->glow.metadata.model.name, model_xstats[entry].name, suffix);
}
#define ML_AVG_FOREACH_QP(cnxk_mldev, layer, qp_id, str, value, count) \
do { \
value = 0; \
for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \
value += layer->glow.burst_xstats[qp_id].str##_latency_tot; \
count += layer->glow.burst_xstats[qp_id].dequeued_count - \
layer->glow.burst_xstats[qp_id].str##_reset_count; \
} \
value += layer->glow.sync_xstats->str##_latency_tot; \
count += layer->glow.sync_xstats->dequeued_count - \
layer->glow.sync_xstats->str##_reset_count; \
if (count != 0) \
value = value / count; \
} while (0)
#define ML_MIN_FOREACH_QP(cnxk_mldev, layer, qp_id, str, value, count) \
do { \
value = UINT64_MAX; \
for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \
value = PLT_MIN(value, layer->glow.burst_xstats[qp_id].str##_latency_min); \
count += layer->glow.burst_xstats[qp_id].dequeued_count - \
layer->glow.burst_xstats[qp_id].str##_reset_count; \
} \
value = PLT_MIN(value, layer->glow.sync_xstats->str##_latency_min); \
count += layer->glow.sync_xstats->dequeued_count - \
layer->glow.sync_xstats->str##_reset_count; \
if (count == 0) \
value = 0; \
} while (0)
#define ML_MAX_FOREACH_QP(cnxk_mldev, layer, qp_id, str, value, count) \
do { \
value = 0; \
for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs; qp_id++) { \
value = PLT_MAX(value, layer->glow.burst_xstats[qp_id].str##_latency_max); \
count += layer->glow.burst_xstats[qp_id].dequeued_count - \
layer->glow.burst_xstats[qp_id].str##_reset_count; \
} \
value = PLT_MAX(value, layer->glow.sync_xstats->str##_latency_max); \
count += layer->glow.sync_xstats->dequeued_count - \
layer->glow.sync_xstats->str##_reset_count; \
if (count == 0) \
value = 0; \
} while (0)
uint64_t
cn10k_ml_model_xstat_get(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer,
enum cnxk_ml_xstats_type type)
{
uint64_t count = 0;
uint64_t value = 0;
uint32_t qp_id;
switch (type) {
case avg_hw_latency:
ML_AVG_FOREACH_QP(cnxk_mldev, layer, qp_id, hw, value, count);
break;
case min_hw_latency:
ML_MIN_FOREACH_QP(cnxk_mldev, layer, qp_id, hw, value, count);
break;
case max_hw_latency:
ML_MAX_FOREACH_QP(cnxk_mldev, layer, qp_id, hw, value, count);
break;
case avg_fw_latency:
ML_AVG_FOREACH_QP(cnxk_mldev, layer, qp_id, fw, value, count);
break;
case min_fw_latency:
ML_MIN_FOREACH_QP(cnxk_mldev, layer, qp_id, fw, value, count);
break;
case max_fw_latency:
ML_MAX_FOREACH_QP(cnxk_mldev, layer, qp_id, fw, value, count);
break;
default:
value = 0;
}
return value;
}
static int
cn10k_ml_cache_model_data(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_layer *layer)
{
struct cn10k_ml_layer_xstats *xstats;
char str[RTE_MEMZONE_NAMESIZE];
const struct plt_memzone *mz;
uint64_t isize = 0;
uint64_t osize = 0;
int ret = 0;
/* Create input and output buffers. */
isize = layer->info.total_input_sz_q;
osize = layer->info.total_output_sz_q;
snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u", "ml_dummy_io", layer->index);
mz = plt_memzone_reserve_aligned(str, isize + osize, 0, ML_CN10K_ALIGN_SIZE);
if (mz == NULL)
return -ENOMEM;
memset(mz->addr, 0, isize + osize);
memset(layer->glow.req, 0, sizeof(struct cnxk_ml_req));
ret = cn10k_ml_inference_sync(cnxk_mldev, layer->index, mz->addr,
PLT_PTR_ADD(mz->addr, isize), 1);
plt_memzone_free(mz);
/* Reset sync xstats. */
xstats = layer->glow.sync_xstats;
xstats->hw_latency_tot = 0;
xstats->hw_latency_min = UINT64_MAX;
xstats->hw_latency_max = 0;
xstats->fw_latency_tot = 0;
xstats->fw_latency_min = UINT64_MAX;
xstats->fw_latency_max = 0;
xstats->dequeued_count = 0;
return ret;
}
int
cn10k_ml_dev_info_get(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_dev_info *dev_info)
{
struct cn10k_ml_dev *cn10k_mldev;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
if (cn10k_mldev->hw_queue_lock)
dev_info->max_queue_pairs = ML_CN10K_MAX_QP_PER_DEVICE_SL;
else
dev_info->max_queue_pairs = ML_CN10K_MAX_QP_PER_DEVICE_LF;
dev_info->max_desc = ML_CN10K_MAX_DESC_PER_QP;
dev_info->max_io = ML_CN10K_MAX_INPUT_OUTPUT;
dev_info->max_segments = ML_CN10K_MAX_SEGMENTS;
dev_info->align_size = ML_CN10K_ALIGN_SIZE;
return 0;
}
int
cn10k_ml_dev_configure(struct cnxk_ml_dev *cnxk_mldev, const struct rte_ml_dev_config *conf)
{
struct cn10k_ml_dev *cn10k_mldev;
struct cn10k_ml_ocm *ocm;
uint16_t tile_id;
RTE_SET_USED(conf);
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
ocm = &cn10k_mldev->ocm;
ocm->num_tiles = ML_CN10K_OCM_NUMTILES;
ocm->size_per_tile = ML_CN10K_OCM_TILESIZE;
ocm->page_size = cn10k_mldev->ocm_page_size;
ocm->num_pages = ocm->size_per_tile / ocm->page_size;
ocm->mask_words = ocm->num_pages / (8 * sizeof(uint8_t));
/* Allocate memory for ocm_mask */
ocm->ocm_mask =
rte_zmalloc("ocm_mask", ocm->mask_words * ocm->num_tiles, RTE_CACHE_LINE_SIZE);
if (ocm->ocm_mask == NULL) {
plt_err("Unable to allocate memory for OCM mask");
return -ENOMEM;
}
for (tile_id = 0; tile_id < ocm->num_tiles; tile_id++) {
ocm->tile_ocm_info[tile_id].ocm_mask = ocm->ocm_mask + tile_id * ocm->mask_words;
ocm->tile_ocm_info[tile_id].last_wb_page = -1;
}
rte_spinlock_init(&ocm->lock);
/* Set JCMDQ enqueue function */
if (cn10k_mldev->hw_queue_lock == 1)
cn10k_mldev->ml_jcmdq_enqueue = roc_ml_jcmdq_enqueue_sl;
else
cn10k_mldev->ml_jcmdq_enqueue = roc_ml_jcmdq_enqueue_lf;
return 0;
}
int
cn10k_ml_dev_close(struct cnxk_ml_dev *cnxk_mldev)
{
struct cn10k_ml_dev *cn10k_mldev;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
/* Release ocm_mask memory */
rte_free(cn10k_mldev->ocm.ocm_mask);
/* Unload firmware */
cn10k_ml_fw_unload(cnxk_mldev);
/* Clear scratch registers */
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_WORK_PTR);
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_FW_CTRL);
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_DBG_BUFFER_HEAD_C0);
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_DBG_BUFFER_TAIL_C0);
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_DBG_BUFFER_HEAD_C1);
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_SCRATCH_DBG_BUFFER_TAIL_C1);
/* Reset ML_MLR_BASE */
roc_ml_reg_write64(&cn10k_mldev->roc, 0, ML_MLR_BASE);
plt_ml_dbg("ML_MLR_BASE = 0x%016lx", roc_ml_reg_read64(&cn10k_mldev->roc, ML_MLR_BASE));
return 0;
}
int
cn10k_ml_dev_start(struct cnxk_ml_dev *cnxk_mldev)
{
struct cn10k_ml_dev *cn10k_mldev;
uint64_t reg_val64;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
reg_val64 = roc_ml_reg_read64(&cn10k_mldev->roc, ML_CFG);
reg_val64 |= ROC_ML_CFG_ENA;
roc_ml_reg_write64(&cn10k_mldev->roc, reg_val64, ML_CFG);
plt_ml_dbg("ML_CFG => 0x%016lx", roc_ml_reg_read64(&cn10k_mldev->roc, ML_CFG));
return 0;
}
int
cn10k_ml_dev_stop(struct cnxk_ml_dev *cnxk_mldev)
{
struct cn10k_ml_dev *cn10k_mldev;
uint64_t reg_val64;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
reg_val64 = roc_ml_reg_read64(&cn10k_mldev->roc, ML_CFG);
reg_val64 &= ~ROC_ML_CFG_ENA;
roc_ml_reg_write64(&cn10k_mldev->roc, reg_val64, ML_CFG);
plt_ml_dbg("ML_CFG => 0x%016lx", roc_ml_reg_read64(&cn10k_mldev->roc, ML_CFG));
return 0;
}
int
cn10k_ml_dev_dump(struct cnxk_ml_dev *cnxk_mldev, FILE *fp)
{
struct cn10k_ml_dev *cn10k_mldev;
struct cn10k_ml_fw *fw;
uint32_t head_loc;
uint32_t tail_loc;
uint32_t bufsize;
char *head_ptr;
int core_id;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
fw = &cn10k_mldev->fw;
/* Dump OCM state */
cn10k_ml_ocm_print(cnxk_mldev, fp);
if (roc_env_is_asim())
return 0;
/* Dump debug buffer */
for (core_id = 0; core_id <= 1; core_id++) {
bufsize = fw->req->cn10k_req.jd.fw_load.debug.debug_buffer_size;
if (core_id == 0) {
head_loc =
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_DBG_BUFFER_HEAD_C0);
tail_loc =
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_DBG_BUFFER_TAIL_C0);
head_ptr =
PLT_PTR_CAST(fw->req->cn10k_req.jd.fw_load.debug.core0_debug_ptr);
head_ptr = roc_ml_addr_mlip2ap(&cn10k_mldev->roc, head_ptr);
} else {
head_loc =
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_DBG_BUFFER_HEAD_C1);
tail_loc =
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_DBG_BUFFER_TAIL_C1);
head_ptr =
PLT_PTR_CAST(fw->req->cn10k_req.jd.fw_load.debug.core1_debug_ptr);
head_ptr = roc_ml_addr_mlip2ap(&cn10k_mldev->roc, head_ptr);
}
if (head_loc < tail_loc) {
fprintf(fp, "%.*s\n", tail_loc - head_loc, &head_ptr[head_loc]);
} else if (head_loc >= tail_loc + 1) {
fprintf(fp, "%.*s\n", bufsize - tail_loc, &head_ptr[head_loc]);
fprintf(fp, "%.*s\n", tail_loc, &head_ptr[0]);
}
}
/* Dump exception info */
for (core_id = 0; core_id <= 1; core_id++) {
bufsize = fw->req->cn10k_req.jd.fw_load.debug.exception_state_size;
if ((core_id == 0) &&
(roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_EXCEPTION_SP_C0) != 0)) {
head_ptr = PLT_PTR_CAST(
fw->req->cn10k_req.jd.fw_load.debug.core0_exception_buffer);
fprintf(fp, "ML_SCRATCH_EXCEPTION_SP_C0 = 0x%016lx",
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_EXCEPTION_SP_C0));
head_ptr = roc_ml_addr_mlip2ap(&cn10k_mldev->roc, head_ptr);
fprintf(fp, "%.*s", bufsize, head_ptr);
} else if ((core_id == 1) && (roc_ml_reg_read64(&cn10k_mldev->roc,
ML_SCRATCH_EXCEPTION_SP_C1) != 0)) {
head_ptr = PLT_PTR_CAST(
fw->req->cn10k_req.jd.fw_load.debug.core1_exception_buffer);
fprintf(fp, "ML_SCRATCH_EXCEPTION_SP_C1 = 0x%016lx",
roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_EXCEPTION_SP_C1));
head_ptr = roc_ml_addr_mlip2ap(&cn10k_mldev->roc, head_ptr);
fprintf(fp, "%.*s", bufsize, head_ptr);
}
}
return 0;
}
int
cn10k_ml_dev_selftest(struct cnxk_ml_dev *cnxk_mldev)
{
struct cn10k_ml_dev *cn10k_mldev;
const struct plt_memzone *mz;
struct cnxk_ml_req *req;
uint64_t timeout_cycle;
bool timeout;
int ret;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
mz = plt_memzone_reserve_aligned("dev_selftest", sizeof(struct cnxk_ml_req), 0,
ML_CN10K_ALIGN_SIZE);
if (mz == NULL) {
plt_err("Could not allocate reserved memzone");
return -ENOMEM;
}
req = mz->addr;
/* Prepare load completion structure */
memset(&req->cn10k_req.jd, 0, sizeof(struct cn10k_ml_jd));
req->cn10k_req.jd.hdr.jce.w1.u64 = PLT_U64_CAST(&req->cn10k_req.status);
req->cn10k_req.jd.hdr.job_type = ML_CN10K_JOB_TYPE_FIRMWARE_SELFTEST;
req->cn10k_req.jd.hdr.result =
roc_ml_addr_ap2mlip(&cn10k_mldev->roc, &req->cn10k_req.result);
req->cn10k_req.jd.fw_load.flags = cn10k_ml_fw_flags_get(&cn10k_mldev->fw);
plt_write64(ML_CNXK_POLL_JOB_START, &req->cn10k_req.status);
plt_wmb();
/* Enqueue firmware selftest request through scratch registers */
timeout = true;
timeout_cycle = plt_tsc_cycles() + ML_CNXK_CMD_TIMEOUT * plt_tsc_hz();
roc_ml_scratch_enqueue(&cn10k_mldev->roc, &req->cn10k_req.jd);
plt_rmb();
do {
if (roc_ml_scratch_is_done_bit_set(&cn10k_mldev->roc) &&
(plt_read64(&req->cn10k_req.status) == ML_CNXK_POLL_JOB_FINISH)) {
timeout = false;
break;
}
} while (plt_tsc_cycles() < timeout_cycle);
/* Check firmware selftest status, clean-up and exit */
ret = 0;
if (timeout) {
ret = -ETIME;
} else {
if (req->cn10k_req.result.error_code != 0)
ret = -1;
}
plt_memzone_free(mz);
return ret;
}
int
cn10k_ml_layer_load(void *device, uint16_t model_id, const char *layer_name, uint8_t *buffer,
size_t size, uint16_t *index)
{
struct cn10k_ml_model_metadata *metadata;
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
char str[RTE_MEMZONE_NAMESIZE];
const struct plt_memzone *mz;
size_t layer_object_size = 0;
size_t layer_scratch_size;
size_t layer_xstats_size;
uint8_t *base_dma_addr;
uint16_t scratch_pages;
uint16_t layer_id;
uint16_t wb_pages;
uint64_t mz_size;
uint16_t idx;
int qp_id;
int ret;
PLT_SET_USED(size);
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
layer = &model->layer[layer_id];
ret = cn10k_ml_model_metadata_check(buffer, size);
if (ret != 0)
return ret;
/* Get index */
for (idx = 0; idx < cnxk_mldev->max_nb_layers; idx++) {
if (!cnxk_mldev->index_map[idx].active) {
layer->index = idx;
break;
}
}
if (idx >= cnxk_mldev->max_nb_layers) {
plt_err("No slots available for model layers, model_id = %u, layer_id = %u",
model->model_id, layer_id);
return -1;
}
layer->model = model;
/* Get WB and scratch pages, check if model can be loaded. */
ret = cn10k_ml_model_ocm_pages_count(cnxk_mldev, layer, buffer, &wb_pages, &scratch_pages);
if (ret < 0)
return ret;
/* Compute layer memzone size */
metadata = (struct cn10k_ml_model_metadata *)buffer;
layer_object_size = metadata->init_model.file_size + metadata->main_model.file_size +
metadata->finish_model.file_size + metadata->weights_bias.file_size;
layer_object_size = PLT_ALIGN_CEIL(layer_object_size, ML_CN10K_ALIGN_SIZE);
layer_scratch_size = PLT_ALIGN_CEIL(metadata->model.ddr_scratch_range_end -
metadata->model.ddr_scratch_range_start + 1,
ML_CN10K_ALIGN_SIZE);
layer_xstats_size = (cnxk_mldev->mldev->data->nb_queue_pairs + 1) *
sizeof(struct cn10k_ml_layer_xstats);
/* Allocate memzone for model data */
mz_size = layer_object_size + layer_scratch_size +
PLT_ALIGN_CEIL(sizeof(struct cnxk_ml_req), ML_CN10K_ALIGN_SIZE) +
layer_xstats_size;
snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u_%u", CN10K_ML_LAYER_MEMZONE_NAME,
model->model_id, layer_id);
mz = plt_memzone_reserve_aligned(str, mz_size, 0, ML_CN10K_ALIGN_SIZE);
if (!mz) {
plt_err("plt_memzone_reserve failed : %s", str);
return -ENOMEM;
}
/* Copy metadata to internal buffer */
rte_memcpy(&layer->glow.metadata, buffer, sizeof(struct cn10k_ml_model_metadata));
cn10k_ml_model_metadata_update(&layer->glow.metadata);
/* Set layer name */
rte_memcpy(layer->name, layer->glow.metadata.model.name, MRVL_ML_MODEL_NAME_LEN);
/* Enable support for batch_size of 256 */
if (layer->glow.metadata.model.batch_size == 0)
layer->batch_size = 256;
else
layer->batch_size = layer->glow.metadata.model.batch_size;
/* Set DMA base address */
base_dma_addr = mz->addr;
cn10k_ml_layer_addr_update(layer, buffer, base_dma_addr);
/* Set scratch base address */
layer->glow.addr.scratch_base_addr = PLT_PTR_ADD(base_dma_addr, layer_object_size);
/* Update internal I/O data structure */
cn10k_ml_layer_io_info_set(&layer->info, &layer->glow.metadata);
/* Initialize model_mem_map */
memset(&layer->glow.ocm_map, 0, sizeof(struct cn10k_ml_ocm_layer_map));
layer->glow.ocm_map.ocm_reserved = false;
layer->glow.ocm_map.tilemask = 0;
layer->glow.ocm_map.wb_page_start = -1;
layer->glow.ocm_map.wb_pages = wb_pages;
layer->glow.ocm_map.scratch_pages = scratch_pages;
/* Set slow-path request address and state */
layer->glow.req = PLT_PTR_ADD(mz->addr, layer_object_size + layer_scratch_size);
/* Reset burst and sync stats */
layer->glow.burst_xstats = PLT_PTR_ADD(
layer->glow.req, PLT_ALIGN_CEIL(sizeof(struct cnxk_ml_req), ML_CN10K_ALIGN_SIZE));
for (qp_id = 0; qp_id < cnxk_mldev->mldev->data->nb_queue_pairs + 1; qp_id++) {
layer->glow.burst_xstats[qp_id].hw_latency_tot = 0;
layer->glow.burst_xstats[qp_id].hw_latency_min = UINT64_MAX;
layer->glow.burst_xstats[qp_id].hw_latency_max = 0;
layer->glow.burst_xstats[qp_id].fw_latency_tot = 0;
layer->glow.burst_xstats[qp_id].fw_latency_min = UINT64_MAX;
layer->glow.burst_xstats[qp_id].fw_latency_max = 0;
layer->glow.burst_xstats[qp_id].hw_reset_count = 0;
layer->glow.burst_xstats[qp_id].fw_reset_count = 0;
layer->glow.burst_xstats[qp_id].dequeued_count = 0;
}
layer->glow.sync_xstats =
PLT_PTR_ADD(layer->glow.burst_xstats, cnxk_mldev->mldev->data->nb_queue_pairs *
sizeof(struct cn10k_ml_layer_xstats));
/* Update xstats names */
cn10k_ml_xstats_layer_name_update(cnxk_mldev, model_id, layer_id);
layer->state = ML_CNXK_LAYER_STATE_LOADED;
cnxk_mldev->index_map[idx].model_id = model->model_id;
cnxk_mldev->index_map[idx].layer_id = layer_id;
cnxk_mldev->index_map[idx].active = true;
*index = idx;
return 0;
}
int
cn10k_ml_model_load(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_model_params *params,
struct cnxk_ml_model *model)
{
struct cnxk_ml_layer *layer;
int ret;
/* Metadata check */
ret = cn10k_ml_model_metadata_check(params->addr, params->size);
if (ret != 0)
return ret;
/* Set model sub type */
model->subtype = ML_CNXK_MODEL_SUBTYPE_GLOW_MRVL;
/* Copy metadata to internal buffer */
rte_memcpy(&model->glow.metadata, params->addr, sizeof(struct cn10k_ml_model_metadata));
cn10k_ml_model_metadata_update(&model->glow.metadata);
/* Set model name */
rte_memcpy(model->name, (char *)model->glow.metadata.model.name, 64);
/* Enable support for batch_size of 256 */
if (model->glow.metadata.model.batch_size == 0)
model->batch_size = 256;
else
model->batch_size = model->glow.metadata.model.batch_size;
/* Since the number of layers that the driver would be handling for glow models is
* always 1. consider the entire model as a model with single layer. This would
* ignore the num_layers from metadata.
*/
model->nb_layers = 1;
/* Load layer and get the index */
layer = &model->layer[0];
layer->type = ML_CNXK_LAYER_TYPE_MRVL;
ret = cn10k_ml_layer_load(cnxk_mldev, model->model_id, NULL, params->addr, params->size,
&layer->index);
if (ret != 0) {
plt_err("Model layer load failed: model_id = %u, layer_id = %u", model->model_id,
0);
return ret;
}
cn10k_ml_model_info_set(cnxk_mldev, model, &model->layer[0].info, &model->glow.metadata);
/* Set fast-path functions */
model->enqueue_single = cn10k_ml_enqueue_single;
model->result_update = cn10k_ml_result_update;
model->set_error_code = cn10k_ml_set_error_code;
model->set_poll_addr = cn10k_ml_set_poll_addr;
return 0;
}
int
cn10k_ml_layer_unload(void *device, uint16_t model_id, const char *layer_name)
{
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
char str[RTE_MEMZONE_NAMESIZE];
uint16_t layer_id;
int ret;
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
layer = &model->layer[layer_id];
snprintf(str, RTE_MEMZONE_NAMESIZE, "%s_%u_%u", CN10K_ML_LAYER_MEMZONE_NAME,
model->model_id, layer_id);
ret = plt_memzone_free(plt_memzone_lookup(str));
layer->state = ML_CNXK_LAYER_STATE_UNKNOWN;
cnxk_mldev->index_map[layer->index].active = false;
return ret;
}
int
cn10k_ml_model_unload(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model)
{
return cn10k_ml_layer_unload(cnxk_mldev, model->model_id, NULL);
}
int
cn10k_ml_layer_start(void *device, uint16_t model_id, const char *layer_name)
{
struct cn10k_ml_dev *cn10k_mldev;
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
struct cn10k_ml_ocm *ocm;
struct cnxk_ml_req *req;
uint16_t layer_id;
bool job_enqueued;
bool job_dequeued;
uint8_t num_tiles;
uint64_t tilemask;
int wb_page_start;
int tile_start;
int tile_end;
bool locked;
int ret = 0;
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
layer = &model->layer[layer_id];
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
ocm = &cn10k_mldev->ocm;
/* Prepare JD */
req = layer->glow.req;
cn10k_ml_prep_sp_job_descriptor(cnxk_mldev, layer, req, ML_CN10K_JOB_TYPE_MODEL_START);
req->cn10k_req.result.error_code = 0x0;
req->cn10k_req.result.user_ptr = NULL;
plt_write64(ML_CNXK_POLL_JOB_START, &req->cn10k_req.status);
plt_wmb();
num_tiles = layer->glow.metadata.model.tile_end - layer->glow.metadata.model.tile_start + 1;
locked = false;
while (!locked) {
if (plt_spinlock_trylock(&model->lock) != 0) {
if (layer->state == ML_CNXK_LAYER_STATE_STARTED) {
plt_ml_dbg("Layer already started, model_id = %u, layer_id = %u",
model->model_id, layer_id);
plt_spinlock_unlock(&model->lock);
return 1;
}
if (layer->state == ML_CNXK_LAYER_STATE_JOB_ACTIVE) {
plt_err("A slow-path job is active for the model_id = %u",
model->model_id);
plt_spinlock_unlock(&model->lock);
return -EBUSY;
}
layer->state = ML_CNXK_LAYER_STATE_JOB_ACTIVE;
plt_spinlock_unlock(&model->lock);
locked = true;
}
}
while (!layer->glow.ocm_map.ocm_reserved) {
if (plt_spinlock_trylock(&ocm->lock) != 0) {
wb_page_start = cn10k_ml_ocm_tilemask_find(
cnxk_mldev, num_tiles, layer->glow.ocm_map.wb_pages,
layer->glow.ocm_map.scratch_pages, &tilemask);
if (wb_page_start == -1) {
plt_err("Free pages not available on OCM tiles");
plt_err("Failed to start layer, model_id = %u, layer_id = %u",
model->model_id, layer_id);
plt_spinlock_unlock(&ocm->lock);
return -ENOMEM;
}
layer->glow.ocm_map.tilemask = tilemask;
layer->glow.ocm_map.wb_page_start = wb_page_start;
cn10k_ml_ocm_reserve_pages(
cnxk_mldev, model->model_id, layer_id, layer->glow.ocm_map.tilemask,
layer->glow.ocm_map.wb_page_start, layer->glow.ocm_map.wb_pages,
layer->glow.ocm_map.scratch_pages);
layer->glow.ocm_map.ocm_reserved = true;
plt_spinlock_unlock(&ocm->lock);
}
}
/* Update JD */
cn10k_ml_ocm_tilecount(layer->glow.ocm_map.tilemask, &tile_start, &tile_end);
req->cn10k_req.jd.model_start.tilemask = GENMASK_ULL(tile_end, tile_start);
req->cn10k_req.jd.model_start.ocm_wb_base_address =
layer->glow.ocm_map.wb_page_start * ocm->page_size;
job_enqueued = false;
job_dequeued = false;
do {
if (!job_enqueued) {
req->timeout = plt_tsc_cycles() + ML_CNXK_CMD_TIMEOUT * plt_tsc_hz();
job_enqueued =
roc_ml_scratch_enqueue(&cn10k_mldev->roc, &req->cn10k_req.jd);
}
if (job_enqueued && !job_dequeued)
job_dequeued =
roc_ml_scratch_dequeue(&cn10k_mldev->roc, &req->cn10k_req.jd);
if (job_dequeued)
break;
} while (plt_tsc_cycles() < req->timeout);
if (job_dequeued) {
if (plt_read64(&req->cn10k_req.status) == ML_CNXK_POLL_JOB_FINISH) {
if (req->cn10k_req.result.error_code == 0)
ret = 0;
else
ret = -1;
}
} else { /* Reset scratch registers */
roc_ml_scratch_queue_reset(&cn10k_mldev->roc);
ret = -ETIME;
}
locked = false;
while (!locked) {
if (plt_spinlock_trylock(&model->lock) != 0) {
if (ret == 0)
layer->state = ML_CNXK_LAYER_STATE_STARTED;
else
layer->state = ML_CNXK_LAYER_STATE_UNKNOWN;
plt_spinlock_unlock(&model->lock);
locked = true;
}
}
if (layer->state == ML_CNXK_LAYER_STATE_UNKNOWN) {
while (layer->glow.ocm_map.ocm_reserved) {
if (plt_spinlock_trylock(&ocm->lock) != 0) {
cn10k_ml_ocm_free_pages(cnxk_mldev, model->model_id, layer_id);
layer->glow.ocm_map.ocm_reserved = false;
layer->glow.ocm_map.tilemask = 0x0;
plt_spinlock_unlock(&ocm->lock);
}
}
}
if (ret < 0) {
cn10k_ml_layer_stop(device, model_id, layer_name);
} else {
if (cn10k_mldev->cache_model_data && model->type == ML_CNXK_MODEL_TYPE_GLOW)
ret = cn10k_ml_cache_model_data(cnxk_mldev, layer);
}
return ret;
}
int
cn10k_ml_model_start(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model)
{
struct cnxk_ml_layer *layer;
int ret;
layer = &model->layer[0];
ret = cn10k_ml_layer_start(cnxk_mldev, model->model_id, layer->name);
if (ret != 0) {
plt_err("CN10K Model start failed, model_id = %u, error = %d", model->model_id,
ret);
return ret;
}
cnxk_mldev->nb_models_started++;
model->state = ML_CNXK_MODEL_STATE_STARTED;
return 0;
}
int
cn10k_ml_layer_stop(void *device, uint16_t model_id, const char *layer_name)
{
struct cn10k_ml_dev *cn10k_mldev;
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
struct cn10k_ml_ocm *ocm;
struct cnxk_ml_req *req;
uint16_t layer_id;
bool job_enqueued;
bool job_dequeued;
bool locked;
int ret = 0;
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
layer = &model->layer[layer_id];
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
ocm = &cn10k_mldev->ocm;
/* Prepare JD */
req = layer->glow.req;
cn10k_ml_prep_sp_job_descriptor(cnxk_mldev, layer, req, ML_CN10K_JOB_TYPE_MODEL_STOP);
req->cn10k_req.result.error_code = 0x0;
req->cn10k_req.result.user_ptr = NULL;
plt_write64(ML_CNXK_POLL_JOB_START, &req->cn10k_req.status);
plt_wmb();
locked = false;
while (!locked) {
if (plt_spinlock_trylock(&model->lock) != 0) {
if (layer->state == ML_CNXK_LAYER_STATE_LOADED) {
plt_ml_dbg("Layer not started, model_id = %u, layer_id = %u",
model->model_id, layer_id);
plt_spinlock_unlock(&model->lock);
return 1;
}
if (layer->state == ML_CNXK_LAYER_STATE_JOB_ACTIVE) {
plt_err("A slow-path job is active for the layer, model_id = %u, layer_id = %u",
model->model_id, layer_id);
plt_spinlock_unlock(&model->lock);
return -EBUSY;
}
layer->state = ML_CNXK_LAYER_STATE_JOB_ACTIVE;
plt_spinlock_unlock(&model->lock);
locked = true;
}
}
while (layer->glow.ocm_map.ocm_reserved) {
if (plt_spinlock_trylock(&ocm->lock) != 0) {
cn10k_ml_ocm_free_pages(cnxk_mldev, model->model_id, layer_id);
layer->glow.ocm_map.ocm_reserved = false;
layer->glow.ocm_map.tilemask = 0x0;
plt_spinlock_unlock(&ocm->lock);
}
}
job_enqueued = false;
job_dequeued = false;
do {
if (!job_enqueued) {
req->timeout = plt_tsc_cycles() + ML_CNXK_CMD_TIMEOUT * plt_tsc_hz();
job_enqueued =
roc_ml_scratch_enqueue(&cn10k_mldev->roc, &req->cn10k_req.jd);
}
if (job_enqueued && !job_dequeued)
job_dequeued =
roc_ml_scratch_dequeue(&cn10k_mldev->roc, &req->cn10k_req.jd);
if (job_dequeued)
break;
} while (plt_tsc_cycles() < req->timeout);
if (job_dequeued) {
if (plt_read64(&req->cn10k_req.status) == ML_CNXK_POLL_JOB_FINISH) {
if (req->cn10k_req.result.error_code == 0x0)
ret = 0;
else
ret = -1;
}
} else {
roc_ml_scratch_queue_reset(&cn10k_mldev->roc);
ret = -ETIME;
}
locked = false;
while (!locked) {
if (plt_spinlock_trylock(&model->lock) != 0) {
if (ret == 0)
layer->state = ML_CNXK_LAYER_STATE_LOADED;
else
layer->state = ML_CNXK_LAYER_STATE_UNKNOWN;
plt_spinlock_unlock(&model->lock);
locked = true;
}
}
return ret;
}
int
cn10k_ml_model_stop(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model)
{
struct cnxk_ml_layer *layer;
int ret;
layer = &model->layer[0];
ret = cn10k_ml_layer_stop(cnxk_mldev, model->model_id, layer->name);
if (ret != 0) {
plt_err("CN10K Model stop failed, model_id = %u, error = %d", model->model_id, ret);
return ret;
}
cnxk_mldev->nb_models_stopped++;
model->state = ML_CNXK_MODEL_STATE_LOADED;
return 0;
}
int
cn10k_ml_model_params_update(struct cnxk_ml_dev *cnxk_mldev, struct cnxk_ml_model *model,
void *buffer)
{
struct cnxk_ml_layer *layer;
RTE_SET_USED(cnxk_mldev);
if (model->state == ML_CNXK_MODEL_STATE_UNKNOWN)
return -1;
else if (model->state != ML_CNXK_MODEL_STATE_LOADED)
return -EBUSY;
layer = &model->layer[0];
/* Update model weights & bias */
rte_memcpy(layer->glow.addr.wb_load_addr, buffer,
layer->glow.metadata.weights_bias.file_size);
return 0;
}
__rte_hot void
cn10k_ml_result_update(struct cnxk_ml_dev *cnxk_mldev, int qp_id, void *request)
{
union cn10k_ml_error_code *error_code;
struct cn10k_ml_layer_xstats *xstats;
struct cn10k_ml_dev *cn10k_mldev;
struct cn10k_ml_result *result;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
struct cnxk_ml_req *req;
struct cnxk_ml_qp *qp;
struct rte_ml_op *op;
uint64_t hw_latency;
uint64_t fw_latency;
uint16_t model_id;
uint16_t layer_id;
uint16_t idx;
req = (struct cnxk_ml_req *)request;
result = &req->cn10k_req.result;
op = req->op;
if (likely(result->error_code == 0)) {
idx = req->cn10k_req.jd.hdr.model_id;
model_id = cnxk_mldev->index_map[idx].model_id;
layer_id = cnxk_mldev->index_map[idx].layer_id;
model = cnxk_mldev->mldev->data->models[model_id];
layer = &model->layer[layer_id];
if (likely(qp_id >= 0)) {
qp = cnxk_mldev->mldev->data->queue_pairs[qp_id];
qp->stats.dequeued_count++;
xstats = &layer->glow.burst_xstats[qp_id];
} else {
xstats = layer->glow.sync_xstats;
}
if (unlikely(xstats->dequeued_count == xstats->hw_reset_count)) {
xstats->hw_latency_min = UINT64_MAX;
xstats->hw_latency_max = 0;
}
if (unlikely(xstats->dequeued_count == xstats->fw_reset_count)) {
xstats->fw_latency_min = UINT64_MAX;
xstats->fw_latency_max = 0;
}
hw_latency = result->stats.hw_end - result->stats.hw_start;
fw_latency = result->stats.fw_end - result->stats.fw_start - hw_latency;
xstats->hw_latency_tot += hw_latency;
xstats->hw_latency_min = PLT_MIN(xstats->hw_latency_min, hw_latency);
xstats->hw_latency_max = PLT_MAX(xstats->hw_latency_max, hw_latency);
xstats->fw_latency_tot += fw_latency;
xstats->fw_latency_min = PLT_MIN(xstats->fw_latency_min, fw_latency);
xstats->fw_latency_max = PLT_MAX(xstats->fw_latency_max, fw_latency);
xstats->dequeued_count++;
op->impl_opaque = result->error_code;
op->status = RTE_ML_OP_STATUS_SUCCESS;
} else {
if (likely(qp_id >= 0)) {
qp = cnxk_mldev->mldev->data->queue_pairs[qp_id];
qp->stats.dequeue_err_count++;
}
/* Handle driver error */
error_code = (union cn10k_ml_error_code *)&result->error_code;
if (error_code->s.etype == ML_CNXK_ETYPE_DRIVER) {
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
/* Check for exception */
if ((roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_EXCEPTION_SP_C0) !=
0) ||
(roc_ml_reg_read64(&cn10k_mldev->roc, ML_SCRATCH_EXCEPTION_SP_C1) != 0))
error_code->s.stype = ML_CN10K_DRIVER_ERR_EXCEPTION;
else if ((roc_ml_reg_read64(&cn10k_mldev->roc, ML_CORE_INT_LO) != 0) ||
(roc_ml_reg_read64(&cn10k_mldev->roc, ML_CORE_INT_HI) != 0))
error_code->s.stype = ML_CN10K_DRIVER_ERR_FW_ERROR;
else
error_code->s.stype = ML_CN10K_DRIVER_ERR_UNKNOWN;
}
op->impl_opaque = result->error_code;
op->status = RTE_ML_OP_STATUS_ERROR;
}
op->user_ptr = result->user_ptr;
}
__rte_hot void
cn10k_ml_set_error_code(struct cnxk_ml_req *req, uint64_t etype, uint64_t stype)
{
union cn10k_ml_error_code *error_code;
error_code = (union cn10k_ml_error_code *)&req->cn10k_req.result.error_code;
error_code->s.etype = etype;
error_code->s.stype = stype;
}
__rte_hot bool
cn10k_ml_enqueue_single(struct cnxk_ml_dev *cnxk_mldev, struct rte_ml_op *op, uint16_t layer_id,
struct cnxk_ml_qp *qp, uint64_t head)
{
union cn10k_ml_error_code *error_code;
struct cn10k_ml_dev *cn10k_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_queue *queue;
struct cnxk_ml_req *req;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
queue = &qp->queue;
req = &queue->reqs[head];
model = cnxk_mldev->mldev->data->models[op->model_id];
model->set_poll_addr(req);
cn10k_ml_prep_fp_job_descriptor(cnxk_mldev, req, model->layer[layer_id].index,
op->input[0]->addr, op->output[0]->addr, op->nb_batches);
memset(&req->cn10k_req.result, 0, sizeof(struct cn10k_ml_result));
error_code = (union cn10k_ml_error_code *)&req->cn10k_req.result.error_code;
error_code->s.etype = ML_CNXK_ETYPE_UNKNOWN;
req->cn10k_req.result.user_ptr = op->user_ptr;
cnxk_ml_set_poll_ptr(req);
if (unlikely(!cn10k_mldev->ml_jcmdq_enqueue(&cn10k_mldev->roc, &req->cn10k_req.jcmd)))
return false;
req->timeout = plt_tsc_cycles() + queue->wait_cycles;
req->op = op;
return true;
}
__rte_hot int
cn10k_ml_op_error_get(struct rte_ml_dev *dev, struct rte_ml_op *op, struct rte_ml_op_error *error)
{
union cn10k_ml_error_code *error_code;
PLT_SET_USED(dev);
error_code = (union cn10k_ml_error_code *)&op->impl_opaque;
/* Copy sub error message */
if (error_code->s.etype == ML_CNXK_ETYPE_HW_NONFATAL) {
if (error_code->s.stype < PLT_DIM(ml_stype_db_hw_nf))
snprintf(error->message, RTE_ML_STR_MAX, "%s : %s",
ml_etype_db[error_code->s.etype].str,
ml_stype_db_hw_nf[error_code->s.stype].str);
else
snprintf(error->message, RTE_ML_STR_MAX, "%s : UNKNOWN ERROR",
ml_etype_db[error_code->s.etype].str);
} else if (error_code->s.etype == ML_CNXK_ETYPE_DRIVER) {
snprintf(error->message, RTE_ML_STR_MAX, "%s : %s",
ml_etype_db[error_code->s.etype].str,
ml_stype_db_driver[error_code->s.stype].str);
} else {
snprintf(error->message, RTE_ML_STR_MAX, "%s",
ml_etype_db[error_code->s.etype].str);
}
error->errcode = error_code->u64;
return 0;
}
__rte_hot int
cn10k_ml_inference_sync(void *device, uint16_t index, void *input, void *output,
uint16_t nb_batches)
{
union cn10k_ml_error_code *error_code;
struct cn10k_ml_dev *cn10k_mldev;
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
struct cnxk_ml_req *req;
struct rte_ml_op op;
uint16_t model_id;
uint16_t layer_id;
bool timeout;
int ret = 0;
cnxk_mldev = (struct cnxk_ml_dev *)device;
cn10k_mldev = &cnxk_mldev->cn10k_mldev;
model_id = cnxk_mldev->index_map[index].model_id;
layer_id = cnxk_mldev->index_map[index].layer_id;
model = cnxk_mldev->mldev->data->models[model_id];
layer = &model->layer[layer_id];
req = layer->glow.req;
op.model_id = index;
op.impl_opaque = 0;
cn10k_ml_set_poll_addr(req);
cn10k_ml_prep_fp_job_descriptor(cnxk_mldev, req, index, input, output, nb_batches);
memset(&req->cn10k_req.result, 0, sizeof(struct cn10k_ml_result));
error_code = (union cn10k_ml_error_code *)&req->cn10k_req.result.error_code;
error_code->s.etype = ML_CNXK_ETYPE_UNKNOWN;
req->cn10k_req.result.user_ptr = NULL;
cnxk_ml_set_poll_ptr(req);
req->cn10k_req.jcmd.w1.s.jobptr = PLT_U64_CAST(&req->cn10k_req.jd);
timeout = true;
req->timeout = plt_tsc_cycles() + ML_CNXK_CMD_TIMEOUT * plt_tsc_hz();
do {
if (cn10k_mldev->ml_jcmdq_enqueue(&cn10k_mldev->roc, &req->cn10k_req.jcmd)) {
req->op = &op;
timeout = false;
break;
}
} while (plt_tsc_cycles() < req->timeout);
if (timeout) {
ret = -EBUSY;
goto error_enqueue;
}
timeout = true;
do {
if (cnxk_ml_get_poll_ptr(req) == ML_CNXK_POLL_JOB_FINISH) {
timeout = false;
break;
}
} while (plt_tsc_cycles() < req->timeout);
if (timeout)
ret = -ETIME;
else
cn10k_ml_result_update(cnxk_mldev, -1, req);
error_enqueue:
return ret;
}
int
cn10k_ml_io_alloc(void *device, uint16_t model_id, const char *layer_name, uint64_t **input_qbuffer,
uint64_t **output_qbuffer)
{
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
struct cnxk_ml_layer *layer;
char str[RTE_MEMZONE_NAMESIZE];
const struct plt_memzone *mz;
uint64_t output_size;
uint64_t input_size;
uint16_t layer_id;
int ret;
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
layer = &model->layer[layer_id];
input_size = PLT_ALIGN_CEIL(layer->info.total_input_sz_q, ML_CN10K_ALIGN_SIZE);
output_size = PLT_ALIGN_CEIL(layer->info.total_output_sz_q, ML_CN10K_ALIGN_SIZE);
sprintf(str, "cn10k_ml_io_mz_%u_%u", model_id, layer_id);
mz = plt_memzone_reserve_aligned(str, input_size + output_size, 0, ML_CN10K_ALIGN_SIZE);
if (mz == NULL) {
plt_err("io_alloc failed: Unable to allocate memory: model_id = %u, layer_name = %s",
model_id, layer_name);
return -ENOMEM;
}
*input_qbuffer = mz->addr;
*output_qbuffer = PLT_PTR_ADD(mz->addr, input_size);
return 0;
}
int
cn10k_ml_io_free(void *device, uint16_t model_id, const char *layer_name)
{
struct cnxk_ml_dev *cnxk_mldev;
struct cnxk_ml_model *model;
char str[RTE_MEMZONE_NAMESIZE];
const struct plt_memzone *mz;
uint16_t layer_id;
int ret;
cnxk_mldev = (struct cnxk_ml_dev *)device;
if (cnxk_mldev == NULL) {
plt_err("Invalid device = %p", device);
return -EINVAL;
}
model = cnxk_mldev->mldev->data->models[model_id];
if (model == NULL) {
plt_err("Invalid model_id = %u", model_id);
return -EINVAL;
}
ret = cn10k_ml_model_get_layer_id(model, layer_name, &layer_id);
if (ret != 0)
return ret;
sprintf(str, "cn10k_ml_io_mz_%u_%u", model_id, layer_id);
mz = plt_memzone_lookup(str);
if (mz == NULL) {
plt_err("io_free failed: Memzone not found: model_id = %u, layer_name = %s",
model_id, layer_name);
return -EINVAL;
}
return plt_memzone_free(mz);
}
int
cn10k_ml_malloc(const char *name, size_t size, uint32_t align, void **addr)
{
const struct plt_memzone *mz;
mz = plt_memzone_reserve_aligned(name, size, 0, align);
if (mz == NULL) {
plt_err("ml_malloc failed: Unable to allocate memory: name = %s", name);
return -ENOMEM;
}
*addr = mz->addr;
return 0;
}
int
cn10k_ml_free(const char *name)
{
const struct plt_memzone *mz;
mz = plt_memzone_lookup(name);
if (mz == NULL) {
plt_err("ml_free failed: Memzone not found: name = %s", name);
return -EINVAL;
}
return plt_memzone_free(mz);
}