Each channel gets its own DMA resources, cleanup and "bufferpool" tasks, and a separate cleanup taskqueue to isolate channels operation as much as possible to avoid various kernel panics under heavy network load. As a side-effect of this work, dpaa2_buf structure is simplified and all of the functions to re-seed those buffers are gathered now in dpaa2_buf.h and .c files; functions to work with channels are extracted into dpaa2_channel.h and .c files as well. Reported by: dch Reviewed by: bz Approved by: bz (mentor) MFC after: 1 week Differential Revision: https://reviews.freebsd.org/D41296
1165 lines
33 KiB
C
1165 lines
33 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
|
|
*
|
|
* Copyright © 2014-2016 Freescale Semiconductor, Inc.
|
|
* Copyright © 2016-2019 NXP
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright notice,
|
|
* this list of conditions and the following disclaimer.
|
|
*
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* 3. Neither the name of the copyright holder nor the names of its
|
|
* contributors may be used to endorse or promote products derived from this
|
|
* software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
|
|
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
* POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* Original source file obtained from:
|
|
* drivers/soc/fsl/dpio/qbman-portal.c
|
|
*
|
|
* Commit: 4c86114194e644b6da9107d75910635c9e87179e
|
|
* Repository: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
|
|
*/
|
|
|
|
/*
|
|
* Copyright © 2021-2022 Dmitry Salychev
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
/*
|
|
* DPAA2 QBMan software portal.
|
|
*/
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/bus.h>
|
|
#include <sys/rman.h>
|
|
#include <sys/module.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/time.h>
|
|
#include <sys/types.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/lock.h>
|
|
|
|
#include <machine/bus.h>
|
|
#include <machine/resource.h>
|
|
#include <machine/atomic.h>
|
|
|
|
#include "pcib_if.h"
|
|
#include "pci_if.h"
|
|
|
|
#include "dpaa2_swp.h"
|
|
#include "dpaa2_mc.h"
|
|
#include "dpaa2_bp.h"
|
|
|
|
#define CMD_SPIN_TIMEOUT 100u /* us */
|
|
#define CMD_SPIN_ATTEMPTS 2000u /* 200 ms max. */
|
|
|
|
#define CMD_VERB_MASK 0x7Fu
|
|
|
|
/* Shifts in the VERB byte of the enqueue command descriptor. */
|
|
#define ENQ_CMD_ORP_ENABLE_SHIFT 2
|
|
#define ENQ_CMD_IRQ_ON_DISPATCH_SHIFT 3
|
|
#define ENQ_CMD_TARGET_TYPE_SHIFT 4
|
|
#define ENQ_CMD_DCA_EN_SHIFT 7
|
|
/* VERB byte options of the enqueue command descriptor. */
|
|
#define ENQ_CMD_EMPTY 0u
|
|
#define ENQ_CMD_RESPONSE_ALWAYS 1u
|
|
#define ENQ_CMD_REJECTS_TO_FQ 2u
|
|
|
|
#define ENQ_DESC_FD_OFFSET 32u
|
|
|
|
#define ENQ_DCA_IDXMASK 0x0Fu
|
|
#define ENQ_FLAG_DCA (1ull << 31)
|
|
|
|
/* QBMan portal command codes. */
|
|
#define CMDID_SWP_MC_ACQUIRE 0x30
|
|
#define CMDID_SWP_BP_QUERY 0x32
|
|
#define CMDID_SWP_WQCHAN_CONFIGURE 0x46
|
|
|
|
/* QBMan portal command result codes. */
|
|
#define QBMAN_CMD_RC_OK 0xF0
|
|
|
|
/* SDQCR attribute codes */
|
|
#define QB_SDQCR_FC_SHIFT 29u
|
|
#define QB_SDQCR_FC_MASK 0x1u
|
|
#define QB_SDQCR_DCT_SHIFT 24u
|
|
#define QB_SDQCR_DCT_MASK 0x3u
|
|
#define QB_SDQCR_TOK_SHIFT 16u
|
|
#define QB_SDQCR_TOK_MASK 0xFFu
|
|
#define QB_SDQCR_SRC_SHIFT 0u
|
|
#define QB_SDQCR_SRC_MASK 0xFFFFu
|
|
|
|
/* Shifts in the VERB byte of the volatile dequeue command. */
|
|
#define QB_VDQCR_VERB_DCT0_SHIFT 0
|
|
#define QB_VDQCR_VERB_DCT1_SHIFT 1
|
|
#define QB_VDQCR_VERB_DT0_SHIFT 2
|
|
#define QB_VDQCR_VERB_DT1_SHIFT 3
|
|
#define QB_VDQCR_VERB_RLS_SHIFT 4
|
|
#define QB_VDQCR_VERB_WAE_SHIFT 5
|
|
#define QB_VDQCR_VERB_RAD_SHIFT 6
|
|
|
|
/* Maximum timeout period for the DQRR interrupt. */
|
|
#define DQRR_MAX_ITP 4096u
|
|
#define DQRR_PI_MASK 0x0Fu
|
|
|
|
/* Release Array Allocation register helpers. */
|
|
#define RAR_IDX(rar) ((rar) & 0x7u)
|
|
#define RAR_VB(rar) ((rar) & 0x80u)
|
|
#define RAR_SUCCESS(rar) ((rar) & 0x100u)
|
|
|
|
MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal");
|
|
|
|
enum qbman_sdqcr_dct {
|
|
qbman_sdqcr_dct_null = 0,
|
|
qbman_sdqcr_dct_prio_ics,
|
|
qbman_sdqcr_dct_active_ics,
|
|
qbman_sdqcr_dct_active
|
|
};
|
|
|
|
enum qbman_sdqcr_fc {
|
|
qbman_sdqcr_fc_one = 0,
|
|
qbman_sdqcr_fc_up_to_3 = 1
|
|
};
|
|
|
|
/* Routines to execute software portal commands. */
|
|
static int dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *,
|
|
struct dpaa2_swp_cmd *, struct dpaa2_swp_rsp *, uint8_t);
|
|
static int dpaa2_swp_exec_br_command(struct dpaa2_swp *, struct dpaa2_swp_cmd *,
|
|
uint32_t);
|
|
static int dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *,
|
|
struct dpaa2_swp_cmd *);
|
|
|
|
/* Management Commands helpers. */
|
|
static int dpaa2_swp_send_mgmt_command(struct dpaa2_swp *,
|
|
struct dpaa2_swp_cmd *, uint8_t);
|
|
static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *,
|
|
struct dpaa2_swp_rsp *);
|
|
|
|
/* Helper subroutines. */
|
|
static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t);
|
|
|
|
int
|
|
dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
|
|
uint16_t flags)
|
|
{
|
|
struct dpaa2_swp *p;
|
|
uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */
|
|
|
|
if (!swp || !desc)
|
|
return (DPAA2_SWP_STAT_EINVAL);
|
|
|
|
p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP,
|
|
flags & DPAA2_SWP_NOWAIT_ALLOC
|
|
? (M_NOWAIT | M_ZERO)
|
|
: (M_WAITOK | M_ZERO));
|
|
if (!p)
|
|
return (DPAA2_SWP_STAT_NO_MEMORY);
|
|
|
|
mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF);
|
|
|
|
p->cfg.mem_backed = false;
|
|
p->cfg.writes_cinh = true;
|
|
|
|
p->desc = desc;
|
|
p->flags = flags;
|
|
p->mc.valid_bit = DPAA2_SWP_VALID_BIT;
|
|
p->mr.valid_bit = DPAA2_SWP_VALID_BIT;
|
|
|
|
/* FIXME: Memory-backed mode doesn't work now. Why? */
|
|
p->cena_res = desc->cena_res;
|
|
p->cena_map = desc->cena_map;
|
|
p->cinh_res = desc->cinh_res;
|
|
p->cinh_map = desc->cinh_map;
|
|
|
|
/* Static Dequeue Command Register configuration. */
|
|
p->sdq = 0;
|
|
p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
|
|
p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
|
|
p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
|
|
|
|
/* Volatile Dequeue Command configuration. */
|
|
p->vdq.valid_bit = DPAA2_SWP_VALID_BIT;
|
|
|
|
/* Dequeue Response Ring configuration */
|
|
p->dqrr.next_idx = 0;
|
|
p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT;
|
|
if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) {
|
|
p->dqrr.ring_size = 4;
|
|
p->dqrr.reset_bug = 1;
|
|
} else {
|
|
p->dqrr.ring_size = 8;
|
|
p->dqrr.reset_bug = 0;
|
|
}
|
|
|
|
if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) {
|
|
reg = dpaa2_swp_set_cfg(
|
|
p->dqrr.ring_size, /* max. entries QMan writes to DQRR */
|
|
1, /* writes enabled in the CINH memory only */
|
|
0, /* EQCR_CI stashing threshold */
|
|
3, /* RPM: RCR in array mode */
|
|
2, /* DCM: Discrete consumption ack */
|
|
2, /* EPM: EQCR in ring mode (FIFO) */
|
|
1, /* mem stashing drop enable enable */
|
|
1, /* mem stashing priority enable */
|
|
1, /* mem stashing enable */
|
|
1, /* dequeue stashing priority enable */
|
|
0, /* dequeue stashing enable enable */
|
|
0 /* EQCR_CI stashing priority enable */
|
|
);
|
|
reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
|
|
} else {
|
|
bus_set_region_4(p->cena_map, 0, 0,
|
|
rman_get_size(p->cena_res) / 4);
|
|
|
|
reg = dpaa2_swp_set_cfg(
|
|
p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ /* DQRR_MF */
|
|
1, /* writes enabled in the CINH memory only */ /* WN */
|
|
0, /* EQCR_CI stashing is disabled */ /* EST */
|
|
3, /* RPM: RCR in array mode */ /* RPM */
|
|
2, /* DCM: Discrete consumption ack */ /* DCM */
|
|
2, /* EPM: EQCR in ring mode (FIFO) */ /* EPM */
|
|
1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */ /* SD */
|
|
1, /* Dequeued frame data, annotation, and FQ context stashing priority */ /* SP */
|
|
1, /* Dequeued frame data, annotation, and FQ context stashing enable */ /* SE */
|
|
1, /* Dequeue response ring (DQRR) entry stashing priority */ /* DP */
|
|
0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */ /* DE */
|
|
0 /* EQCR_CI stashing priority */ /* EP */
|
|
);
|
|
/* TODO: Switch to memory-backed mode. */
|
|
reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
|
|
}
|
|
dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_CFG, reg);
|
|
reg = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_CFG);
|
|
if (!reg) {
|
|
free(p, M_DPAA2_SWP);
|
|
return (DPAA2_SWP_STAT_PORTAL_DISABLED);
|
|
}
|
|
|
|
/*
|
|
* Static Dequeue Command Register needs to be initialized to 0 when no
|
|
* channels are being dequeued from or else the QMan HW will indicate an
|
|
* error. The values that were calculated above will be applied when
|
|
* dequeues from a specific channel are enabled.
|
|
*/
|
|
dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_SDQCR, 0);
|
|
|
|
p->eqcr.pi_ring_size = 8;
|
|
/* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */
|
|
/* p->eqcr.pi_ring_size = 32; */
|
|
|
|
for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
|
|
p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
|
|
|
|
eqcr_pi = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_PI);
|
|
p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
|
|
p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT;
|
|
p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI)
|
|
& p->eqcr.pi_ci_mask;
|
|
p->eqcr.available = p->eqcr.pi_ring_size;
|
|
|
|
/* TODO: sysctl(9) for the IRQ timeout? */
|
|
/* Initialize the portal with an IRQ threshold and timeout of 120us. */
|
|
dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 120);
|
|
|
|
*swp = p;
|
|
|
|
return (0);
|
|
}
|
|
|
|
void
|
|
dpaa2_swp_free_portal(struct dpaa2_swp *swp)
|
|
{
|
|
uint16_t flags;
|
|
|
|
KASSERT(swp != NULL, ("%s: swp is NULL", __func__));
|
|
|
|
DPAA2_SWP_LOCK(swp, &flags);
|
|
swp->flags |= DPAA2_SWP_DESTROYED;
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
|
|
/* Let threads stop using this portal. */
|
|
DELAY(DPAA2_SWP_TIMEOUT);
|
|
|
|
mtx_destroy(&swp->lock);
|
|
free(swp, M_DPAA2_SWP);
|
|
}
|
|
|
|
uint32_t
|
|
dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm,
|
|
uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp, int de, int ep)
|
|
{
|
|
return (
|
|
max_fill << DPAA2_SWP_CFG_DQRR_MF_SHIFT |
|
|
est << DPAA2_SWP_CFG_EST_SHIFT |
|
|
wn << DPAA2_SWP_CFG_WN_SHIFT |
|
|
rpm << DPAA2_SWP_CFG_RPM_SHIFT |
|
|
dcm << DPAA2_SWP_CFG_DCM_SHIFT |
|
|
epm << DPAA2_SWP_CFG_EPM_SHIFT |
|
|
sd << DPAA2_SWP_CFG_SD_SHIFT |
|
|
sp << DPAA2_SWP_CFG_SP_SHIFT |
|
|
se << DPAA2_SWP_CFG_SE_SHIFT |
|
|
dp << DPAA2_SWP_CFG_DP_SHIFT |
|
|
de << DPAA2_SWP_CFG_DE_SHIFT |
|
|
ep << DPAA2_SWP_CFG_EP_SHIFT
|
|
);
|
|
}
|
|
|
|
/* Read/write registers of a software portal. */
|
|
|
|
void
|
|
dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v)
|
|
{
|
|
bus_write_4(swp->cinh_map, o, v);
|
|
}
|
|
|
|
uint32_t
|
|
dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o)
|
|
{
|
|
return (bus_read_4(swp->cinh_map, o));
|
|
}
|
|
|
|
/* Helper routines. */
|
|
|
|
/**
|
|
* @brief Set enqueue descriptor without Order Point Record ID.
|
|
*
|
|
* ed: Enqueue descriptor.
|
|
* resp_always: Enqueue with response always (1); FD from a rejected enqueue
|
|
* will be returned on a FQ (0).
|
|
*/
|
|
void
|
|
dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc *ed, bool resp_always)
|
|
{
|
|
ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT);
|
|
if (resp_always)
|
|
ed->verb |= ENQ_CMD_RESPONSE_ALWAYS;
|
|
else
|
|
ed->verb |= ENQ_CMD_REJECTS_TO_FQ;
|
|
}
|
|
|
|
/**
|
|
* @brief Set FQ of the enqueue descriptor.
|
|
*/
|
|
void
|
|
dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc *ed, uint32_t fqid)
|
|
{
|
|
ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT);
|
|
ed->tgtid = fqid;
|
|
}
|
|
|
|
/**
|
|
* @brief Enable interrupts for a software portal.
|
|
*/
|
|
void
|
|
dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask)
|
|
{
|
|
if (swp != NULL)
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask);
|
|
}
|
|
|
|
/**
|
|
* @brief Return the value in the SWP_IER register.
|
|
*/
|
|
uint32_t
|
|
dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp)
|
|
{
|
|
if (swp != NULL)
|
|
return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER);
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* @brief Return the value in the SWP_ISR register.
|
|
*/
|
|
uint32_t
|
|
dpaa2_swp_read_intr_status(struct dpaa2_swp *swp)
|
|
{
|
|
if (swp != NULL)
|
|
return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR);
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* @brief Clear SWP_ISR register according to the given mask.
|
|
*/
|
|
void
|
|
dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask)
|
|
{
|
|
if (swp != NULL)
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask);
|
|
}
|
|
|
|
/**
|
|
* @brief Enable or disable push dequeue.
|
|
*
|
|
* swp: the software portal object
|
|
* chan_idx: the channel index (0 to 15)
|
|
* en: enable or disable push dequeue
|
|
*/
|
|
void
|
|
dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en)
|
|
{
|
|
uint16_t dqsrc;
|
|
|
|
if (swp != NULL) {
|
|
if (chan_idx > 15u) {
|
|
device_printf(swp->desc->dpio_dev, "channel index "
|
|
"should be <= 15: chan_idx=%d\n", chan_idx);
|
|
return;
|
|
}
|
|
|
|
if (en)
|
|
swp->sdq |= 1 << chan_idx;
|
|
else
|
|
swp->sdq &= ~(1 << chan_idx);
|
|
/*
|
|
* Read make the complete src map. If no channels are enabled
|
|
* the SDQCR must be 0 or else QMan will assert errors.
|
|
*/
|
|
dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) &
|
|
DPAA2_SDQCR_SRC_MASK;
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0
|
|
? swp->sdq : 0);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Set new IRQ coalescing values.
|
|
*
|
|
* swp: The software portal object.
|
|
* threshold: Threshold for DQRR interrupt generation. The DQRR interrupt
|
|
* asserts when the ring contains greater than "threshold" entries.
|
|
* holdoff: DQRR interrupt holdoff (timeout) period in us.
|
|
*/
|
|
int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold,
|
|
uint32_t holdoff)
|
|
{
|
|
uint32_t itp; /* Interrupt Timeout Period */
|
|
|
|
if (swp == NULL)
|
|
return (EINVAL);
|
|
|
|
/*
|
|
* Convert "holdoff" value from us to 256 QBMAN clock cycles
|
|
* increments. This depends on the QBMAN internal frequency.
|
|
*/
|
|
itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio;
|
|
if (itp > DQRR_MAX_ITP)
|
|
itp = DQRR_MAX_ITP;
|
|
if (threshold >= swp->dqrr.ring_size)
|
|
threshold = swp->dqrr.ring_size - 1;
|
|
|
|
swp->dqrr.irq_threshold = threshold;
|
|
swp->dqrr.irq_itp = itp;
|
|
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold);
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Software portal commands.
|
|
*/
|
|
|
|
/**
|
|
* @brief Configure the channel data availability notification (CDAN)
|
|
* in a particular WQ channel.
|
|
*/
|
|
int
|
|
dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id,
|
|
uint8_t we_mask, bool cdan_en, uint64_t ctx)
|
|
{
|
|
/* NOTE: 64 bytes command. */
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t result; /* in response only! */
|
|
uint16_t chan_id;
|
|
uint8_t we;
|
|
uint8_t ctrl;
|
|
uint16_t _reserved2;
|
|
uint64_t ctx;
|
|
uint8_t _reserved3[48];
|
|
} cmd = {0};
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t result;
|
|
uint16_t chan_id;
|
|
uint8_t _reserved[60];
|
|
} rsp;
|
|
int error;
|
|
|
|
if (swp == NULL)
|
|
return (EINVAL);
|
|
|
|
cmd.chan_id = chan_id;
|
|
cmd.we = we_mask;
|
|
cmd.ctrl = cdan_en ? 1u : 0u;
|
|
cmd.ctx = ctx;
|
|
|
|
error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
|
|
(struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_WQCHAN_CONFIGURE);
|
|
if (error)
|
|
return (error);
|
|
|
|
if (rsp.result != QBMAN_CMD_RC_OK) {
|
|
device_printf(swp->desc->dpio_dev, "WQ channel configuration "
|
|
"error: channel_id=%d, result=0x%02x\n", chan_id,
|
|
rsp.result);
|
|
return (EIO);
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* @brief Query current configuration/state of the buffer pool.
|
|
*/
|
|
int
|
|
dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid,
|
|
struct dpaa2_bp_conf *conf)
|
|
{
|
|
/* NOTE: 64 bytes command. */
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t _reserved1;
|
|
uint16_t bpid;
|
|
uint8_t _reserved2[60];
|
|
} cmd = {0};
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t result;
|
|
uint32_t _reserved1;
|
|
uint8_t bdi;
|
|
uint8_t state;
|
|
uint32_t fill;
|
|
/* TODO: Support the other fields as well. */
|
|
uint8_t _reserved2[52];
|
|
} rsp;
|
|
int error;
|
|
|
|
if (swp == NULL || conf == NULL)
|
|
return (EINVAL);
|
|
|
|
cmd.bpid = bpid;
|
|
|
|
error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
|
|
(struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_BP_QUERY);
|
|
if (error)
|
|
return (error);
|
|
|
|
if (rsp.result != QBMAN_CMD_RC_OK) {
|
|
device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, "
|
|
"result=0x%02x\n", bpid, rsp.result);
|
|
return (EIO);
|
|
}
|
|
|
|
conf->bdi = rsp.bdi;
|
|
conf->state = rsp.state;
|
|
conf->free_bufn = rsp.fill;
|
|
|
|
return (0);
|
|
}
|
|
|
|
int
|
|
dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf,
|
|
uint32_t buf_num)
|
|
{
|
|
/* NOTE: 64 bytes command. */
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t _reserved1;
|
|
uint16_t bpid;
|
|
uint32_t _reserved2;
|
|
uint64_t buf[DPAA2_SWP_BUFS_PER_CMD];
|
|
} cmd = {0};
|
|
int error;
|
|
|
|
if (swp == NULL || buf == NULL || buf_num == 0u ||
|
|
buf_num > DPAA2_SWP_BUFS_PER_CMD)
|
|
return (EINVAL);
|
|
|
|
for (uint32_t i = 0; i < buf_num; i++)
|
|
cmd.buf[i] = buf[i];
|
|
cmd.bpid = bpid;
|
|
cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */
|
|
|
|
error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd,
|
|
buf_num);
|
|
if (error) {
|
|
device_printf(swp->desc->dpio_dev, "buffers release command "
|
|
"failed\n");
|
|
return (error);
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
int
|
|
dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq,
|
|
uint32_t *idx)
|
|
{
|
|
struct resource_map *map = swp->cinh_map;
|
|
struct dpaa2_swp_rsp *rsp = (struct dpaa2_swp_rsp *) dq;
|
|
uint32_t verb, pi; /* producer index */
|
|
uint32_t offset = swp->cfg.mem_backed
|
|
? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx)
|
|
: DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx);
|
|
|
|
if (swp == NULL || dq == NULL)
|
|
return (EINVAL);
|
|
|
|
/*
|
|
* Before using valid-bit to detect if something is there, we have to
|
|
* handle the case of the DQRR reset bug...
|
|
*/
|
|
if (swp->dqrr.reset_bug) {
|
|
/*
|
|
* We pick up new entries by cache-inhibited producer index,
|
|
* which means that a non-coherent mapping would require us to
|
|
* invalidate and read *only* once that PI has indicated that
|
|
* there's an entry here. The first trip around the DQRR ring
|
|
* will be much less efficient than all subsequent trips around
|
|
* it...
|
|
*/
|
|
pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK;
|
|
|
|
/* There are new entries if pi != next_idx */
|
|
if (pi == swp->dqrr.next_idx)
|
|
return (ENOENT);
|
|
|
|
/*
|
|
* If next_idx is/was the last ring index, and 'pi' is
|
|
* different, we can disable the workaround as all the ring
|
|
* entries have now been DMA'd to so valid-bit checking is
|
|
* repaired.
|
|
*
|
|
* NOTE: This logic needs to be based on next_idx (which
|
|
* increments one at a time), rather than on pi (which
|
|
* can burst and wrap-around between our snapshots of it).
|
|
*/
|
|
if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1))
|
|
swp->dqrr.reset_bug = 0;
|
|
}
|
|
|
|
verb = bus_read_4(map, offset);
|
|
if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit)
|
|
return (ENOENT);
|
|
|
|
/* Read dequeue response message. */
|
|
for (int i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
|
|
rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
|
|
|
|
/* Return index of the current entry (if requested). */
|
|
if (idx != NULL)
|
|
*idx = swp->dqrr.next_idx;
|
|
|
|
/*
|
|
* There's something there. Move "next_idx" attention to the next ring
|
|
* entry before returning what we found.
|
|
*/
|
|
swp->dqrr.next_idx++;
|
|
swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */
|
|
if (swp->dqrr.next_idx == 0u)
|
|
swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT;
|
|
|
|
return (0);
|
|
}
|
|
|
|
int
|
|
dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf,
|
|
uint32_t frames_n)
|
|
{
|
|
/* NOTE: 64 bytes command. */
|
|
struct __packed {
|
|
uint8_t verb;
|
|
uint8_t numf;
|
|
uint8_t tok;
|
|
uint8_t _reserved;
|
|
uint32_t dq_src;
|
|
uint64_t rsp_addr;
|
|
uint64_t _reserved1[6];
|
|
} cmd = {0};
|
|
struct dpaa2_dq *msg;
|
|
uint16_t flags;
|
|
int i, error;
|
|
|
|
KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__));
|
|
KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__));
|
|
|
|
cmd.numf = frames_n - 1;
|
|
cmd.tok = DPAA2_SWP_VDQCR_TOKEN;
|
|
cmd.dq_src = chan_id;
|
|
cmd.rsp_addr = (uint64_t)buf->paddr;
|
|
|
|
/* Dequeue command type */
|
|
cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT);
|
|
cmd.verb |= (1 << QB_VDQCR_VERB_DCT1_SHIFT);
|
|
/* Dequeue from a specific software portal channel (ID's in DQ_SRC). */
|
|
cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT);
|
|
cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT);
|
|
/* Write the response to this command into memory (at the RSP_ADDR). */
|
|
cmd.verb |= (1 << QB_VDQCR_VERB_RLS_SHIFT);
|
|
/* Response writes won't attempt to allocate into a cache. */
|
|
cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
|
|
/* Allow the FQ to remain active in the portal after dequeue. */
|
|
cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
|
|
|
|
DPAA2_SWP_LOCK(swp, &flags);
|
|
if (flags & DPAA2_SWP_DESTROYED) {
|
|
/* Terminate operation if portal is destroyed. */
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (ENOENT);
|
|
}
|
|
|
|
error = dpaa2_swp_exec_vdc_command_locked(swp,
|
|
(struct dpaa2_swp_cmd *) &cmd);
|
|
if (error != 0) {
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (error);
|
|
}
|
|
|
|
/* Let's sync before reading VDQ response from QBMan. */
|
|
bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);
|
|
|
|
/* Read VDQ response from QBMan. */
|
|
msg = (struct dpaa2_dq *)buf->vaddr;
|
|
for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
|
|
if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) &&
|
|
(msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) {
|
|
/* Reset token. */
|
|
msg->fdr.desc.tok = 0;
|
|
break;
|
|
}
|
|
DELAY(CMD_SPIN_TIMEOUT);
|
|
}
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
|
|
/* Return an error on expired timeout. */
|
|
return (i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0);
|
|
}
|
|
|
|
/**
|
|
* @brief Issue a command to enqueue a frame using one enqueue descriptor.
|
|
*
|
|
* swp: Software portal used to send this command to.
|
|
* ed: Enqueue command descriptor.
|
|
* fd: Frame descriptor to enqueue.
|
|
*/
|
|
int
|
|
dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
|
|
struct dpaa2_fd *fd)
|
|
{
|
|
uint32_t flags = 0;
|
|
int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1);
|
|
|
|
return (rc >= 0 ? 0 : EBUSY);
|
|
}
|
|
|
|
/**
|
|
* @brief Issue a command to enqueue frames using one enqueue descriptor.
|
|
*
|
|
* swp: Software portal used to send this command to.
|
|
* ed: Enqueue command descriptor.
|
|
* fd: Frame descriptor to enqueue.
|
|
* flags: Table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL.
|
|
* frames_n: Number of FDs to enqueue.
|
|
*
|
|
* NOTE: Enqueue command (64 bytes): 32 (eq. descriptor) + 32 (frame descriptor).
|
|
*/
|
|
int
|
|
dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
|
|
struct dpaa2_fd *fd, uint32_t *flags, int frames_n)
|
|
{
|
|
const uint8_t *ed_pdat8 = (const uint8_t *) ed;
|
|
const uint32_t *ed_pdat32 = (const uint32_t *) ed;
|
|
const uint64_t *ed_pdat64 = (const uint64_t *) ed;
|
|
const uint64_t *fd_pdat64 = (const uint64_t *) fd;
|
|
struct resource_map *map;
|
|
uint32_t eqcr_ci, eqcr_pi; /* EQCR consumer/producer index */
|
|
uint32_t half_mask, full_mask, val, ci_offset;
|
|
uint16_t swp_flags;
|
|
int num_enq = 0;
|
|
|
|
if (swp == NULL || ed == NULL || fd == NULL || flags == NULL ||
|
|
frames_n == 0)
|
|
return (EINVAL);
|
|
|
|
DPAA2_SWP_LOCK(swp, &swp_flags);
|
|
if (swp_flags & DPAA2_SWP_DESTROYED) {
|
|
/* Terminate operation if portal is destroyed. */
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (ENOENT);
|
|
}
|
|
|
|
map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
|
|
ci_offset = swp->cfg.mem_backed
|
|
? DPAA2_SWP_CENA_EQCR_CI_MEMBACK
|
|
: DPAA2_SWP_CENA_EQCR_CI;
|
|
|
|
half_mask = swp->eqcr.pi_ci_mask >> 1;
|
|
full_mask = swp->eqcr.pi_ci_mask;
|
|
|
|
if (swp->eqcr.available == 0) {
|
|
val = dpaa2_swp_read_reg(swp, ci_offset);
|
|
eqcr_ci = swp->eqcr.ci;
|
|
swp->eqcr.ci = val & full_mask;
|
|
|
|
swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size,
|
|
eqcr_ci, swp->eqcr.ci);
|
|
|
|
if (swp->eqcr.available == 0) {
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (0);
|
|
}
|
|
}
|
|
|
|
eqcr_pi = swp->eqcr.pi;
|
|
num_enq = swp->eqcr.available < frames_n
|
|
? swp->eqcr.available : frames_n;
|
|
swp->eqcr.available -= num_enq;
|
|
|
|
KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size,
|
|
("%s: unexpected num_enq=%d", __func__, num_enq));
|
|
KASSERT(swp->eqcr.available >= 0 &&
|
|
swp->eqcr.available <= swp->eqcr.pi_ring_size,
|
|
("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available));
|
|
|
|
/* Fill in the EQCR ring. */
|
|
for (int i = 0; i < num_enq; i++) {
|
|
/* Write enq. desc. without the VERB, DCA, SEQNUM and OPRID. */
|
|
for (int j = 1; j <= 3; j++)
|
|
bus_write_8(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
|
|
sizeof(uint64_t) * j, ed_pdat64[j]);
|
|
/* Write OPRID. */
|
|
bus_write_4(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + sizeof(uint32_t),
|
|
ed_pdat32[1]);
|
|
/* Write DCA and SEQNUM without VERB byte. */
|
|
for (int j = 1; j <= 3; j++)
|
|
bus_write_1(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
|
|
sizeof(uint8_t) * j, ed_pdat8[j]);
|
|
|
|
/* Write frame descriptor. */
|
|
for (int j = 0; j <= 3; j++)
|
|
bus_write_8(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
|
|
ENQ_DESC_FD_OFFSET +
|
|
sizeof(uint64_t) * j, fd_pdat64[j]);
|
|
eqcr_pi++;
|
|
}
|
|
|
|
wmb();
|
|
|
|
/* Write the VERB byte of enqueue descriptor. */
|
|
eqcr_pi = swp->eqcr.pi;
|
|
for (int i = 0; i < num_enq; i++) {
|
|
bus_write_1(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask),
|
|
ed_pdat8[0] | swp->eqcr.pi_vb);
|
|
|
|
if (flags && (flags[i] & ENQ_FLAG_DCA)) {
|
|
/* Update DCA byte. */
|
|
bus_write_1(map,
|
|
DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1,
|
|
(1 << ENQ_CMD_DCA_EN_SHIFT) |
|
|
(flags[i] & ENQ_DCA_IDXMASK));
|
|
}
|
|
eqcr_pi++;
|
|
if (!(eqcr_pi & half_mask))
|
|
swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT;
|
|
}
|
|
swp->eqcr.pi = eqcr_pi & full_mask;
|
|
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
|
|
return (num_enq);
|
|
}
|
|
|
|
static int
|
|
dpaa2_swp_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
|
|
{
|
|
/* 'first' is included, 'last' is excluded */
|
|
return ((first <= last)
|
|
? (last - first) : ((2 * ringsize) - (first - last)));
|
|
}
|
|
|
|
/**
|
|
* @brief Execute Buffer Release Command (BRC).
|
|
*/
|
|
static int
|
|
dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
|
|
uint32_t buf_num)
|
|
{
|
|
struct __packed with_verb {
|
|
uint8_t verb;
|
|
uint8_t _reserved[63];
|
|
} *c;
|
|
const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
|
|
const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
|
|
struct resource_map *map;
|
|
uint32_t offset, rar; /* Release Array Allocation register */
|
|
uint16_t flags;
|
|
|
|
if (!swp || !cmd)
|
|
return (EINVAL);
|
|
|
|
DPAA2_SWP_LOCK(swp, &flags);
|
|
if (flags & DPAA2_SWP_DESTROYED) {
|
|
/* Terminate operation if portal is destroyed. */
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (ENOENT);
|
|
}
|
|
|
|
rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR);
|
|
if (!RAR_SUCCESS(rar)) {
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (EBUSY);
|
|
}
|
|
|
|
map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
|
|
offset = swp->cfg.mem_backed
|
|
? DPAA2_SWP_CENA_RCR_MEM(RAR_IDX(rar))
|
|
: DPAA2_SWP_CENA_RCR(RAR_IDX(rar));
|
|
c = (struct with_verb *) cmd;
|
|
|
|
/* Write command bytes (without VERB byte). */
|
|
for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
|
|
bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
|
|
bus_write_4(map, offset + 4, cmd_pdat32[1]);
|
|
for (uint32_t i = 1; i <= 3; i++)
|
|
bus_write_1(map, offset + i, cmd_pdat8[i]);
|
|
|
|
/* Write VERB byte and trigger command execution. */
|
|
if (swp->cfg.mem_backed) {
|
|
bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
|
|
wmb();
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT +
|
|
RAR_IDX(rar) * 4, DPAA2_SWP_RT_MODE);
|
|
} else {
|
|
wmb();
|
|
bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
|
|
}
|
|
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* @brief Execute Volatile Dequeue Command (VDC).
|
|
*
|
|
* This command will be executed by QBMan only once in order to deliver requested
|
|
* number of frames (1-16 or 1-32 depending on QBMan version) to the driver via
|
|
* DQRR or arbitrary DMA-mapped memory.
|
|
*
|
|
* NOTE: There is a counterpart to the volatile dequeue command called static
|
|
* dequeue command (SDQC) which is executed periodically all the time the
|
|
* command is present in the SDQCR register.
|
|
*/
|
|
static int
|
|
dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp,
|
|
struct dpaa2_swp_cmd *cmd)
|
|
{
|
|
struct __packed with_verb {
|
|
uint8_t verb;
|
|
uint8_t _reserved[63];
|
|
} *c;
|
|
const uint8_t *p8 = (const uint8_t *) cmd->params;
|
|
const uint32_t *p32 = (const uint32_t *) cmd->params;
|
|
struct resource_map *map;
|
|
uint32_t offset;
|
|
|
|
map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
|
|
offset = swp->cfg.mem_backed
|
|
? DPAA2_SWP_CENA_VDQCR_MEM : DPAA2_SWP_CENA_VDQCR;
|
|
c = (struct with_verb *) cmd;
|
|
|
|
/* Write command bytes (without VERB byte). */
|
|
for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
|
|
bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
|
|
bus_write_4(map, offset + 4, p32[1]);
|
|
for (uint32_t i = 1; i <= 3; i++)
|
|
bus_write_1(map, offset + i, p8[i]);
|
|
|
|
/* Write VERB byte and trigger command execution. */
|
|
if (swp->cfg.mem_backed) {
|
|
bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
|
|
swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
|
|
wmb();
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT,
|
|
DPAA2_SWP_RT_MODE);
|
|
} else {
|
|
wmb();
|
|
bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
|
|
swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
/**
|
|
* @brief Execute a QBMan management command.
|
|
*/
|
|
static int
|
|
dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
|
|
struct dpaa2_swp_rsp *rsp, uint8_t cmdid)
|
|
{
|
|
#if (defined(_KERNEL) && defined(INVARIANTS))
|
|
struct __packed with_verb {
|
|
uint8_t verb;
|
|
uint8_t _reserved[63];
|
|
} *r;
|
|
#endif
|
|
uint16_t flags;
|
|
int error;
|
|
|
|
if (swp == NULL || cmd == NULL || rsp == NULL)
|
|
return (EINVAL);
|
|
|
|
DPAA2_SWP_LOCK(swp, &flags);
|
|
if (flags & DPAA2_SWP_DESTROYED) {
|
|
/* Terminate operation if portal is destroyed. */
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (ENOENT);
|
|
}
|
|
|
|
/*
|
|
* Send a command to QBMan using Management Command register and wait
|
|
* for response from the Management Response registers.
|
|
*/
|
|
dpaa2_swp_send_mgmt_command(swp, cmd, cmdid);
|
|
error = dpaa2_swp_wait_for_mgmt_response(swp, rsp);
|
|
if (error) {
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
return (error);
|
|
}
|
|
DPAA2_SWP_UNLOCK(swp);
|
|
|
|
#if (defined(_KERNEL) && defined(INVARIANTS))
|
|
r = (struct with_verb *) rsp;
|
|
KASSERT((r->verb & CMD_VERB_MASK) == cmdid,
|
|
("wrong VERB byte in response: resp=0x%02x, expected=0x%02x",
|
|
r->verb, cmdid));
|
|
#endif
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
|
|
uint8_t cmdid)
|
|
{
|
|
const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
|
|
const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
|
|
struct resource_map *map;
|
|
uint32_t offset;
|
|
|
|
map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
|
|
offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR;
|
|
|
|
/* Write command bytes (without VERB byte). */
|
|
for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
|
|
bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
|
|
bus_write_4(map, offset + 4, cmd_pdat32[1]);
|
|
for (uint32_t i = 1; i <= 3; i++)
|
|
bus_write_1(map, offset + i, cmd_pdat8[i]);
|
|
|
|
/* Write VERB byte and trigger command execution. */
|
|
if (swp->cfg.mem_backed) {
|
|
bus_write_1(map, offset, cmdid | swp->mr.valid_bit);
|
|
wmb();
|
|
dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT,
|
|
DPAA2_SWP_RT_MODE);
|
|
} else {
|
|
wmb();
|
|
bus_write_1(map, offset, cmdid | swp->mc.valid_bit);
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp)
|
|
{
|
|
struct resource_map *map = swp->cfg.mem_backed
|
|
? swp->cena_map : swp->cinh_map;
|
|
/* Management command response to be read from the only RR or RR0/RR1. */
|
|
const uint32_t offset = swp->cfg.mem_backed
|
|
? DPAA2_SWP_CENA_RR_MEM
|
|
: DPAA2_SWP_CENA_RR(swp->mc.valid_bit);
|
|
uint32_t i, verb, ret;
|
|
int rc;
|
|
|
|
/* Wait for a command response from QBMan. */
|
|
for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
|
|
if (swp->cfg.mem_backed) {
|
|
verb = (uint32_t) (bus_read_4(map, offset) & 0xFFu);
|
|
if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT))
|
|
goto wait;
|
|
if (!(verb & ~DPAA2_SWP_VALID_BIT))
|
|
goto wait;
|
|
swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT;
|
|
} else {
|
|
ret = bus_read_4(map, offset);
|
|
verb = ret & ~DPAA2_SWP_VALID_BIT; /* remove valid bit */
|
|
if (verb == 0u)
|
|
goto wait;
|
|
swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT;
|
|
}
|
|
break;
|
|
wait:
|
|
DELAY(CMD_SPIN_TIMEOUT);
|
|
}
|
|
/* Return an error on expired timeout. */
|
|
rc = i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0;
|
|
|
|
/* Read command response. */
|
|
for (i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
|
|
rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
|
|
|
|
return (rc);
|
|
}
|