d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
80b5058dcc
freebsd-dev/sys/dev/ocs_fc/ocs_hw_queues.c
Kenneth D. Merry ef270ab1b6 Bring in the Broadcom/Emulex Fibre Channel driver, ocs_fc(4). 
The ocs_fc(4) driver supports the following hardware:

Emulex 16/8G FC GEN 5 HBAS
	LPe15004 FC Host Bus Adapters
	LPe160XX FC Host Bus Adapters

Emulex 32/16G FC GEN 6 HBAS
	LPe3100X FC Host Bus Adapters
	LPe3200X FC Host Bus Adapters

The driver supports target and initiator mode, and also supports FC-Tape.

Note that the driver only currently works on little endian platforms.  It
is only included in the module build for amd64 and i386, and in GENERIC
on amd64 only.

Submitted by:	Ram Kishore Vegesna <ram.vegesna@broadcom.com>
Reviewed by:	mav
MFC after:	5 days
Relnotes:	yes
Sponsored by:	Broadcom
Differential Revision:	https://reviews.freebsd.org/D11423
2018-03-30 15:28:25 +00:00

2603 lines
66 KiB
C
Raw Blame History

/*-
* Copyright (c) 2017 Broadcom. All rights reserved.
* The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*/
/**
* @file
*
*/
#include "ocs_os.h"
#include "ocs_hw.h"
#include "ocs_hw_queues.h"
#define HW_QTOP_DEBUG 0
/**
* @brief Initialize queues
*
* Given the parsed queue topology spec, the SLI queues are created and
* initialized
*
* @param hw pointer to HW object
* @param qtop pointer to queue topology
*
* @return returns 0 for success, an error code value for failure.
*/
ocs_hw_rtn_e
ocs_hw_init_queues(ocs_hw_t *hw, ocs_hw_qtop_t *qtop)
{
uint32_t i, j;
uint32_t default_lengths[QTOP_LAST], len;
uint32_t rqset_len = 0, rqset_ulp = 0, rqset_count = 0;
uint8_t rqset_filter_mask = 0;
hw_eq_t *eqs[hw->config.n_rq];
hw_cq_t *cqs[hw->config.n_rq];
hw_rq_t *rqs[hw->config.n_rq];
ocs_hw_qtop_entry_t *qt, *next_qt;
ocs_hw_mrq_t mrq;
bool use_mrq = FALSE;
hw_eq_t *eq = NULL;
hw_cq_t *cq = NULL;
hw_wq_t *wq = NULL;
hw_rq_t *rq = NULL;
hw_mq_t *mq = NULL;
mrq.num_pairs = 0;
default_lengths[QTOP_EQ] = 1024;
default_lengths[QTOP_CQ] = hw->num_qentries[SLI_QTYPE_CQ];
default_lengths[QTOP_WQ] = hw->num_qentries[SLI_QTYPE_WQ];
default_lengths[QTOP_RQ] = hw->num_qentries[SLI_QTYPE_RQ];
default_lengths[QTOP_MQ] = OCS_HW_MQ_DEPTH;
ocs_hw_verify(hw != NULL, OCS_HW_RTN_INVALID_ARG);
hw->eq_count = 0;
hw->cq_count = 0;
hw->mq_count = 0;
hw->wq_count = 0;
hw->rq_count = 0;
hw->hw_rq_count = 0;
ocs_list_init(&hw->eq_list, hw_eq_t, link);
/* If MRQ is requested, Check if it is supported by SLI. */
if ((hw->config.n_rq > 1 ) && !hw->sli.config.features.flag.mrqp) {
ocs_log_err(hw->os, "MRQ topology not supported by SLI4.\n");
return OCS_HW_RTN_ERROR;
}
if (hw->config.n_rq > 1)
use_mrq = TRUE;
/* Allocate class WQ pools */
for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) {
hw->wq_class_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
if (hw->wq_class_array[i] == NULL) {
ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n");
return OCS_HW_RTN_NO_MEMORY;
}
}
/* Allocate per CPU WQ pools */
for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) {
hw->wq_cpu_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
if (hw->wq_cpu_array[i] == NULL) {
ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n");
return OCS_HW_RTN_NO_MEMORY;
}
}
ocs_hw_assert(qtop != NULL);
for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) {
if (i == qtop->inuse_count - 1)
next_qt = NULL;
else
next_qt = qt + 1;
switch(qt->entry) {
case QTOP_EQ:
len = (qt->len) ? qt->len : default_lengths[QTOP_EQ];
if (qt->set_default) {
default_lengths[QTOP_EQ] = len;
break;
}
eq = hw_new_eq(hw, len);
if (eq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
break;
case QTOP_CQ:
len = (qt->len) ? qt->len : default_lengths[QTOP_CQ];
if (qt->set_default) {
default_lengths[QTOP_CQ] = len;
break;
}
/* If this CQ is for MRQ, then delay the creation */
if (!use_mrq || next_qt->entry != QTOP_RQ) {
cq = hw_new_cq(eq, len);
if (cq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
}
break;
case QTOP_WQ: {
len = (qt->len) ? qt->len : default_lengths[QTOP_WQ];
if (qt->set_default) {
default_lengths[QTOP_WQ] = len;
break;
}
if ((hw->ulp_start + qt->ulp) > hw->ulp_max) {
ocs_log_err(hw->os, "invalid ULP %d for WQ\n", qt->ulp);
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
wq = hw_new_wq(cq, len, qt->class, hw->ulp_start + qt->ulp);
if (wq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
/* Place this WQ on the EQ WQ array */
if (ocs_varray_add(eq->wq_array, wq)) {
ocs_log_err(hw->os, "QTOP_WQ: EQ ocs_varray_add failed\n");
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
/* Place this WQ on the HW class array */
if (qt->class < ARRAY_SIZE(hw->wq_class_array)) {
if (ocs_varray_add(hw->wq_class_array[qt->class], wq)) {
ocs_log_err(hw->os, "HW wq_class_array ocs_varray_add failed\n");
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
} else {
ocs_log_err(hw->os, "Invalid class value: %d\n", qt->class);
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
/*
* Place this WQ on the per CPU list, asumming that EQs are mapped to cpu given
* by the EQ instance modulo number of CPUs
*/
if (ocs_varray_add(hw->wq_cpu_array[eq->instance % ocs_get_num_cpus()], wq)) {
ocs_log_err(hw->os, "HW wq_cpu_array ocs_varray_add failed\n");
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
break;
}
case QTOP_RQ: {
len = (qt->len) ? qt->len : default_lengths[QTOP_RQ];
if (qt->set_default) {
default_lengths[QTOP_RQ] = len;
break;
}
if ((hw->ulp_start + qt->ulp) > hw->ulp_max) {
ocs_log_err(hw->os, "invalid ULP %d for RQ\n", qt->ulp);
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
if (use_mrq) {
mrq.rq_cfg[mrq.num_pairs].len = len;
mrq.rq_cfg[mrq.num_pairs].ulp = hw->ulp_start + qt->ulp;
mrq.rq_cfg[mrq.num_pairs].filter_mask = qt->filter_mask;
mrq.rq_cfg[mrq.num_pairs].eq = eq;
mrq.num_pairs ++;
} else {
rq = hw_new_rq(cq, len, hw->ulp_start + qt->ulp);
if (rq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
rq->filter_mask = qt->filter_mask;
}
break;
}
case QTOP_MQ:
len = (qt->len) ? qt->len : default_lengths[QTOP_MQ];
if (qt->set_default) {
default_lengths[QTOP_MQ] = len;
break;
}
mq = hw_new_mq(cq, len);
if (mq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
break;
default:
ocs_hw_assert(0);
break;
}
}
if (mrq.num_pairs) {
/* First create normal RQs. */
for (i = 0; i < mrq.num_pairs; i++) {
for (j = 0; j < mrq.num_pairs; j++) {
if ((i != j) && (mrq.rq_cfg[i].filter_mask == mrq.rq_cfg[j].filter_mask)) {
/* This should be created using set */
if (rqset_filter_mask && (rqset_filter_mask != mrq.rq_cfg[i].filter_mask)) {
ocs_log_crit(hw->os, "Cant create morethan one RQ Set\n");
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
} else if (!rqset_filter_mask){
rqset_filter_mask = mrq.rq_cfg[i].filter_mask;
rqset_len = mrq.rq_cfg[i].len;
rqset_ulp = mrq.rq_cfg[i].ulp;
}
eqs[rqset_count] = mrq.rq_cfg[i].eq;
rqset_count++;
break;
}
}
if (j == mrq.num_pairs) {
/* Normal RQ */
cq = hw_new_cq(mrq.rq_cfg[i].eq, default_lengths[QTOP_CQ]);
if (cq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
rq = hw_new_rq(cq, mrq.rq_cfg[i].len, mrq.rq_cfg[i].ulp);
if (rq == NULL) {
hw_queue_teardown(hw);
return OCS_HW_RTN_NO_MEMORY;
}
rq->filter_mask = mrq.rq_cfg[i].filter_mask;
}
}
/* Now create RQ Set */
if (rqset_count) {
if (rqset_count > OCE_HW_MAX_NUM_MRQ_PAIRS) {
ocs_log_crit(hw->os,
"Max Supported MRQ pairs = %d\n",
OCE_HW_MAX_NUM_MRQ_PAIRS);
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
/* Create CQ set */
if (hw_new_cq_set(eqs, cqs, rqset_count, default_lengths[QTOP_CQ])) {
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
/* Create RQ set */
if (hw_new_rq_set(cqs, rqs, rqset_count, rqset_len, rqset_ulp)) {
hw_queue_teardown(hw);
return OCS_HW_RTN_ERROR;
}
for (i = 0; i < rqset_count ; i++) {
rqs[i]->filter_mask = rqset_filter_mask;
rqs[i]->is_mrq = TRUE;
rqs[i]->base_mrq_id = rqs[0]->hdr->id;
}
hw->hw_mrq_count = rqset_count;
}
}
return OCS_HW_RTN_SUCCESS;
}
/**
* @brief Allocate a new EQ object
*
* A new EQ object is instantiated
*
* @param hw pointer to HW object
* @param entry_count number of entries in the EQ
*
* @return pointer to allocated EQ object
*/
hw_eq_t*
hw_new_eq(ocs_hw_t *hw, uint32_t entry_count)
{
hw_eq_t *eq = ocs_malloc(hw->os, sizeof(*eq), OCS_M_ZERO | OCS_M_NOWAIT);
if (eq != NULL) {
eq->type = SLI_QTYPE_EQ;
eq->hw = hw;
eq->entry_count = entry_count;
eq->instance = hw->eq_count++;
eq->queue = &hw->eq[eq->instance];
ocs_list_init(&eq->cq_list, hw_cq_t, link);
eq->wq_array = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ);
if (eq->wq_array == NULL) {
ocs_free(hw->os, eq, sizeof(*eq));
eq = NULL;
} else {
if (sli_queue_alloc(&hw->sli, SLI_QTYPE_EQ, eq->queue, entry_count, NULL, 0)) {
ocs_log_err(hw->os, "EQ[%d] allocation failure\n", eq->instance);
ocs_free(hw->os, eq, sizeof(*eq));
eq = NULL;
} else {
sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8);
hw->hw_eq[eq->instance] = eq;
ocs_list_add_tail(&hw->eq_list, eq);
ocs_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance, eq->queue->id,
eq->entry_count);
}
}
}
return eq;
}
/**
* @brief Allocate a new CQ object
*
* A new CQ object is instantiated
*
* @param eq pointer to parent EQ object
* @param entry_count number of entries in the CQ
*
* @return pointer to allocated CQ object
*/
hw_cq_t*
hw_new_cq(hw_eq_t *eq, uint32_t entry_count)
{
ocs_hw_t *hw = eq->hw;
hw_cq_t *cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT);
if (cq != NULL) {
cq->eq = eq;
cq->type = SLI_QTYPE_CQ;
cq->instance = eq->hw->cq_count++;
cq->entry_count = entry_count;
cq->queue = &hw->cq[cq->instance];
ocs_list_init(&cq->q_list, hw_q_t, link);
if (sli_queue_alloc(&hw->sli, SLI_QTYPE_CQ, cq->queue, cq->entry_count, eq->queue, 0)) {
ocs_log_err(hw->os, "CQ[%d] allocation failure len=%d\n",
eq->instance,
eq->entry_count);
ocs_free(hw->os, cq, sizeof(*cq));
cq = NULL;
} else {
hw->hw_cq[cq->instance] = cq;
ocs_list_add_tail(&eq->cq_list, cq);
ocs_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance, cq->queue->id,
cq->entry_count);
}
}
return cq;
}
/**
* @brief Allocate a new CQ Set of objects.
*
* @param eqs pointer to a set of EQ objects.
* @param cqs pointer to a set of CQ objects to be returned.
* @param num_cqs number of CQ queues in the set.
* @param entry_count number of entries in the CQ.
*
* @return 0 on success and -1 on failure.
*/
uint32_t
hw_new_cq_set(hw_eq_t *eqs[], hw_cq_t *cqs[], uint32_t num_cqs, uint32_t entry_count)
{
uint32_t i;
ocs_hw_t *hw = eqs[0]->hw;
sli4_t *sli4 = &hw->sli;
hw_cq_t *cq = NULL;
sli4_queue_t *qs[SLI_MAX_CQ_SET_COUNT], *assocs[SLI_MAX_CQ_SET_COUNT];
/* Initialise CQS pointers to NULL */
for (i = 0; i < num_cqs; i++) {
cqs[i] = NULL;
}
for (i = 0; i < num_cqs; i++) {
cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT);
if (cq == NULL)
goto error;
cqs[i] = cq;
cq->eq = eqs[i];
cq->type = SLI_QTYPE_CQ;
cq->instance = hw->cq_count++;
cq->entry_count = entry_count;
cq->queue = &hw->cq[cq->instance];
qs[i] = cq->queue;
assocs[i] = eqs[i]->queue;
ocs_list_init(&cq->q_list, hw_q_t, link);
}
if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assocs)) {
ocs_log_err(NULL, "Failed to create CQ Set. \n");
goto error;
}
for (i = 0; i < num_cqs; i++) {
hw->hw_cq[cqs[i]->instance] = cqs[i];
ocs_list_add_tail(&cqs[i]->eq->cq_list, cqs[i]);
}
return 0;
error:
for (i = 0; i < num_cqs; i++) {
if (cqs[i]) {
ocs_free(hw->os, cqs[i], sizeof(*cqs[i]));
cqs[i] = NULL;
}
}
return -1;
}
/**
* @brief Allocate a new MQ object
*
* A new MQ object is instantiated
*
* @param cq pointer to parent CQ object
* @param entry_count number of entries in the MQ
*
* @return pointer to allocated MQ object
*/
hw_mq_t*
hw_new_mq(hw_cq_t *cq, uint32_t entry_count)
{
ocs_hw_t *hw = cq->eq->hw;
hw_mq_t *mq = ocs_malloc(hw->os, sizeof(*mq), OCS_M_ZERO | OCS_M_NOWAIT);
if (mq != NULL) {
mq->cq = cq;
mq->type = SLI_QTYPE_MQ;
mq->instance = cq->eq->hw->mq_count++;
mq->entry_count = entry_count;
mq->entry_size = OCS_HW_MQ_DEPTH;
mq->queue = &hw->mq[mq->instance];
if (sli_queue_alloc(&hw->sli, SLI_QTYPE_MQ,
mq->queue,
mq->entry_size,
cq->queue, 0)) {
ocs_log_err(hw->os, "MQ allocation failure\n");
ocs_free(hw->os, mq, sizeof(*mq));
mq = NULL;
} else {
hw->hw_mq[mq->instance] = mq;
ocs_list_add_tail(&cq->q_list, mq);
ocs_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance, mq->queue->id,
mq->entry_count);
}
}
return mq;
}
/**
* @brief Allocate a new WQ object
*
* A new WQ object is instantiated
*
* @param cq pointer to parent CQ object
* @param entry_count number of entries in the WQ
* @param class WQ class
* @param ulp index of chute
*
* @return pointer to allocated WQ object
*/
hw_wq_t*
hw_new_wq(hw_cq_t *cq, uint32_t entry_count, uint32_t class, uint32_t ulp)
{
ocs_hw_t *hw = cq->eq->hw;
hw_wq_t *wq = ocs_malloc(hw->os, sizeof(*wq), OCS_M_ZERO | OCS_M_NOWAIT);
if (wq != NULL) {
wq->hw = cq->eq->hw;
wq->cq = cq;
wq->type = SLI_QTYPE_WQ;
wq->instance = cq->eq->hw->wq_count++;
wq->entry_count = entry_count;
wq->queue = &hw->wq[wq->instance];
wq->ulp = ulp;
wq->wqec_set_count = OCS_HW_WQEC_SET_COUNT;
wq->wqec_count = wq->wqec_set_count;
wq->free_count = wq->entry_count - 1;
wq->class = class;
ocs_list_init(&wq->pending_list, ocs_hw_wqe_t, link);
if (sli_queue_alloc(&hw->sli, SLI_QTYPE_WQ, wq->queue, wq->entry_count, cq->queue, ulp)) {
ocs_log_err(hw->os, "WQ allocation failure\n");
ocs_free(hw->os, wq, sizeof(*wq));
wq = NULL;
} else {
hw->hw_wq[wq->instance] = wq;
ocs_list_add_tail(&cq->q_list, wq);
ocs_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d ulp %d\n", wq->instance, wq->queue->id,
wq->entry_count, wq->class, wq->ulp);
}
}
return wq;
}
/**
* @brief Allocate a hw_rq_t object
*
* Allocate an RQ object, which encapsulates 2 SLI queues (for rq pair)
*
* @param cq pointer to parent CQ object
* @param entry_count number of entries in the RQs
* @param ulp ULP index for this RQ
*
* @return pointer to newly allocated hw_rq_t
*/
hw_rq_t*
hw_new_rq(hw_cq_t *cq, uint32_t entry_count, uint32_t ulp)
{
ocs_hw_t *hw = cq->eq->hw;
hw_rq_t *rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT);
uint32_t max_hw_rq;
ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq);
if (rq != NULL) {
rq->instance = hw->hw_rq_count++;
rq->cq = cq;
rq->type = SLI_QTYPE_RQ;
rq->ulp = ulp;
rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR));
/* Create the header RQ */
ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq));
rq->hdr = &hw->rq[hw->rq_count];
rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE;
if (sli_fc_rq_alloc(&hw->sli, rq->hdr,
rq->entry_count,
rq->hdr_entry_size,
cq->queue,
ulp, TRUE)) {
ocs_log_err(hw->os, "RQ allocation failure - header\n");
ocs_free(hw->os, rq, sizeof(*rq));
return NULL;
}
hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */
hw->rq_count++;
ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d hdr size %4d ulp %d\n",
rq->instance, rq->hdr->id, rq->entry_count, rq->hdr_entry_size, rq->ulp);
/* Create the default data RQ */
ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq));
rq->data = &hw->rq[hw->rq_count];
rq->data_entry_size = hw->config.rq_default_buffer_size;
if (sli_fc_rq_alloc(&hw->sli, rq->data,
rq->entry_count,
rq->data_entry_size,
cq->queue,
ulp, FALSE)) {
ocs_log_err(hw->os, "RQ allocation failure - first burst\n");
ocs_free(hw->os, rq, sizeof(*rq));
return NULL;
}
hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */
hw->rq_count++;
ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d data size %4d ulp %d\n", rq->instance,
rq->data->id, rq->entry_count, rq->data_entry_size, rq->ulp);
hw->hw_rq[rq->instance] = rq;
ocs_list_add_tail(&cq->q_list, rq);
rq->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) *
rq->entry_count, OCS_M_ZERO | OCS_M_NOWAIT);
if (rq->rq_tracker == NULL) {
ocs_log_err(hw->os, "RQ tracker buf allocation failure\n");
return NULL;
}
}
return rq;
}
/**
* @brief Allocate a hw_rq_t object SET
*
* Allocate an RQ object SET, where each element in set
* encapsulates 2 SLI queues (for rq pair)
*
* @param cqs pointers to be associated with RQs.
* @param rqs RQ pointers to be returned on success.
* @param num_rq_pairs number of rq pairs in the Set.
* @param entry_count number of entries in the RQs
* @param ulp ULP index for this RQ
*
* @return 0 in success and -1 on failure.
*/
uint32_t
hw_new_rq_set(hw_cq_t *cqs[], hw_rq_t *rqs[], uint32_t num_rq_pairs, uint32_t entry_count, uint32_t ulp)
{
ocs_hw_t *hw = cqs[0]->eq->hw;
hw_rq_t *rq = NULL;
sli4_queue_t *qs[SLI_MAX_RQ_SET_COUNT * 2] = { NULL };
uint32_t max_hw_rq, i, q_count;
ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq);
/* Initialise RQS pointers */
for (i = 0; i < num_rq_pairs; i++) {
rqs[i] = NULL;
}
for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) {
rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT);
if (rq == NULL)
goto error;
rqs[i] = rq;
rq->instance = hw->hw_rq_count++;
rq->cq = cqs[i];
rq->type = SLI_QTYPE_RQ;
rq->ulp = ulp;
rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR));
/* Header RQ */
rq->hdr = &hw->rq[hw->rq_count];
rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE;
hw->hw_rq_lookup[hw->rq_count] = rq->instance;
hw->rq_count++;
qs[q_count] = rq->hdr;
/* Data RQ */
rq->data = &hw->rq[hw->rq_count];
rq->data_entry_size = hw->config.rq_default_buffer_size;
hw->hw_rq_lookup[hw->rq_count] = rq->instance;
hw->rq_count++;
qs[q_count + 1] = rq->data;
rq->rq_tracker = NULL;
}
if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs,
cqs[0]->queue->id,
rqs[0]->entry_count,
rqs[0]->hdr_entry_size,
rqs[0]->data_entry_size,
ulp)) {
ocs_log_err(hw->os, "RQ Set allocation failure for base CQ=%d\n", cqs[0]->queue->id);
goto error;
}
for (i = 0; i < num_rq_pairs; i++) {
hw->hw_rq[rqs[i]->instance] = rqs[i];
ocs_list_add_tail(&cqs[i]->q_list, rqs[i]);
rqs[i]->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) *
rqs[i]->entry_count, OCS_M_ZERO | OCS_M_NOWAIT);
if (rqs[i]->rq_tracker == NULL) {
ocs_log_err(hw->os, "RQ tracker buf allocation failure\n");
goto error;
}
}
return 0;
error:
for (i = 0; i < num_rq_pairs; i++) {
if (rqs[i] != NULL) {
if (rqs[i]->rq_tracker != NULL) {
ocs_free(hw->os, rq->rq_tracker,
sizeof(ocs_hw_sequence_t*) * rq->entry_count);
}
ocs_free(hw->os, rqs[i], sizeof(*rqs[i]));
}
}
return -1;
}
/**
* @brief Free an EQ object
*
* The EQ object and any child queue objects are freed
*
* @param eq pointer to EQ object
*
* @return none
*/
void
hw_del_eq(hw_eq_t *eq)
{
if (eq != NULL) {
hw_cq_t *cq;
hw_cq_t *cq_next;
ocs_list_foreach_safe(&eq->cq_list, cq, cq_next) {
hw_del_cq(cq);
}
ocs_varray_free(eq->wq_array);
ocs_list_remove(&eq->hw->eq_list, eq);
eq->hw->hw_eq[eq->instance] = NULL;
ocs_free(eq->hw->os, eq, sizeof(*eq));
}
}
/**
* @brief Free a CQ object
*
* The CQ object and any child queue objects are freed
*
* @param cq pointer to CQ object
*
* @return none
*/
void
hw_del_cq(hw_cq_t *cq)
{
if (cq != NULL) {
hw_q_t *q;
hw_q_t *q_next;
ocs_list_foreach_safe(&cq->q_list, q, q_next) {
switch(q->type) {
case SLI_QTYPE_MQ:
hw_del_mq((hw_mq_t*) q);
break;
case SLI_QTYPE_WQ:
hw_del_wq((hw_wq_t*) q);
break;
case SLI_QTYPE_RQ:
hw_del_rq((hw_rq_t*) q);
break;
default:
break;
}
}
ocs_list_remove(&cq->eq->cq_list, cq);
cq->eq->hw->hw_cq[cq->instance] = NULL;
ocs_free(cq->eq->hw->os, cq, sizeof(*cq));
}
}
/**
* @brief Free a MQ object
*
* The MQ object is freed
*
* @param mq pointer to MQ object
*
* @return none
*/
void
hw_del_mq(hw_mq_t *mq)
{
if (mq != NULL) {
ocs_list_remove(&mq->cq->q_list, mq);
mq->cq->eq->hw->hw_mq[mq->instance] = NULL;
ocs_free(mq->cq->eq->hw->os, mq, sizeof(*mq));
}
}
/**
* @brief Free a WQ object
*
* The WQ object is freed
*
* @param wq pointer to WQ object
*
* @return none
*/
void
hw_del_wq(hw_wq_t *wq)
{
if (wq != NULL) {
ocs_list_remove(&wq->cq->q_list, wq);
wq->cq->eq->hw->hw_wq[wq->instance] = NULL;
ocs_free(wq->cq->eq->hw->os, wq, sizeof(*wq));
}
}
/**
* @brief Free an RQ object
*
* The RQ object is freed
*
* @param rq pointer to RQ object
*
* @return none
*/
void
hw_del_rq(hw_rq_t *rq)
{
ocs_hw_t *hw = rq->cq->eq->hw;
if (rq != NULL) {
/* Free RQ tracker */
if (rq->rq_tracker != NULL) {
ocs_free(hw->os, rq->rq_tracker, sizeof(ocs_hw_sequence_t*) * rq->entry_count);
rq->rq_tracker = NULL;
}
ocs_list_remove(&rq->cq->q_list, rq);
hw->hw_rq[rq->instance] = NULL;
ocs_free(hw->os, rq, sizeof(*rq));
}
}
/**
* @brief Display HW queue objects
*
* The HW queue objects are displayed using ocs_log
*
* @param hw pointer to HW object
*
* @return none
*/
void
hw_queue_dump(ocs_hw_t *hw)
{
hw_eq_t *eq;
hw_cq_t *cq;
hw_q_t *q;
hw_mq_t *mq;
hw_wq_t *wq;
hw_rq_t *rq;
ocs_list_foreach(&hw->eq_list, eq) {
ocs_printf("eq[%d] id %2d\n", eq->instance, eq->queue->id);
ocs_list_foreach(&eq->cq_list, cq) {
ocs_printf(" cq[%d] id %2d current\n", cq->instance, cq->queue->id);
ocs_list_foreach(&cq->q_list, q) {
switch(q->type) {
case SLI_QTYPE_MQ:
mq = (hw_mq_t *) q;
ocs_printf(" mq[%d] id %2d\n", mq->instance, mq->queue->id);
break;
case SLI_QTYPE_WQ:
wq = (hw_wq_t *) q;
ocs_printf(" wq[%d] id %2d\n", wq->instance, wq->queue->id);
break;
case SLI_QTYPE_RQ:
rq = (hw_rq_t *) q;
ocs_printf(" rq[%d] hdr id %2d\n", rq->instance, rq->hdr->id);
break;
default:
break;
}
}
}
}
}
/**
* @brief Teardown HW queue objects
*
* The HW queue objects are freed
*
* @param hw pointer to HW object
*
* @return none
*/
void
hw_queue_teardown(ocs_hw_t *hw)
{
uint32_t i;
hw_eq_t *eq;
hw_eq_t *eq_next;
if (ocs_list_valid(&hw->eq_list)) {
ocs_list_foreach_safe(&hw->eq_list, eq, eq_next) {
hw_del_eq(eq);
}
}
for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) {
ocs_varray_free(hw->wq_cpu_array[i]);
hw->wq_cpu_array[i] = NULL;
}
for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) {
ocs_varray_free(hw->wq_class_array[i]);
hw->wq_class_array[i] = NULL;
}
}
/**
* @brief Allocate a WQ to an IO object
*
* The next work queue index is used to assign a WQ to an IO.
*
* If wq_steering is OCS_HW_WQ_STEERING_CLASS, a WQ from io->wq_class is
* selected.
*
* If wq_steering is OCS_HW_WQ_STEERING_REQUEST, then a WQ from the EQ that
* the IO request came in on is selected.
*
* If wq_steering is OCS_HW_WQ_STEERING_CPU, then a WQ associted with the
* CPU the request is made on is selected.
*
* @param hw pointer to HW object
* @param io pointer to IO object
*
* @return Return pointer to next WQ
*/
hw_wq_t *
ocs_hw_queue_next_wq(ocs_hw_t *hw, ocs_hw_io_t *io)
{
hw_eq_t *eq;
hw_wq_t *wq = NULL;
switch(io->wq_steering) {
case OCS_HW_WQ_STEERING_CLASS:
if (likely(io->wq_class < ARRAY_SIZE(hw->wq_class_array))) {
wq = ocs_varray_iter_next(hw->wq_class_array[io->wq_class]);
}
break;
case OCS_HW_WQ_STEERING_REQUEST:
eq = io->eq;
if (likely(eq != NULL)) {
wq = ocs_varray_iter_next(eq->wq_array);
}
break;
case OCS_HW_WQ_STEERING_CPU: {
uint32_t cpuidx = ocs_thread_getcpu();
if (likely(cpuidx < ARRAY_SIZE(hw->wq_cpu_array))) {
wq = ocs_varray_iter_next(hw->wq_cpu_array[cpuidx]);
}
break;
}
}
if (unlikely(wq == NULL)) {
wq = hw->hw_wq[0];
}
return wq;
}
/**
* @brief Return count of EQs for a queue topology object
*
* The EQ count for in the HWs queue topology (hw->qtop) object is returned
*
* @param hw pointer to HW object
*
* @return count of EQs
*/
uint32_t
ocs_hw_qtop_eq_count(ocs_hw_t *hw)
{
return hw->qtop->entry_counts[QTOP_EQ];
}
#define TOKEN_LEN 32
/**
* @brief return string given a QTOP entry
*
* @param entry QTOP entry
*
* @return returns string or "unknown"
*/
#if HW_QTOP_DEBUG
static char *
qtopentry2s(ocs_hw_qtop_entry_e entry) {
switch(entry) {
#define P(x) case x: return #x;
P(QTOP_EQ)
P(QTOP_CQ)
P(QTOP_WQ)
P(QTOP_RQ)
P(QTOP_MQ)
P(QTOP_THREAD_START)
P(QTOP_THREAD_END)
P(QTOP_LAST)
#undef P
}
return "unknown";
}
#endif
/**
* @brief Declare token types
*/
typedef enum {
TOK_LPAREN = 1,
TOK_RPAREN,
TOK_COLON,
TOK_EQUALS,
TOK_QUEUE,
TOK_ATTR_NAME,
TOK_NUMBER,
TOK_NUMBER_VALUE,
TOK_NUMBER_LIST,
} tok_type_e;
/**
* @brief Declare token sub-types
*/
typedef enum {
TOK_SUB_EQ = 100,
TOK_SUB_CQ,
TOK_SUB_RQ,
TOK_SUB_MQ,
TOK_SUB_WQ,
TOK_SUB_LEN,
TOK_SUB_CLASS,
TOK_SUB_ULP,
TOK_SUB_FILTER,
} tok_subtype_e;
/**
* @brief convert queue subtype to QTOP entry
*
* @param q queue subtype
*
* @return QTOP entry or 0
*/
static ocs_hw_qtop_entry_e
subtype2qtop(tok_subtype_e q)
{
switch(q) {
case TOK_SUB_EQ: return QTOP_EQ;
case TOK_SUB_CQ: return QTOP_CQ;
case TOK_SUB_RQ: return QTOP_RQ;
case TOK_SUB_MQ: return QTOP_MQ;
case TOK_SUB_WQ: return QTOP_WQ;
default:
break;
}
return 0;
}
/**
* @brief Declare token object
*/
typedef struct {
tok_type_e type;
tok_subtype_e subtype;
char string[TOKEN_LEN];
} tok_t;
/**
* @brief Declare token array object
*/
typedef struct {
tok_t *tokens; /* Pointer to array of tokens */
uint32_t alloc_count; /* Number of tokens in the array */
uint32_t inuse_count; /* Number of tokens posted to array */
uint32_t iter_idx; /* Iterator index */
} tokarray_t;
/**
* @brief Declare token match structure
*/
typedef struct {
char *s;
tok_type_e type;
tok_subtype_e subtype;
} tokmatch_t;
/**
* @brief test if character is ID start character
*
* @param c character to test
*
* @return TRUE if character is an ID start character
*/
static int32_t
idstart(int c)
{
return isalpha(c) || (c == '_') || (c == '$');
}
/**
* @brief test if character is an ID character
*
* @param c character to test
*
* @return TRUE if character is an ID character
*/
static int32_t
idchar(int c)
{
return idstart(c) || ocs_isdigit(c);
}
/**
* @brief Declare single character matches
*/
static tokmatch_t cmatches[] = {
{"(", TOK_LPAREN},
{")", TOK_RPAREN},
{":", TOK_COLON},
{"=", TOK_EQUALS},
};
/**
* @brief Declare identifier match strings
*/
static tokmatch_t smatches[] = {
{"eq", TOK_QUEUE, TOK_SUB_EQ},
{"cq", TOK_QUEUE, TOK_SUB_CQ},
{"rq", TOK_QUEUE, TOK_SUB_RQ},
{"mq", TOK_QUEUE, TOK_SUB_MQ},
{"wq", TOK_QUEUE, TOK_SUB_WQ},
{"len", TOK_ATTR_NAME, TOK_SUB_LEN},
{"class", TOK_ATTR_NAME, TOK_SUB_CLASS},
{"ulp", TOK_ATTR_NAME, TOK_SUB_ULP},
{"filter", TOK_ATTR_NAME, TOK_SUB_FILTER},
};
/**
* @brief Scan string and return next token
*
* The string is scanned and the next token is returned
*
* @param s input string to scan
* @param tok pointer to place scanned token
*
* @return pointer to input string following scanned token, or NULL
*/
static const char *
tokenize(const char *s, tok_t *tok)
{
uint32_t i;
memset(tok, 0, sizeof(*tok));
/* Skip over whitespace */
while (*s && ocs_isspace(*s)) {
s++;
}
/* Return if nothing left in this string */
if (*s == 0) {
return NULL;
}
/* Look for single character matches */
for (i = 0; i < ARRAY_SIZE(cmatches); i++) {
if (cmatches[i].s[0] == *s) {
tok->type = cmatches[i].type;
tok->subtype = cmatches[i].subtype;
tok->string[0] = *s++;
return s;
}
}
/* Scan for a hex number or decimal */
if ((s[0] == '0') && ((s[1] == 'x') || (s[1] == 'X'))) {
char *p = tok->string;
tok->type = TOK_NUMBER;
*p++ = *s++;
*p++ = *s++;
while ((*s == '.') || ocs_isxdigit(*s)) {
if ((p - tok->string) < (int32_t)sizeof(tok->string)) {
*p++ = *s;
}
if (*s == ',') {
tok->type = TOK_NUMBER_LIST;
}
s++;
}
*p = 0;
return s;
} else if (ocs_isdigit(*s)) {
char *p = tok->string;
tok->type = TOK_NUMBER;
while ((*s == ',') || ocs_isdigit(*s)) {
if ((p - tok->string) < (int32_t)sizeof(tok->string)) {
*p++ = *s;
}
if (*s == ',') {
tok->type = TOK_NUMBER_LIST;
}
s++;
}
*p = 0;
return s;
}
/* Scan for an ID */
if (idstart(*s)) {
char *p = tok->string;
for (*p++ = *s++; idchar(*s); s++) {
if ((p - tok->string) < TOKEN_LEN) {
*p++ = *s;
}
}
/* See if this is a $ number value */
if (tok->string[0] == '$') {
tok->type = TOK_NUMBER_VALUE;
} else {
/* Look for a string match */
for (i = 0; i < ARRAY_SIZE(smatches); i++) {
if (strcmp(smatches[i].s, tok->string) == 0) {
tok->type = smatches[i].type;
tok->subtype = smatches[i].subtype;
return s;
}
}
}
}
return s;
}
/**
* @brief convert token type to string
*
* @param type token type
*
* @return string, or "unknown"
*/
static const char *
token_type2s(tok_type_e type)
{
switch(type) {
#define P(x) case x: return #x;
P(TOK_LPAREN)
P(TOK_RPAREN)
P(TOK_COLON)
P(TOK_EQUALS)
P(TOK_QUEUE)
P(TOK_ATTR_NAME)
P(TOK_NUMBER)
P(TOK_NUMBER_VALUE)
P(TOK_NUMBER_LIST)
#undef P
}
return "unknown";
}
/**
* @brief convert token sub-type to string
*
* @param subtype token sub-type
*
* @return string, or "unknown"
*/
static const char *
token_subtype2s(tok_subtype_e subtype)
{
switch(subtype) {
#define P(x) case x: return #x;
P(TOK_SUB_EQ)
P(TOK_SUB_CQ)
P(TOK_SUB_RQ)
P(TOK_SUB_MQ)
P(TOK_SUB_WQ)
P(TOK_SUB_LEN)
P(TOK_SUB_CLASS)
P(TOK_SUB_ULP)
P(TOK_SUB_FILTER)
#undef P
}
return "";
}
/**
* @brief Generate syntax error message
*
* A syntax error message is found, the input tokens are dumped up to and including
* the token that failed as indicated by the current iterator index.
*
* @param hw pointer to HW object
* @param tokarray pointer to token array object
*
* @return none
*/
static void
tok_syntax(ocs_hw_t *hw, tokarray_t *tokarray)
{
uint32_t i;
tok_t *tok;
ocs_log_test(hw->os, "Syntax error:\n");
for (i = 0, tok = tokarray->tokens; (i <= tokarray->inuse_count); i++, tok++) {
ocs_log_test(hw->os, "%s [%2d] %-16s %-16s %s\n", (i == tokarray->iter_idx) ? ">>>" : " ", i,
token_type2s(tok->type), token_subtype2s(tok->subtype), tok->string);
}
}
/**
* @brief parse a number
*
* Parses tokens of type TOK_NUMBER and TOK_NUMBER_VALUE, returning a numeric value
*
* @param hw pointer to HW object
* @param qtop pointer to QTOP object
* @param tok pointer to token to parse
*
* @return numeric value
*/
static uint32_t
tok_getnumber(ocs_hw_t *hw, ocs_hw_qtop_t *qtop, tok_t *tok)
{
uint32_t rval = 0;
uint32_t num_cpus = ocs_get_num_cpus();
switch(tok->type) {
case TOK_NUMBER_VALUE:
if (ocs_strcmp(tok->string, "$ncpu") == 0) {
rval = num_cpus;
} else if (ocs_strcmp(tok->string, "$ncpu1") == 0) {
rval = num_cpus - 1;
} else if (ocs_strcmp(tok->string, "$nwq") == 0) {
if (hw != NULL) {
rval = hw->config.n_wq;
}
} else if (ocs_strcmp(tok->string, "$maxmrq") == 0) {
rval = MIN(num_cpus, OCS_HW_MAX_MRQS);
} else if (ocs_strcmp(tok->string, "$nulp") == 0) {
rval = hw->ulp_max - hw->ulp_start + 1;
} else if ((qtop->rptcount_idx > 0) && ocs_strcmp(tok->string, "$rpt0") == 0) {
rval = qtop->rptcount[qtop->rptcount_idx-1];
} else if ((qtop->rptcount_idx > 1) && ocs_strcmp(tok->string, "$rpt1") == 0) {
rval = qtop->rptcount[qtop->rptcount_idx-2];
} else if ((qtop->rptcount_idx > 2) && ocs_strcmp(tok->string, "$rpt2") == 0) {
rval = qtop->rptcount[qtop->rptcount_idx-3];
} else if ((qtop->rptcount_idx > 3) && ocs_strcmp(tok->string, "$rpt3") == 0) {
rval = qtop->rptcount[qtop->rptcount_idx-4];
} else {
rval = ocs_strtoul(tok->string, 0, 0);
}
break;
case TOK_NUMBER:
rval = ocs_strtoul(tok->string, 0, 0);
break;
default:
break;
}
return rval;
}
/**
* @brief parse an array of tokens
*
* The tokens are semantically parsed, to generate QTOP entries.
*
* @param hw pointer to HW object
* @param tokarray array array of tokens
* @param qtop ouptut QTOP object
*
* @return returns 0 for success, a negative error code value for failure.
*/
static int32_t
parse_topology(ocs_hw_t *hw, tokarray_t *tokarray, ocs_hw_qtop_t *qtop)
{
ocs_hw_qtop_entry_t *qt = qtop->entries + qtop->inuse_count;
tok_t *tok;
for (; (tokarray->iter_idx < tokarray->inuse_count) &&
((tok = &tokarray->tokens[tokarray->iter_idx]) != NULL); ) {
if (qtop->inuse_count >= qtop->alloc_count) {
return -1;
}
qt = qtop->entries + qtop->inuse_count;
switch (tok[0].type)
{
case TOK_QUEUE:
qt->entry = subtype2qtop(tok[0].subtype);
qt->set_default = FALSE;
qt->len = 0;
qt->class = 0;
qtop->inuse_count++;
tokarray->iter_idx++; /* Advance current token index */
/* Parse for queue attributes, possibly multiple instances */
while ((tokarray->iter_idx + 4) <= tokarray->inuse_count) {
tok = &tokarray->tokens[tokarray->iter_idx];
if( (tok[0].type == TOK_COLON) &&
(tok[1].type == TOK_ATTR_NAME) &&
(tok[2].type == TOK_EQUALS) &&
((tok[3].type == TOK_NUMBER) ||
(tok[3].type == TOK_NUMBER_VALUE) ||
(tok[3].type == TOK_NUMBER_LIST))) {
switch (tok[1].subtype) {
case TOK_SUB_LEN:
qt->len = tok_getnumber(hw, qtop, &tok[3]);
break;
case TOK_SUB_CLASS:
qt->class = tok_getnumber(hw, qtop, &tok[3]);
break;
case TOK_SUB_ULP:
qt->ulp = tok_getnumber(hw, qtop, &tok[3]);
break;
case TOK_SUB_FILTER:
if (tok[3].type == TOK_NUMBER_LIST) {
uint32_t mask = 0;
char *p = tok[3].string;
while ((p != NULL) && *p) {
uint32_t v;
v = ocs_strtoul(p, 0, 0);
if (v < 32) {
mask |= (1U << v);
}
p = ocs_strchr(p, ',');
if (p != NULL) {
p++;
}
}
qt->filter_mask = mask;
} else {
qt->filter_mask = (1U << tok_getnumber(hw, qtop, &tok[3]));
}
break;
default:
break;
}
/* Advance current token index */
tokarray->iter_idx += 4;
} else {
break;
}
}
qtop->entry_counts[qt->entry]++;
break;
case TOK_ATTR_NAME:
if ( ((tokarray->iter_idx + 5) <= tokarray->inuse_count) &&
(tok[1].type == TOK_COLON) &&
(tok[2].type == TOK_QUEUE) &&
(tok[3].type == TOK_EQUALS) &&
((tok[4].type == TOK_NUMBER) || (tok[4].type == TOK_NUMBER_VALUE))) {
qt->entry = subtype2qtop(tok[2].subtype);
qt->set_default = TRUE;
switch(tok[0].subtype) {
case TOK_SUB_LEN:
qt->len = tok_getnumber(hw, qtop, &tok[4]);
break;
case TOK_SUB_CLASS:
qt->class = tok_getnumber(hw, qtop, &tok[4]);
break;
case TOK_SUB_ULP:
qt->ulp = tok_getnumber(hw, qtop, &tok[4]);
break;
default:
break;
}
qtop->inuse_count++;
tokarray->iter_idx += 5;
} else {
tok_syntax(hw, tokarray);
return -1;
}
break;
case TOK_NUMBER:
case TOK_NUMBER_VALUE: {
uint32_t rpt_count = 1;
uint32_t i;
rpt_count = tok_getnumber(hw, qtop, tok);
if (tok[1].type == TOK_LPAREN) {
uint32_t iter_idx_save;
tokarray->iter_idx += 2;
/* save token array iteration index */
iter_idx_save = tokarray->iter_idx;
for (i = 0; i < rpt_count; i++) {
uint32_t rptcount_idx = qtop->rptcount_idx;
if (qtop->rptcount_idx < ARRAY_SIZE(qtop->rptcount)) {
qtop->rptcount[qtop->rptcount_idx++] = i;
}
/* restore token array iteration index */
tokarray->iter_idx = iter_idx_save;
/* parse, append to qtop */
parse_topology(hw, tokarray, qtop);
qtop->rptcount_idx = rptcount_idx;
}
}
break;
}
case TOK_RPAREN:
tokarray->iter_idx++;
return 0;
default:
tok_syntax(hw, tokarray);
return -1;
}
}
return 0;
}
/**
* @brief Parse queue topology string
*
* The queue topology object is allocated, and filled with the results of parsing the
* passed in queue topology string
*
* @param hw pointer to HW object
* @param qtop_string input queue topology string
*
* @return pointer to allocated QTOP object, or NULL if there was an error
*/
ocs_hw_qtop_t *
ocs_hw_qtop_parse(ocs_hw_t *hw, const char *qtop_string)
{
ocs_hw_qtop_t *qtop;
tokarray_t tokarray;
const char *s;
#if HW_QTOP_DEBUG
uint32_t i;
ocs_hw_qtop_entry_t *qt;
#endif
ocs_log_debug(hw->os, "queue topology: %s\n", qtop_string);
/* Allocate a token array */
tokarray.tokens = ocs_malloc(hw->os, MAX_TOKENS * sizeof(*tokarray.tokens), OCS_M_ZERO | OCS_M_NOWAIT);
if (tokarray.tokens == NULL) {
return NULL;
}
tokarray.alloc_count = MAX_TOKENS;
tokarray.inuse_count = 0;
tokarray.iter_idx = 0;
/* Parse the tokens */
for (s = qtop_string; (tokarray.inuse_count < tokarray.alloc_count) &&
((s = tokenize(s, &tokarray.tokens[tokarray.inuse_count]))) != NULL; ) {
tokarray.inuse_count++;;
}
/* Allocate a queue topology structure */
qtop = ocs_malloc(hw->os, sizeof(*qtop), OCS_M_ZERO | OCS_M_NOWAIT);
if (qtop == NULL) {
ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
ocs_log_err(hw->os, "malloc qtop failed\n");
return NULL;
}
qtop->os = hw->os;
/* Allocate queue topology entries */
qtop->entries = ocs_malloc(hw->os, OCS_HW_MAX_QTOP_ENTRIES*sizeof(*qtop->entries), OCS_M_ZERO | OCS_M_NOWAIT);
if (qtop->entries == NULL) {
ocs_log_err(hw->os, "malloc qtop entries failed\n");
ocs_free(hw->os, qtop, sizeof(*qtop));
ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
return NULL;
}
qtop->alloc_count = OCS_HW_MAX_QTOP_ENTRIES;
qtop->inuse_count = 0;
/* Parse the tokens */
parse_topology(hw, &tokarray, qtop);
#if HW_QTOP_DEBUG
for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) {
ocs_log_debug(hw->os, "entry %s set_df %d len %4d class %d ulp %d\n", qtopentry2s(qt->entry), qt->set_default, qt->len,
qt->class, qt->ulp);
}
#endif
/* Free the tokens array */
ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens));
return qtop;
}
/**
* @brief free queue topology object
*
* @param qtop pointer to QTOP object
*
* @return none
*/
void
ocs_hw_qtop_free(ocs_hw_qtop_t *qtop)
{
if (qtop != NULL) {
if (qtop->entries != NULL) {
ocs_free(qtop->os, qtop->entries, qtop->alloc_count*sizeof(*qtop->entries));
}
ocs_free(qtop->os, qtop, sizeof(*qtop));
}
}
/* Uncomment this to turn on RQ debug */
// #define ENABLE_DEBUG_RQBUF
static int32_t ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id);
static ocs_hw_sequence_t * ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex);
static int32_t ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq);
static ocs_hw_rtn_e ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq);
/**
* @brief Process receive queue completions for RQ Pair mode.
*
* @par Description
* RQ completions are processed. In RQ pair mode, a single header and single payload
* buffer are received, and passed to the function that has registered for unsolicited
* callbacks.
*
* @param hw Hardware context.
* @param cq Pointer to HW completion queue.
* @param cqe Completion queue entry.
*
* @return Returns 0 for success, or a negative error code value for failure.
*/
int32_t
ocs_hw_rqpair_process_rq(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
{
uint16_t rq_id;
uint32_t index;
int32_t rqindex;
int32_t rq_status;
uint32_t h_len;
uint32_t p_len;
ocs_hw_sequence_t *seq;
rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index);
if (0 != rq_status) {
switch (rq_status) {
case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED:
case SLI4_FC_ASYNC_RQ_DMA_FAILURE:
/* just get RQ buffer then return to chip */
rqindex = ocs_hw_rqpair_find(hw, rq_id);
if (rqindex < 0) {
ocs_log_test(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n",
rq_status, rq_id);
break;
}
/* get RQ buffer */
seq = ocs_hw_rqpair_get(hw, rqindex, index);
/* return to chip */
if (ocs_hw_rqpair_sequence_free(hw, seq)) {
ocs_log_test(hw->os, "status=%#x, failed to return buffers to RQ\n",
rq_status);
break;
}
break;
case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED:
case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC:
/* since RQ buffers were not consumed, cannot return them to chip */
/* fall through */
ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status);
default:
break;
}
return -1;
}
rqindex = ocs_hw_rqpair_find(hw, rq_id);
if (rqindex < 0) {
ocs_log_test(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id);
return -1;
}
OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++;
rq->payload_use_count++;})
seq = ocs_hw_rqpair_get(hw, rqindex, index);
ocs_hw_assert(seq != NULL);
seq->hw = hw;
seq->auto_xrdy = 0;
seq->out_of_xris = 0;
seq->xri = 0;
seq->hio = NULL;
sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len);
seq->header->dma.len = h_len;
seq->payload->dma.len = p_len;
seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe);
seq->hw_priv = cq->eq;
/* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
if (hw->config.bounce) {
fc_header_t *hdr = seq->header->dma.virt;
uint32_t s_id = fc_be24toh(hdr->s_id);
uint32_t d_id = fc_be24toh(hdr->d_id);
uint32_t ox_id = ocs_be16toh(hdr->ox_id);
if (hw->callback.bounce != NULL) {
(*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
}
} else {
hw->callback.unsolicited(hw->args.unsolicited, seq);
}
return 0;
}
/**
* @brief Process receive queue completions for RQ Pair mode - Auto xfer rdy
*
* @par Description
* RQ completions are processed. In RQ pair mode, a single header and single payload
* buffer are received, and passed to the function that has registered for unsolicited
* callbacks.
*
* @param hw Hardware context.
* @param cq Pointer to HW completion queue.
* @param cqe Completion queue entry.
*
* @return Returns 0 for success, or a negative error code value for failure.
*/
int32_t
ocs_hw_rqpair_process_auto_xfr_rdy_cmd(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
{
/* Seems silly to call a SLI function to decode - use the structure directly for performance */
sli4_fc_optimized_write_cmd_cqe_t *opt_wr = (sli4_fc_optimized_write_cmd_cqe_t*)cqe;
uint16_t rq_id;
uint32_t index;
int32_t rqindex;
int32_t rq_status;
uint32_t h_len;
uint32_t p_len;
ocs_hw_sequence_t *seq;
uint8_t axr_lock_taken = 0;
#if defined(OCS_DISC_SPIN_DELAY)
uint32_t delay = 0;
char prop_buf[32];
#endif
rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index);
if (0 != rq_status) {
switch (rq_status) {
case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED:
case SLI4_FC_ASYNC_RQ_DMA_FAILURE:
/* just get RQ buffer then return to chip */
rqindex = ocs_hw_rqpair_find(hw, rq_id);
if (rqindex < 0) {
ocs_log_err(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n",
rq_status, rq_id);
break;
}
/* get RQ buffer */
seq = ocs_hw_rqpair_get(hw, rqindex, index);
/* return to chip */
if (ocs_hw_rqpair_sequence_free(hw, seq)) {
ocs_log_err(hw->os, "status=%#x, failed to return buffers to RQ\n",
rq_status);
break;
}
break;
case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED:
case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC:
/* since RQ buffers were not consumed, cannot return them to chip */
ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status);
/* fall through */
default:
break;
}
return -1;
}
rqindex = ocs_hw_rqpair_find(hw, rq_id);
if (rqindex < 0) {
ocs_log_err(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id);
return -1;
}
OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++;
rq->payload_use_count++;})
seq = ocs_hw_rqpair_get(hw, rqindex, index);
ocs_hw_assert(seq != NULL);
seq->hw = hw;
seq->auto_xrdy = opt_wr->agxr;
seq->out_of_xris = opt_wr->oox;
seq->xri = opt_wr->xri;
seq->hio = NULL;
sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len);
seq->header->dma.len = h_len;
seq->payload->dma.len = p_len;
seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe);
seq->hw_priv = cq->eq;
if (seq->auto_xrdy) {
fc_header_t *fc_hdr = seq->header->dma.virt;
seq->hio = ocs_hw_io_lookup(hw, seq->xri);
ocs_lock(&seq->hio->axr_lock);
axr_lock_taken = 1;
/* save the FCFI, src_id, dest_id and ox_id because we need it for the sequence object when the data comes. */
seq->hio->axr_buf->fcfi = seq->fcfi;
seq->hio->axr_buf->hdr.ox_id = fc_hdr->ox_id;
seq->hio->axr_buf->hdr.s_id = fc_hdr->s_id;
seq->hio->axr_buf->hdr.d_id = fc_hdr->d_id;
seq->hio->axr_buf->cmd_cqe = 1;
/*
* Since auto xfer rdy is used for this IO, then clear the sequence
* initiative bit in the header so that the upper layers wait for the
* data. This should flow exactly like the first burst case.
*/
fc_hdr->f_ctl &= fc_htobe24(~FC_FCTL_SEQUENCE_INITIATIVE);
/* If AXR CMD CQE came before previous TRSP CQE of same XRI */
if (seq->hio->type == OCS_HW_IO_TARGET_RSP) {
seq->hio->axr_buf->call_axr_cmd = 1;
seq->hio->axr_buf->cmd_seq = seq;
goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd;
}
}
/* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
if (hw->config.bounce) {
fc_header_t *hdr = seq->header->dma.virt;
uint32_t s_id = fc_be24toh(hdr->s_id);
uint32_t d_id = fc_be24toh(hdr->d_id);
uint32_t ox_id = ocs_be16toh(hdr->ox_id);
if (hw->callback.bounce != NULL) {
(*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
}
} else {
hw->callback.unsolicited(hw->args.unsolicited, seq);
}
if (seq->auto_xrdy) {
/* If data cqe came before cmd cqe in out of order in case of AXR */
if(seq->hio->axr_buf->data_cqe == 1) {
#if defined(OCS_DISC_SPIN_DELAY)
if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) {
delay = ocs_strtoul(prop_buf, 0, 0);
ocs_udelay(delay);
}
#endif
/* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
if (hw->config.bounce) {
fc_header_t *hdr = seq->header->dma.virt;
uint32_t s_id = fc_be24toh(hdr->s_id);
uint32_t d_id = fc_be24toh(hdr->d_id);
uint32_t ox_id = ocs_be16toh(hdr->ox_id);
if (hw->callback.bounce != NULL) {
(*hw->callback.bounce)(ocs_hw_unsol_process_bounce, &seq->hio->axr_buf->seq, s_id, d_id, ox_id);
}
} else {
hw->callback.unsolicited(hw->args.unsolicited, &seq->hio->axr_buf->seq);
}
}
}
exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd:
if(axr_lock_taken) {
ocs_unlock(&seq->hio->axr_lock);
}
return 0;
}
/**
* @brief Process CQ completions for Auto xfer rdy data phases.
*
* @par Description
* The data is DMA'd into the data buffer posted to the SGL prior to the XRI
* being assigned to an IO. When the completion is received, All of the data
* is in the single buffer.
*
* @param hw Hardware context.
* @param cq Pointer to HW completion queue.
* @param cqe Completion queue entry.
*
* @return Returns 0 for success, or a negative error code value for failure.
*/
int32_t
ocs_hw_rqpair_process_auto_xfr_rdy_data(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe)
{
/* Seems silly to call a SLI function to decode - use the structure directly for performance */
sli4_fc_optimized_write_data_cqe_t *opt_wr = (sli4_fc_optimized_write_data_cqe_t*)cqe;
ocs_hw_sequence_t *seq;
ocs_hw_io_t *io;
ocs_hw_auto_xfer_rdy_buffer_t *buf;
#if defined(OCS_DISC_SPIN_DELAY)
uint32_t delay = 0;
char prop_buf[32];
#endif
/* Look up the IO */
io = ocs_hw_io_lookup(hw, opt_wr->xri);
ocs_lock(&io->axr_lock);
buf = io->axr_buf;
buf->data_cqe = 1;
seq = &buf->seq;
seq->hw = hw;
seq->auto_xrdy = 1;
seq->out_of_xris = 0;
seq->xri = opt_wr->xri;
seq->hio = io;
seq->header = &buf->header;
seq->payload = &buf->payload;
seq->header->dma.len = sizeof(fc_header_t);
seq->payload->dma.len = opt_wr->total_data_placed;
seq->fcfi = buf->fcfi;
seq->hw_priv = cq->eq;
if (opt_wr->status == SLI4_FC_WCQE_STATUS_SUCCESS) {
seq->status = OCS_HW_UNSOL_SUCCESS;
} else if (opt_wr->status == SLI4_FC_WCQE_STATUS_REMOTE_STOP) {
seq->status = OCS_HW_UNSOL_ABTS_RCVD;
} else {
seq->status = OCS_HW_UNSOL_ERROR;
}
/* If AXR CMD CQE came before previous TRSP CQE of same XRI */
if(io->type == OCS_HW_IO_TARGET_RSP) {
io->axr_buf->call_axr_data = 1;
goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data;
}
if(!buf->cmd_cqe) {
/* if data cqe came before cmd cqe, return here, cmd cqe will handle */
goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data;
}
#if defined(OCS_DISC_SPIN_DELAY)
if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) {
delay = ocs_strtoul(prop_buf, 0, 0);
ocs_udelay(delay);
}
#endif
/* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */
if (hw->config.bounce) {
fc_header_t *hdr = seq->header->dma.virt;
uint32_t s_id = fc_be24toh(hdr->s_id);
uint32_t d_id = fc_be24toh(hdr->d_id);
uint32_t ox_id = ocs_be16toh(hdr->ox_id);
if (hw->callback.bounce != NULL) {
(*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id);
}
} else {
hw->callback.unsolicited(hw->args.unsolicited, seq);
}
exit_ocs_hw_rqpair_process_auto_xfr_rdy_data:
ocs_unlock(&io->axr_lock);
return 0;
}
/**
* @brief Return pointer to RQ buffer entry.
*
* @par Description
* Returns a pointer to the RQ buffer entry given by @c rqindex and @c bufindex.
*
* @param hw Hardware context.
* @param rqindex Index of the RQ that is being processed.
* @param bufindex Index into the RQ that is being processed.
*
* @return Pointer to the sequence structure, or NULL otherwise.
*/
static ocs_hw_sequence_t *
ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex)
{
sli4_queue_t *rq_hdr = &hw->rq[rqindex];
sli4_queue_t *rq_payload = &hw->rq[rqindex+1];
ocs_hw_sequence_t *seq = NULL;
hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]];
#if defined(ENABLE_DEBUG_RQBUF)
uint64_t rqbuf_debug_value = 0xdead0000 | ((rq->id & 0xf) << 12) | (bufindex & 0xfff);
#endif
if (bufindex >= rq_hdr->length) {
ocs_log_err(hw->os, "RQ index %d bufindex %d exceed ring length %d for id %d\n",
rqindex, bufindex, rq_hdr->length, rq_hdr->id);
return NULL;
}
sli_queue_lock(rq_hdr);
sli_queue_lock(rq_payload);
#if defined(ENABLE_DEBUG_RQBUF)
/* Put a debug value into the rq, to track which entries are still valid */
_sli_queue_poke(&hw->sli, rq_hdr, bufindex, (uint8_t *)&rqbuf_debug_value);
_sli_queue_poke(&hw->sli, rq_payload, bufindex, (uint8_t *)&rqbuf_debug_value);
#endif
seq = rq->rq_tracker[bufindex];
rq->rq_tracker[bufindex] = NULL;
if (seq == NULL ) {
ocs_log_err(hw->os, "RQ buffer NULL, rqindex %d, bufindex %d, current q index = %d\n",
rqindex, bufindex, rq_hdr->index);
}
sli_queue_unlock(rq_payload);
sli_queue_unlock(rq_hdr);
return seq;
}
/**
* @brief Posts an RQ buffer to a queue and update the verification structures
*
* @param hw hardware context
* @param seq Pointer to sequence object.
*
* @return Returns 0 on success, or a non-zero value otherwise.
*/
static int32_t
ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
{
sli4_queue_t *rq_hdr = &hw->rq[seq->header->rqindex];
sli4_queue_t *rq_payload = &hw->rq[seq->payload->rqindex];
uint32_t hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex];
hw_rq_t *rq = hw->hw_rq[hw_rq_index];
uint32_t phys_hdr[2];
uint32_t phys_payload[2];
int32_t qindex_hdr;
int32_t qindex_payload;
/* Update the RQ verification lookup tables */
phys_hdr[0] = ocs_addr32_hi(seq->header->dma.phys);
phys_hdr[1] = ocs_addr32_lo(seq->header->dma.phys);
phys_payload[0] = ocs_addr32_hi(seq->payload->dma.phys);
phys_payload[1] = ocs_addr32_lo(seq->payload->dma.phys);
sli_queue_lock(rq_hdr);
sli_queue_lock(rq_payload);
/*
* Note: The header must be posted last for buffer pair mode because
* posting on the header queue posts the payload queue as well.
* We do not ring the payload queue independently in RQ pair mode.
*/
qindex_payload = _sli_queue_write(&hw->sli, rq_payload, (void *)phys_payload);
qindex_hdr = _sli_queue_write(&hw->sli, rq_hdr, (void *)phys_hdr);
if (qindex_hdr < 0 ||
qindex_payload < 0) {
ocs_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id);
sli_queue_unlock(rq_payload);
sli_queue_unlock(rq_hdr);
return OCS_HW_RTN_ERROR;
}
/* ensure the indexes are the same */
ocs_hw_assert(qindex_hdr == qindex_payload);
/* Update the lookup table */
if (rq->rq_tracker[qindex_hdr] == NULL) {
rq->rq_tracker[qindex_hdr] = seq;
} else {
ocs_log_test(hw->os, "expected rq_tracker[%d][%d] buffer to be NULL\n",
hw_rq_index, qindex_hdr);
}
sli_queue_unlock(rq_payload);
sli_queue_unlock(rq_hdr);
return OCS_HW_RTN_SUCCESS;
}
/**
* @brief Return RQ buffers (while in RQ pair mode).
*
* @par Description
* The header and payload buffers are returned to the Receive Queue.
*
* @param hw Hardware context.
* @param seq Header/payload sequence buffers.
*
* @return Returns OCS_HW_RTN_SUCCESS on success, or an error code value on failure.
*/
ocs_hw_rtn_e
ocs_hw_rqpair_sequence_free(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
{
ocs_hw_rtn_e rc = OCS_HW_RTN_SUCCESS;
/* Check for auto xfer rdy dummy buffers and call the proper release function. */
if (seq->header->rqindex == OCS_HW_RQ_INDEX_DUMMY_HDR) {
return ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(hw, seq);
}
/*
* Post the data buffer first. Because in RQ pair mode, ringing the
* doorbell of the header ring will post the data buffer as well.
*/
if (ocs_hw_rqpair_put(hw, seq)) {
ocs_log_err(hw->os, "error writing buffers\n");
return OCS_HW_RTN_ERROR;
}
return rc;
}
/**
* @brief Find the RQ index of RQ_ID.
*
* @param hw Hardware context.
* @param rq_id RQ ID to find.
*
* @return Returns the RQ index, or -1 if not found
*/
static inline int32_t
ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id)
{
return ocs_hw_queue_hash_find(hw->rq_hash, rq_id);
}
/**
* @ingroup devInitShutdown
* @brief Allocate auto xfer rdy buffers.
*
* @par Description
* Allocates the auto xfer rdy buffers and places them on the free list.
*
* @param hw Hardware context allocated by the caller.
* @param num_buffers Number of buffers to allocate.
*
* @return Returns 0 on success, or a non-zero value on failure.
*/
ocs_hw_rtn_e
ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(ocs_hw_t *hw, uint32_t num_buffers)
{
ocs_hw_auto_xfer_rdy_buffer_t *buf;
uint32_t i;
hw->auto_xfer_rdy_buf_pool = ocs_pool_alloc(hw->os, sizeof(ocs_hw_auto_xfer_rdy_buffer_t), num_buffers, FALSE);
if (hw->auto_xfer_rdy_buf_pool == NULL) {
ocs_log_err(hw->os, "Failure to allocate auto xfer ready buffer pool\n");
return OCS_HW_RTN_NO_MEMORY;
}
for (i = 0; i < num_buffers; i++) {
/* allocate the wrapper object */
buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i);
ocs_hw_assert(buf != NULL);
/* allocate the auto xfer ready buffer */
if (ocs_dma_alloc(hw->os, &buf->payload.dma, hw->config.auto_xfer_rdy_size, OCS_MIN_DMA_ALIGNMENT)) {
ocs_log_err(hw->os, "DMA allocation failed\n");
ocs_free(hw->os, buf, sizeof(*buf));
return OCS_HW_RTN_NO_MEMORY;
}
/* build a fake data header in big endian */
buf->hdr.info = FC_RCTL_INFO_SOL_DATA;
buf->hdr.r_ctl = FC_RCTL_FC4_DATA;
buf->hdr.type = FC_TYPE_FCP;
buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER |
FC_FCTL_FIRST_SEQUENCE |
FC_FCTL_LAST_SEQUENCE |
FC_FCTL_END_SEQUENCE |
FC_FCTL_SEQUENCE_INITIATIVE);
/* build the fake header DMA object */
buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR;
buf->header.dma.virt = &buf->hdr;
buf->header.dma.alloc = buf;
buf->header.dma.size = sizeof(buf->hdr);
buf->header.dma.len = sizeof(buf->hdr);
buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA;
}
return OCS_HW_RTN_SUCCESS;
}
/**
* @ingroup devInitShutdown
* @brief Post Auto xfer rdy buffers to the XRIs posted with DNRX.
*
* @par Description
* When new buffers are freed, check existing XRIs waiting for buffers.
*
* @param hw Hardware context allocated by the caller.
*/
static void
ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(ocs_hw_t *hw)
{
ocs_hw_io_t *io;
int32_t rc;
ocs_lock(&hw->io_lock);
while (!ocs_list_empty(&hw->io_port_dnrx)) {
io = ocs_list_remove_head(&hw->io_port_dnrx);
rc = ocs_hw_reque_xri(hw, io);
if(rc) {
break;
}
}
ocs_unlock(&hw->io_lock);
}
/**
* @brief Called when the POST_SGL_PAGE command completes.
*
* @par Description
* Free the mailbox command buffer.
*
* @param hw Hardware context.
* @param status Status field from the mbox completion.
* @param mqe Mailbox response structure.
* @param arg Pointer to a callback function that signals the caller that the command is done.
*
* @return Returns 0.
*/
static int32_t
ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void *arg)
{
if (status != 0) {
ocs_log_debug(hw->os, "Status 0x%x\n", status);
}
ocs_free(hw->os, mqe, SLI4_BMBX_SIZE);
return 0;
}
/**
* @brief Prepares an XRI to move to the chip.
*
* @par Description
* Puts the data SGL into the SGL list for the IO object and possibly registers
* an SGL list for the XRI. Since both the POST_XRI and POST_SGL_PAGES commands are
* mailbox commands, we don't need to wait for completion before preceding.
*
* @param hw Hardware context allocated by the caller.
* @param io Pointer to the IO object.
*
* @return Returns OCS_HW_RTN_SUCCESS for success, or an error code value for failure.
*/
ocs_hw_rtn_e
ocs_hw_rqpair_auto_xfer_rdy_move_to_port(ocs_hw_t *hw, ocs_hw_io_t *io)
{
/* We only need to preregister the SGL if it has not yet been done. */
if (!sli_get_sgl_preregister(&hw->sli)) {
uint8_t *post_sgl;
ocs_dma_t *psgls = &io->def_sgl;
ocs_dma_t **sgls = &psgls;
/* non-local buffer required for mailbox queue */
post_sgl = ocs_malloc(hw->os, SLI4_BMBX_SIZE, OCS_M_NOWAIT);
if (post_sgl == NULL) {
ocs_log_err(hw->os, "no buffer for command\n");
return OCS_HW_RTN_NO_MEMORY;
}
if (sli_cmd_fcoe_post_sgl_pages(&hw->sli, post_sgl, SLI4_BMBX_SIZE,
io->indicator, 1, sgls, NULL, NULL)) {
if (ocs_hw_command(hw, post_sgl, OCS_CMD_NOWAIT,
ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb, NULL)) {
ocs_free(hw->os, post_sgl, SLI4_BMBX_SIZE);
ocs_log_err(hw->os, "SGL post failed\n");
return OCS_HW_RTN_ERROR;
}
}
}
ocs_lock(&hw->io_lock);
if (ocs_hw_rqpair_auto_xfer_rdy_buffer_post(hw, io, 0) != 0) { /* DNRX set - no buffer */
ocs_unlock(&hw->io_lock);
return OCS_HW_RTN_ERROR;
}
ocs_unlock(&hw->io_lock);
return OCS_HW_RTN_SUCCESS;
}
/**
* @brief Prepares an XRI to move back to the host.
*
* @par Description
* Releases any attached buffer back to the pool.
*
* @param hw Hardware context allocated by the caller.
* @param io Pointer to the IO object.
*/
void
ocs_hw_rqpair_auto_xfer_rdy_move_to_host(ocs_hw_t *hw, ocs_hw_io_t *io)
{
if (io->axr_buf != NULL) {
ocs_lock(&hw->io_lock);
/* check list and remove if there */
if (ocs_list_on_list(&io->dnrx_link)) {
ocs_list_remove(&hw->io_port_dnrx, io);
io->auto_xfer_rdy_dnrx = 0;
/* release the count for waiting for a buffer */
ocs_hw_io_free(hw, io);
}
ocs_pool_put(hw->auto_xfer_rdy_buf_pool, io->axr_buf);
io->axr_buf = NULL;
ocs_unlock(&hw->io_lock);
ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw);
}
return;
}
/**
* @brief Posts an auto xfer rdy buffer to an IO.
*
* @par Description
* Puts the data SGL into the SGL list for the IO object
* @n @name
* @b Note: io_lock must be held.
*
* @param hw Hardware context allocated by the caller.
* @param io Pointer to the IO object.
*
* @return Returns the value of DNRX bit in the TRSP and ABORT WQEs.
*/
uint8_t
ocs_hw_rqpair_auto_xfer_rdy_buffer_post(ocs_hw_t *hw, ocs_hw_io_t *io, int reuse_buf)
{
ocs_hw_auto_xfer_rdy_buffer_t *buf;
sli4_sge_t *data;
if(!reuse_buf) {
buf = ocs_pool_get(hw->auto_xfer_rdy_buf_pool);
io->axr_buf = buf;
}
data = io->def_sgl.virt;
data[0].sge_type = SLI4_SGE_TYPE_SKIP;
data[0].last = 0;
/*
* Note: if we are doing DIF assists, then the SGE[1] must contain the
* DI_SEED SGE. The host is responsible for programming:
* SGE Type (Word 2, bits 30:27)
* Replacement App Tag (Word 2 bits 15:0)
* App Tag (Word 3 bits 15:0)
* New Ref Tag (Word 3 bit 23)
* Metadata Enable (Word 3 bit 20)
* Auto-Increment RefTag (Word 3 bit 19)
* Block Size (Word 3 bits 18:16)
* The following fields are managed by the SLI Port:
* Ref Tag Compare (Word 0)
* Replacement Ref Tag (Word 1) - In not the LBA
* NA (Word 2 bit 25)
* Opcode RX (Word 3 bits 27:24)
* Checksum Enable (Word 3 bit 22)
* RefTag Enable (Word 3 bit 21)
*
* The first two SGLs are cleared by ocs_hw_io_init_sges(), so assume eveything is cleared.
*/
if (hw->config.auto_xfer_rdy_p_type) {
sli4_diseed_sge_t *diseed = (sli4_diseed_sge_t*)&data[1];
diseed->sge_type = SLI4_SGE_TYPE_DISEED;
diseed->repl_app_tag = hw->config.auto_xfer_rdy_app_tag_value;
diseed->app_tag_cmp = hw->config.auto_xfer_rdy_app_tag_value;
diseed->check_app_tag = hw->config.auto_xfer_rdy_app_tag_valid;
diseed->auto_incr_ref_tag = TRUE; /* Always the LBA */
diseed->dif_blk_size = hw->config.auto_xfer_rdy_blk_size_chip;
} else {
data[1].sge_type = SLI4_SGE_TYPE_SKIP;
data[1].last = 0;
}
data[2].sge_type = SLI4_SGE_TYPE_DATA;
data[2].buffer_address_high = ocs_addr32_hi(io->axr_buf->payload.dma.phys);
data[2].buffer_address_low = ocs_addr32_lo(io->axr_buf->payload.dma.phys);
data[2].buffer_length = io->axr_buf->payload.dma.size;
data[2].last = TRUE;
data[3].sge_type = SLI4_SGE_TYPE_SKIP;
return 0;
}
/**
* @brief Return auto xfer ready buffers (while in RQ pair mode).
*
* @par Description
* The header and payload buffers are returned to the auto xfer rdy pool.
*
* @param hw Hardware context.
* @param seq Header/payload sequence buffers.
*
* @return Returns OCS_HW_RTN_SUCCESS for success, an error code value for failure.
*/
static ocs_hw_rtn_e
ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq)
{
ocs_hw_auto_xfer_rdy_buffer_t *buf = seq->header->dma.alloc;
buf->data_cqe = 0;
buf->cmd_cqe = 0;
buf->fcfi = 0;
buf->call_axr_cmd = 0;
buf->call_axr_data = 0;
/* build a fake data header in big endian */
buf->hdr.info = FC_RCTL_INFO_SOL_DATA;
buf->hdr.r_ctl = FC_RCTL_FC4_DATA;
buf->hdr.type = FC_TYPE_FCP;
buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER |
FC_FCTL_FIRST_SEQUENCE |
FC_FCTL_LAST_SEQUENCE |
FC_FCTL_END_SEQUENCE |
FC_FCTL_SEQUENCE_INITIATIVE);
/* build the fake header DMA object */
buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR;
buf->header.dma.virt = &buf->hdr;
buf->header.dma.alloc = buf;
buf->header.dma.size = sizeof(buf->hdr);
buf->header.dma.len = sizeof(buf->hdr);
buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA;
ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw);
return OCS_HW_RTN_SUCCESS;
}
/**
* @ingroup devInitShutdown
* @brief Free auto xfer rdy buffers.
*
* @par Description
* Frees the auto xfer rdy buffers.
*
* @param hw Hardware context allocated by the caller.
*
* @return Returns 0 on success, or a non-zero value on failure.
*/
static void
ocs_hw_rqpair_auto_xfer_rdy_buffer_free(ocs_hw_t *hw)
{
ocs_hw_auto_xfer_rdy_buffer_t *buf;
uint32_t i;
if (hw->auto_xfer_rdy_buf_pool != NULL) {
ocs_lock(&hw->io_lock);
for (i = 0; i < ocs_pool_get_count(hw->auto_xfer_rdy_buf_pool); i++) {
buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i);
if (buf != NULL) {
ocs_dma_free(hw->os, &buf->payload.dma);
}
}
ocs_unlock(&hw->io_lock);
ocs_pool_free(hw->auto_xfer_rdy_buf_pool);
hw->auto_xfer_rdy_buf_pool = NULL;
}
}
/**
* @ingroup devInitShutdown
* @brief Configure the rq_pair function from ocs_hw_init().
*
* @par Description
* Allocates the buffers to auto xfer rdy and posts initial XRIs for this feature.
*
* @param hw Hardware context allocated by the caller.
*
* @return Returns 0 on success, or a non-zero value on failure.
*/
ocs_hw_rtn_e
ocs_hw_rqpair_init(ocs_hw_t *hw)
{
ocs_hw_rtn_e rc;
uint32_t xris_posted;
ocs_log_debug(hw->os, "RQ Pair mode\n");
/*
* If we get this far, the auto XFR_RDY feature was enabled successfully, otherwise ocs_hw_init() would
* return with an error. So allocate the buffers based on the initial XRI pool required to support this
* feature.
*/
if (sli_get_auto_xfer_rdy_capable(&hw->sli) &&
hw->config.auto_xfer_rdy_size > 0) {
if (hw->auto_xfer_rdy_buf_pool == NULL) {
/*
* Allocate one more buffer than XRIs so that when all the XRIs are in use, we still have
* one to post back for the case where the response phase is started in the context of
* the data completion.
*/
rc = ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(hw, hw->config.auto_xfer_rdy_xri_cnt + 1);
if (rc != OCS_HW_RTN_SUCCESS) {
return rc;
}
} else {
ocs_pool_reset(hw->auto_xfer_rdy_buf_pool);
}
/* Post the auto XFR_RDY XRIs */
xris_posted = ocs_hw_xri_move_to_port_owned(hw, hw->config.auto_xfer_rdy_xri_cnt);
if (xris_posted != hw->config.auto_xfer_rdy_xri_cnt) {
ocs_log_err(hw->os, "post_xri failed, only posted %d XRIs\n", xris_posted);
return OCS_HW_RTN_ERROR;
}
}
return 0;
}
/**
* @ingroup devInitShutdown
* @brief Tear down the rq_pair function from ocs_hw_teardown().
*
* @par Description
* Frees the buffers to auto xfer rdy.
*
* @param hw Hardware context allocated by the caller.
*/
void
ocs_hw_rqpair_teardown(ocs_hw_t *hw)
{
/* We need to free any auto xfer ready buffers */
ocs_hw_rqpair_auto_xfer_rdy_buffer_free(hw);
}
Reference in New Issue View Git Blame Copy Permalink