numam-spdk/lib/nvme/nvme_qpair.c
Seth Howell f5d88e46e2 nvme: always set ctrlr->is_failed through API
Use the standard API function to fail the controller in all cases.

This patch, and the several following patches are aimed at creating a
mechanism for reporting up to the application layer that a controller is
failed and or removed. To do this, I use the reset_cb to inform the
upper layer that the controller is failed.
This also requires changes to how we handle a controller reset to
pave the way for doing optional reset retries in the libraries.

Change-Id: I06dfce08326c23472a1caa8f6efbac2fd1a720f2
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/469635
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
2019-10-07 15:05:00 +00:00

775 lines
25 KiB
C

/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "nvme_internal.h"
#include "spdk/nvme_ocssd.h"
static void nvme_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr);
static int nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req);
struct nvme_string {
uint16_t value;
const char *str;
};
static const struct nvme_string admin_opcode[] = {
{ SPDK_NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" },
{ SPDK_NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" },
{ SPDK_NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" },
{ SPDK_NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" },
{ SPDK_NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" },
{ SPDK_NVME_OPC_IDENTIFY, "IDENTIFY" },
{ SPDK_NVME_OPC_ABORT, "ABORT" },
{ SPDK_NVME_OPC_SET_FEATURES, "SET FEATURES" },
{ SPDK_NVME_OPC_GET_FEATURES, "GET FEATURES" },
{ SPDK_NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" },
{ SPDK_NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" },
{ SPDK_NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" },
{ SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" },
{ SPDK_NVME_OPC_DEVICE_SELF_TEST, "DEVICE SELF-TEST" },
{ SPDK_NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" },
{ SPDK_NVME_OPC_KEEP_ALIVE, "KEEP ALIVE" },
{ SPDK_NVME_OPC_DIRECTIVE_SEND, "DIRECTIVE SEND" },
{ SPDK_NVME_OPC_DIRECTIVE_RECEIVE, "DIRECTIVE RECEIVE" },
{ SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT, "VIRTUALIZATION MANAGEMENT" },
{ SPDK_NVME_OPC_NVME_MI_SEND, "NVME-MI SEND" },
{ SPDK_NVME_OPC_NVME_MI_RECEIVE, "NVME-MI RECEIVE" },
{ SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG, "DOORBELL BUFFER CONFIG" },
{ SPDK_NVME_OPC_FORMAT_NVM, "FORMAT NVM" },
{ SPDK_NVME_OPC_SECURITY_SEND, "SECURITY SEND" },
{ SPDK_NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" },
{ SPDK_NVME_OPC_SANITIZE, "SANITIZE" },
{ SPDK_OCSSD_OPC_GEOMETRY, "OCSSD / GEOMETRY" },
{ 0xFFFF, "ADMIN COMMAND" }
};
static const struct nvme_string io_opcode[] = {
{ SPDK_NVME_OPC_FLUSH, "FLUSH" },
{ SPDK_NVME_OPC_WRITE, "WRITE" },
{ SPDK_NVME_OPC_READ, "READ" },
{ SPDK_NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" },
{ SPDK_NVME_OPC_COMPARE, "COMPARE" },
{ SPDK_NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" },
{ SPDK_NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" },
{ SPDK_NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" },
{ SPDK_NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" },
{ SPDK_NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" },
{ SPDK_NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" },
{ SPDK_OCSSD_OPC_VECTOR_RESET, "OCSSD / VECTOR RESET" },
{ SPDK_OCSSD_OPC_VECTOR_WRITE, "OCSSD / VECTOR WRITE" },
{ SPDK_OCSSD_OPC_VECTOR_READ, "OCSSD / VECTOR READ" },
{ SPDK_OCSSD_OPC_VECTOR_COPY, "OCSSD / VECTOR COPY" },
{ 0xFFFF, "IO COMMAND" }
};
static const char *
nvme_get_string(const struct nvme_string *strings, uint16_t value)
{
const struct nvme_string *entry;
entry = strings;
while (entry->value != 0xFFFF) {
if (entry->value == value) {
return entry->str;
}
entry++;
}
return entry->str;
}
static void
nvme_admin_qpair_print_command(struct spdk_nvme_qpair *qpair,
struct spdk_nvme_cmd *cmd)
{
SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%x "
"cdw10:%08x cdw11:%08x\n",
nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qpair->id, cmd->cid,
cmd->nsid, cmd->cdw10, cmd->cdw11);
}
static void
nvme_io_qpair_print_command(struct spdk_nvme_qpair *qpair,
struct spdk_nvme_cmd *cmd)
{
assert(qpair != NULL);
assert(cmd != NULL);
switch ((int)cmd->opc) {
case SPDK_NVME_OPC_WRITE:
case SPDK_NVME_OPC_READ:
case SPDK_NVME_OPC_WRITE_UNCORRECTABLE:
case SPDK_NVME_OPC_COMPARE:
SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d "
"lba:%llu len:%d\n",
nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
cmd->nsid,
((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10,
(cmd->cdw12 & 0xFFFF) + 1);
break;
case SPDK_NVME_OPC_FLUSH:
case SPDK_NVME_OPC_DATASET_MANAGEMENT:
SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d\n",
nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid,
cmd->nsid);
break;
default:
SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%d\n",
nvme_get_string(io_opcode, cmd->opc), cmd->opc, qpair->id,
cmd->cid, cmd->nsid);
break;
}
}
void
spdk_nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd)
{
assert(qpair != NULL);
assert(cmd != NULL);
if (nvme_qpair_is_admin_queue(qpair)) {
nvme_admin_qpair_print_command(qpair, cmd);
} else {
nvme_io_qpair_print_command(qpair, cmd);
}
}
static const struct nvme_string generic_status[] = {
{ SPDK_NVME_SC_SUCCESS, "SUCCESS" },
{ SPDK_NVME_SC_INVALID_OPCODE, "INVALID OPCODE" },
{ SPDK_NVME_SC_INVALID_FIELD, "INVALID FIELD" },
{ SPDK_NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" },
{ SPDK_NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" },
{ SPDK_NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" },
{ SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" },
{ SPDK_NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" },
{ SPDK_NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" },
{ SPDK_NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" },
{ SPDK_NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" },
{ SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" },
{ SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" },
{ SPDK_NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" },
{ SPDK_NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" },
{ SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" },
{ SPDK_NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" },
{ SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" },
{ SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" },
{ SPDK_NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" },
{ SPDK_NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" },
{ SPDK_NVME_SC_OPERATION_DENIED, "OPERATION DENIED" },
{ SPDK_NVME_SC_INVALID_SGL_OFFSET, "INVALID SGL OFFSET" },
{ SPDK_NVME_SC_HOSTID_INCONSISTENT_FORMAT, "HOSTID INCONSISTENT FORMAT" },
{ SPDK_NVME_SC_KEEP_ALIVE_EXPIRED, "KEEP ALIVE EXPIRED" },
{ SPDK_NVME_SC_KEEP_ALIVE_INVALID, "KEEP ALIVE INVALID" },
{ SPDK_NVME_SC_ABORTED_PREEMPT, "ABORTED - PREEMPT AND ABORT" },
{ SPDK_NVME_SC_SANITIZE_FAILED, "SANITIZE FAILED" },
{ SPDK_NVME_SC_SANITIZE_IN_PROGRESS, "SANITIZE IN PROGRESS" },
{ SPDK_NVME_SC_SGL_DATA_BLOCK_GRANULARITY_INVALID, "DATA BLOCK GRANULARITY INVALID" },
{ SPDK_NVME_SC_COMMAND_INVALID_IN_CMB, "COMMAND NOT SUPPORTED FOR QUEUE IN CMB" },
{ SPDK_NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" },
{ SPDK_NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" },
{ SPDK_NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" },
{ SPDK_NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" },
{ SPDK_NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" },
{ 0xFFFF, "GENERIC" }
};
static const struct nvme_string command_specific_status[] = {
{ SPDK_NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" },
{ SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" },
{ SPDK_NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED, "MAX QUEUE SIZE EXCEEDED" },
{ SPDK_NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" },
{ SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" },
{ SPDK_NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" },
{ SPDK_NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" },
{ SPDK_NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" },
{ SPDK_NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" },
{ SPDK_NVME_SC_INVALID_FORMAT, "INVALID FORMAT" },
{ SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET, "FIRMWARE REQUIRES CONVENTIONAL RESET" },
{ SPDK_NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" },
{ SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" },
{ SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" },
{ SPDK_NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC, "FEATURE NOT NAMESPACE SPECIFIC" },
{ SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" },
{ SPDK_NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" },
{ SPDK_NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION, "FIRMWARE REQUIRES MAX TIME VIOLATION" },
{ SPDK_NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED, "FIRMWARE ACTIVATION PROHIBITED" },
{ SPDK_NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" },
{ SPDK_NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY, "NAMESPACE INSUFFICIENT CAPACITY" },
{ SPDK_NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" },
{ SPDK_NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" },
{ SPDK_NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" },
{ SPDK_NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" },
{ SPDK_NVME_SC_THINPROVISIONING_NOT_SUPPORTED, "THINPROVISIONING NOT SUPPORTED" },
{ SPDK_NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" },
{ SPDK_NVME_SC_DEVICE_SELF_TEST_IN_PROGRESS, "DEVICE SELF-TEST IN PROGRESS" },
{ SPDK_NVME_SC_BOOT_PARTITION_WRITE_PROHIBITED, "BOOT PARTITION WRITE PROHIBITED" },
{ SPDK_NVME_SC_INVALID_CTRLR_ID, "INVALID CONTROLLER ID" },
{ SPDK_NVME_SC_INVALID_SECONDARY_CTRLR_STATE, "INVALID SECONDARY CONTROLLER STATE" },
{ SPDK_NVME_SC_INVALID_NUM_CTRLR_RESOURCES, "INVALID NUMBER OF CONTROLLER RESOURCES" },
{ SPDK_NVME_SC_INVALID_RESOURCE_ID, "INVALID RESOURCE IDENTIFIER" },
{ SPDK_NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" },
{ SPDK_NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" },
{ SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE, "WRITE TO RO RANGE" },
{ 0xFFFF, "COMMAND SPECIFIC" }
};
static const struct nvme_string media_error_status[] = {
{ SPDK_NVME_SC_WRITE_FAULTS, "WRITE FAULTS" },
{ SPDK_NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" },
{ SPDK_NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" },
{ SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" },
{ SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" },
{ SPDK_NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" },
{ SPDK_NVME_SC_ACCESS_DENIED, "ACCESS DENIED" },
{ SPDK_NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" },
{ SPDK_OCSSD_SC_OFFLINE_CHUNK, "RESET OFFLINE CHUNK" },
{ SPDK_OCSSD_SC_INVALID_RESET, "INVALID RESET" },
{ SPDK_OCSSD_SC_WRITE_FAIL_WRITE_NEXT_UNIT, "WRITE FAIL WRITE NEXT UNIT" },
{ SPDK_OCSSD_SC_WRITE_FAIL_CHUNK_EARLY_CLOSE, "WRITE FAIL CHUNK EARLY CLOSE" },
{ SPDK_OCSSD_SC_OUT_OF_ORDER_WRITE, "OUT OF ORDER WRITE" },
{ SPDK_OCSSD_SC_READ_HIGH_ECC, "READ HIGH ECC" },
{ 0xFFFF, "MEDIA ERROR" }
};
static const struct nvme_string path_status[] = {
{ SPDK_NVME_SC_INTERNAL_PATH_ERROR, "INTERNAL PATH ERROR" },
{ SPDK_NVME_SC_CONTROLLER_PATH_ERROR, "CONTROLLER PATH ERROR" },
{ SPDK_NVME_SC_HOST_PATH_ERROR, "HOST PATH ERROR" },
{ SPDK_NVME_SC_ABORTED_BY_HOST, "ABORTED BY HOST" },
{ 0xFFFF, "PATH ERROR" }
};
const char *
spdk_nvme_cpl_get_status_string(const struct spdk_nvme_status *status)
{
const struct nvme_string *entry;
switch (status->sct) {
case SPDK_NVME_SCT_GENERIC:
entry = generic_status;
break;
case SPDK_NVME_SCT_COMMAND_SPECIFIC:
entry = command_specific_status;
break;
case SPDK_NVME_SCT_MEDIA_ERROR:
entry = media_error_status;
break;
case SPDK_NVME_SCT_PATH:
entry = path_status;
break;
case SPDK_NVME_SCT_VENDOR_SPECIFIC:
return "VENDOR SPECIFIC";
default:
return "RESERVED";
}
return nvme_get_string(entry, status->sc);
}
void
spdk_nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair,
struct spdk_nvme_cpl *cpl)
{
SPDK_NOTICELOG("%s (%02x/%02x) sqid:%d cid:%d cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n",
spdk_nvme_cpl_get_status_string(&cpl->status),
cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0,
cpl->sqhd, cpl->status.p, cpl->status.m, cpl->status.dnr);
}
bool
nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl)
{
/*
* TODO: spec is not clear how commands that are aborted due
* to TLER will be marked. So for now, it seems
* NAMESPACE_NOT_READY is the only case where we should
* look at the DNR bit.
*/
switch ((int)cpl->status.sct) {
case SPDK_NVME_SCT_GENERIC:
switch ((int)cpl->status.sc) {
case SPDK_NVME_SC_NAMESPACE_NOT_READY:
case SPDK_NVME_SC_FORMAT_IN_PROGRESS:
if (cpl->status.dnr) {
return false;
} else {
return true;
}
case SPDK_NVME_SC_INVALID_OPCODE:
case SPDK_NVME_SC_INVALID_FIELD:
case SPDK_NVME_SC_COMMAND_ID_CONFLICT:
case SPDK_NVME_SC_DATA_TRANSFER_ERROR:
case SPDK_NVME_SC_ABORTED_POWER_LOSS:
case SPDK_NVME_SC_INTERNAL_DEVICE_ERROR:
case SPDK_NVME_SC_ABORTED_BY_REQUEST:
case SPDK_NVME_SC_ABORTED_SQ_DELETION:
case SPDK_NVME_SC_ABORTED_FAILED_FUSED:
case SPDK_NVME_SC_ABORTED_MISSING_FUSED:
case SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
case SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR:
case SPDK_NVME_SC_LBA_OUT_OF_RANGE:
case SPDK_NVME_SC_CAPACITY_EXCEEDED:
default:
return false;
}
case SPDK_NVME_SCT_PATH:
/*
* Per NVMe TP 4028 (Path and Transport Error Enhancements), retries should be
* based on the setting of the DNR bit for Internal Path Error
*/
switch ((int)cpl->status.sc) {
case SPDK_NVME_SC_INTERNAL_PATH_ERROR:
return !cpl->status.dnr;
default:
return false;
}
case SPDK_NVME_SCT_COMMAND_SPECIFIC:
case SPDK_NVME_SCT_MEDIA_ERROR:
case SPDK_NVME_SCT_VENDOR_SPECIFIC:
default:
return false;
}
}
static void
nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair,
struct nvme_request *req, uint32_t sct, uint32_t sc,
uint32_t dnr, bool print_on_error)
{
struct spdk_nvme_cpl cpl;
bool error;
memset(&cpl, 0, sizeof(cpl));
cpl.sqid = qpair->id;
cpl.status.sct = sct;
cpl.status.sc = sc;
cpl.status.dnr = dnr;
error = spdk_nvme_cpl_is_error(&cpl);
if (error && print_on_error && !qpair->ctrlr->opts.disable_error_logging) {
SPDK_NOTICELOG("Command completed manually:\n");
spdk_nvme_qpair_print_command(qpair, &req->cmd);
spdk_nvme_qpair_print_completion(qpair, &cpl);
}
nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &cpl);
nvme_free_request(req);
}
static void
nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
{
struct nvme_request *req;
while (!STAILQ_EMPTY(&qpair->queued_req)) {
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
if (!qpair->ctrlr->opts.disable_error_logging) {
SPDK_ERRLOG("aborting queued i/o\n");
}
nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC,
SPDK_NVME_SC_ABORTED_BY_REQUEST, dnr, true);
}
}
static inline bool
nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair)
{
if (!qpair->is_enabled && !qpair->ctrlr->is_resetting) {
nvme_qpair_complete_error_reqs(qpair);
nvme_qpair_abort_queued_reqs(qpair, 0 /* retry */);
nvme_qpair_enable(qpair);
nvme_transport_qpair_abort_reqs(qpair, 0 /* retry */);
}
return qpair->is_enabled;
}
int32_t
spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
{
int32_t ret;
int32_t resubmit_rc;
int32_t i;
struct nvme_request *req, *tmp;
if (qpair->ctrlr->is_failed) {
nvme_qpair_abort_reqs(qpair, 1 /* do not retry */);
return 0;
}
if (spdk_unlikely(!nvme_qpair_check_enabled(qpair) && !qpair->is_connecting)) {
/*
* qpair is not enabled, likely because a controller reset is
* in progress.
*/
return 0;
}
/* error injection for those queued error requests */
if (spdk_unlikely(!STAILQ_EMPTY(&qpair->err_req_head))) {
STAILQ_FOREACH_SAFE(req, &qpair->err_req_head, stailq, tmp) {
if (spdk_get_ticks() - req->submit_tick > req->timeout_tsc) {
STAILQ_REMOVE(&qpair->err_req_head, req, nvme_request, stailq);
nvme_qpair_manual_complete_request(qpair, req,
req->cpl.status.sct,
req->cpl.status.sc, 0, true);
}
}
}
qpair->in_completion_context = 1;
ret = nvme_transport_qpair_process_completions(qpair, max_completions);
if (ret < 0) {
SPDK_ERRLOG("CQ error, abort requests after transport retry counter exceeded\n");
nvme_ctrlr_fail(qpair->ctrlr, false);
}
qpair->in_completion_context = 0;
if (qpair->delete_after_completion_context) {
/*
* A request to delete this qpair was made in the context of this completion
* routine - so it is safe to delete it now.
*/
spdk_nvme_ctrlr_free_io_qpair(qpair);
return ret;
}
/*
* At this point, ret must represent the number of completions we reaped.
* submit as many queued requests as we completed.
*/
i = 0;
while (i < ret && !STAILQ_EMPTY(&qpair->queued_req) && !qpair->ctrlr->is_resetting) {
req = STAILQ_FIRST(&qpair->queued_req);
STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
resubmit_rc = nvme_qpair_resubmit_request(qpair, req);
if (spdk_unlikely(resubmit_rc != 0)) {
SPDK_ERRLOG("Unable to resubmit as many requests as we completed.\n");
break;
}
i++;
}
return ret;
}
int
nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,
struct spdk_nvme_ctrlr *ctrlr,
enum spdk_nvme_qprio qprio,
uint32_t num_requests)
{
size_t req_size_padded;
uint32_t i;
qpair->id = id;
qpair->qprio = qprio;
qpair->in_completion_context = 0;
qpair->delete_after_completion_context = 0;
qpair->no_deletion_notification_needed = 0;
qpair->ctrlr = ctrlr;
qpair->trtype = ctrlr->trid.trtype;
STAILQ_INIT(&qpair->free_req);
STAILQ_INIT(&qpair->queued_req);
TAILQ_INIT(&qpair->err_cmd_head);
STAILQ_INIT(&qpair->err_req_head);
req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63;
qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL,
SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE);
if (qpair->req_buf == NULL) {
SPDK_ERRLOG("no memory to allocate qpair(cntlid:0x%x sqid:%d) req_buf with %d request\n",
ctrlr->cntlid, qpair->id, num_requests);
return -ENOMEM;
}
for (i = 0; i < num_requests; i++) {
struct nvme_request *req = qpair->req_buf + i * req_size_padded;
req->qpair = qpair;
STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);
}
return 0;
}
void
nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair)
{
struct nvme_request *req;
while (!STAILQ_EMPTY(&qpair->err_req_head)) {
req = STAILQ_FIRST(&qpair->err_req_head);
STAILQ_REMOVE_HEAD(&qpair->err_req_head, stailq);
nvme_qpair_manual_complete_request(qpair, req,
req->cpl.status.sct,
req->cpl.status.sc, 0, true);
}
}
void
nvme_qpair_deinit(struct spdk_nvme_qpair *qpair)
{
struct nvme_error_cmd *cmd, *entry;
nvme_qpair_complete_error_reqs(qpair);
TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) {
TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link);
spdk_free(cmd);
}
spdk_free(qpair->req_buf);
}
static inline int
_nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
int rc = 0;
struct nvme_request *child_req, *tmp;
struct nvme_error_cmd *cmd;
struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
bool child_req_failed = false;
nvme_qpair_check_enabled(qpair);
if (req->num_children) {
/*
* This is a split (parent) request. Submit all of the children but not the parent
* request itself, since the parent is the original unsplit request.
*/
TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) {
if (spdk_likely(!child_req_failed)) {
rc = nvme_qpair_submit_request(qpair, child_req);
if (spdk_unlikely(rc != 0)) {
child_req_failed = true;
}
} else { /* free remaining child_reqs since one child_req fails */
nvme_request_remove_child(req, child_req);
nvme_request_free_children(child_req);
nvme_free_request(child_req);
}
}
if (spdk_unlikely(child_req_failed)) {
/* part of children requests have been submitted,
* return success for this case.
*/
if (req->num_children) {
return 0;
}
goto error;
}
return rc;
}
/* queue those requests which matches with opcode in err_cmd list */
if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head))) {
TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) {
if (!cmd->do_not_submit) {
continue;
}
if ((cmd->opc == req->cmd.opc) && cmd->err_count) {
/* add to error request list and set cpl */
req->timeout_tsc = cmd->timeout_tsc;
req->submit_tick = spdk_get_ticks();
req->cpl.status.sct = cmd->status.sct;
req->cpl.status.sc = cmd->status.sc;
STAILQ_INSERT_TAIL(&qpair->err_req_head, req, stailq);
cmd->err_count--;
return 0;
}
}
}
if (spdk_unlikely(ctrlr->is_failed)) {
rc = -ENXIO;
goto error;
}
/* assign submit_tick before submitting req to specific transport */
if (spdk_unlikely(ctrlr->timeout_enabled)) {
if (req->submit_tick == 0) { /* req submitted for the first time */
req->submit_tick = spdk_get_ticks();
req->timed_out = false;
}
} else {
req->submit_tick = 0;
}
if (spdk_likely(qpair->is_enabled)) {
rc = nvme_transport_qpair_submit_request(qpair, req);
} else if (nvme_qpair_is_admin_queue(qpair) && req->cmd.opc == SPDK_NVME_OPC_FABRIC) {
/* Always allow fabrics commands through on the admin qpair - these get
* the controller out of reset state.
*/
rc = nvme_transport_qpair_submit_request(qpair, req);
} else {
/* The controller is being reset - queue this request and
* submit it later when the reset is completed.
*/
return -EAGAIN;
}
if (spdk_likely(rc == 0)) {
return 0;
}
if (rc == -EAGAIN) {
return -EAGAIN;
}
error:
if (req->parent != NULL) {
nvme_request_remove_child(req->parent, req);
}
nvme_free_request(req);
return rc;
}
int
nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
int rc;
rc = _nvme_qpair_submit_request(qpair, req);
if (rc == -EAGAIN) {
STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
rc = 0;
}
return rc;
}
static int
nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
{
int rc;
rc = _nvme_qpair_submit_request(qpair, req);
if (spdk_unlikely(rc == -EAGAIN)) {
STAILQ_INSERT_HEAD(&qpair->queued_req, req, stailq);
}
return rc;
}
void
nvme_qpair_enable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = true;
}
void
nvme_qpair_disable(struct spdk_nvme_qpair *qpair)
{
qpair->is_enabled = false;
}
static void
nvme_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
{
nvme_qpair_abort_queued_reqs(qpair, dnr);
nvme_transport_qpair_abort_reqs(qpair, dnr);
}
int
spdk_nvme_qpair_add_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr,
struct spdk_nvme_qpair *qpair,
uint8_t opc, bool do_not_submit,
uint64_t timeout_in_us,
uint32_t err_count,
uint8_t sct, uint8_t sc)
{
struct nvme_error_cmd *entry, *cmd = NULL;
if (qpair == NULL) {
qpair = ctrlr->adminq;
}
TAILQ_FOREACH(entry, &qpair->err_cmd_head, link) {
if (entry->opc == opc) {
cmd = entry;
break;
}
}
if (cmd == NULL) {
cmd = spdk_zmalloc(sizeof(*cmd), 64, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (!cmd) {
return -ENOMEM;
}
TAILQ_INSERT_TAIL(&qpair->err_cmd_head, cmd, link);
}
cmd->do_not_submit = do_not_submit;
cmd->err_count = err_count;
cmd->timeout_tsc = timeout_in_us * spdk_get_ticks_hz() / 1000000ULL;
cmd->opc = opc;
cmd->status.sct = sct;
cmd->status.sc = sc;
return 0;
}
void
spdk_nvme_qpair_remove_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr,
struct spdk_nvme_qpair *qpair,
uint8_t opc)
{
struct nvme_error_cmd *cmd, *entry;
if (qpair == NULL) {
qpair = ctrlr->adminq;
}
TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) {
if (cmd->opc == opc) {
TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link);
spdk_free(cmd);
return;
}
}
return;
}