2018-10-29 11:29:03 +00:00
|
|
|
/*-
|
|
|
|
* BSD LICENSE
|
|
|
|
*
|
|
|
|
* Copyright (c) Intel Corporation.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
*
|
|
|
|
* * Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in
|
|
|
|
* the documentation and/or other materials provided with the
|
|
|
|
* distribution.
|
|
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
|
|
* contributors may be used to endorse or promote products derived
|
|
|
|
* from this software without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef FTL_IO_H
|
|
|
|
#define FTL_IO_H
|
|
|
|
|
2018-10-29 12:17:34 +00:00
|
|
|
#include "spdk/stdinc.h"
|
|
|
|
#include "spdk/nvme.h"
|
|
|
|
#include "spdk/ftl.h"
|
|
|
|
|
2018-10-29 11:29:03 +00:00
|
|
|
#include "ftl_ppa.h"
|
|
|
|
#include "ftl_trace.h"
|
|
|
|
|
|
|
|
struct spdk_ftl_dev;
|
|
|
|
struct ftl_rwb_batch;
|
|
|
|
struct ftl_band;
|
|
|
|
struct ftl_io;
|
|
|
|
|
2019-04-26 14:53:13 +00:00
|
|
|
typedef int (*ftl_md_pack_fn)(struct ftl_band *);
|
2019-05-27 12:56:48 +00:00
|
|
|
typedef void (*ftl_io_fn)(struct ftl_io *, void *, int);
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* IO flags */
|
|
|
|
enum ftl_io_flags {
|
|
|
|
/* Indicates whether IO is already initialized */
|
|
|
|
FTL_IO_INITIALIZED = (1 << 0),
|
|
|
|
/* Internal based IO (defrag, metadata etc.) */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_INTERNAL = (1 << 1),
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Indicates that the IO should not go through if there's */
|
|
|
|
/* already another one scheduled to the same LBA */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_WEAK = (1 << 2),
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Indicates that the IO is used for padding */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_PAD = (1 << 3),
|
2018-10-29 11:29:03 +00:00
|
|
|
/* The IO operates on metadata */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_MD = (1 << 4),
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Using PPA instead of LBA */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_PPA_MODE = (1 << 5),
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Indicates that IO contains noncontiguous LBAs */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_VECTOR_LBA = (1 << 6),
|
2019-03-19 15:29:34 +00:00
|
|
|
/* Indicates that IO is being retried */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_RETRY = (1 << 7),
|
2019-04-05 11:31:19 +00:00
|
|
|
/* The IO is directed to non-volatile cache */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_CACHE = (1 << 8),
|
2019-05-23 08:34:30 +00:00
|
|
|
/* Indicates that PPA should be taken from IO struct, */
|
|
|
|
/* not assigned by wptr, only works if wptr is also in direct mode */
|
2019-05-13 12:11:10 +00:00
|
|
|
FTL_IO_DIRECT_ACCESS = (1 << 9),
|
2019-06-07 10:02:48 +00:00
|
|
|
/* Bypass the non-volatile cache */
|
|
|
|
FTL_IO_BYPASS_CACHE = (1 << 10),
|
2018-10-29 11:29:03 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
enum ftl_io_type {
|
|
|
|
FTL_IO_READ,
|
|
|
|
FTL_IO_WRITE,
|
|
|
|
FTL_IO_ERASE,
|
|
|
|
};
|
|
|
|
|
|
|
|
struct ftl_io_init_opts {
|
|
|
|
struct spdk_ftl_dev *dev;
|
|
|
|
|
|
|
|
/* IO descriptor */
|
|
|
|
struct ftl_io *io;
|
|
|
|
|
2019-03-22 15:45:46 +00:00
|
|
|
/* Parent request */
|
|
|
|
struct ftl_io *parent;
|
|
|
|
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Size of IO descriptor */
|
|
|
|
size_t size;
|
|
|
|
|
|
|
|
/* IO flags */
|
|
|
|
int flags;
|
|
|
|
|
|
|
|
/* IO type */
|
|
|
|
enum ftl_io_type type;
|
|
|
|
|
|
|
|
/* RWB entry */
|
|
|
|
struct ftl_rwb_batch *rwb_batch;
|
|
|
|
|
|
|
|
/* Band to which the IO is directed */
|
|
|
|
struct ftl_band *band;
|
|
|
|
|
2019-05-09 14:04:26 +00:00
|
|
|
/* Number of logical blocks */
|
|
|
|
size_t lbk_cnt;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* Data */
|
|
|
|
void *data;
|
|
|
|
|
|
|
|
/* Metadata */
|
|
|
|
void *md;
|
|
|
|
|
2019-05-27 12:56:48 +00:00
|
|
|
/* Callback's function */
|
|
|
|
ftl_io_fn cb_fn;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* Callback's context */
|
2019-05-27 12:56:48 +00:00
|
|
|
void *cb_ctx;
|
2018-10-29 11:29:03 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct ftl_io_channel {
|
2019-05-21 10:17:36 +00:00
|
|
|
/* Device */
|
|
|
|
struct spdk_ftl_dev *dev;
|
2018-10-29 11:29:03 +00:00
|
|
|
/* IO pool element size */
|
|
|
|
size_t elem_size;
|
|
|
|
/* IO pool */
|
|
|
|
struct spdk_mempool *io_pool;
|
2019-03-26 15:33:45 +00:00
|
|
|
/* Persistent cache IO channel */
|
|
|
|
struct spdk_io_channel *cache_ioch;
|
2018-10-29 11:29:03 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* General IO descriptor */
|
|
|
|
struct ftl_io {
|
|
|
|
/* Device */
|
|
|
|
struct spdk_ftl_dev *dev;
|
|
|
|
|
|
|
|
/* IO channel */
|
2019-03-22 15:45:46 +00:00
|
|
|
struct spdk_io_channel *ioch;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
union {
|
|
|
|
/* LBA table */
|
2019-05-08 10:46:58 +00:00
|
|
|
uint64_t *vector;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* First LBA */
|
2019-05-08 10:46:58 +00:00
|
|
|
uint64_t single;
|
|
|
|
} lba;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* First PPA */
|
|
|
|
struct ftl_ppa ppa;
|
|
|
|
|
|
|
|
/* Number of processed lbks */
|
|
|
|
size_t pos;
|
|
|
|
|
|
|
|
/* Number of lbks */
|
|
|
|
size_t lbk_cnt;
|
|
|
|
|
2019-05-21 10:24:57 +00:00
|
|
|
#define FTL_IO_MAX_IOVEC 64
|
|
|
|
struct iovec iov[FTL_IO_MAX_IOVEC];
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* Metadata */
|
|
|
|
void *md;
|
|
|
|
|
|
|
|
/* Number of IO vectors */
|
|
|
|
size_t iov_cnt;
|
|
|
|
|
|
|
|
/* Position within the iovec */
|
|
|
|
size_t iov_pos;
|
|
|
|
|
|
|
|
/* Offset within the iovec (in lbks) */
|
|
|
|
size_t iov_off;
|
|
|
|
|
|
|
|
/* RWB entry (valid only for RWB-based IO) */
|
|
|
|
struct ftl_rwb_batch *rwb_batch;
|
|
|
|
|
|
|
|
/* Band this IO is being written to */
|
|
|
|
struct ftl_band *band;
|
|
|
|
|
|
|
|
/* Request status */
|
|
|
|
int status;
|
|
|
|
|
|
|
|
/* Number of split requests */
|
|
|
|
size_t req_cnt;
|
|
|
|
|
2019-05-27 12:56:48 +00:00
|
|
|
/* Callback's function */
|
|
|
|
ftl_io_fn cb_fn;
|
|
|
|
|
|
|
|
/* Callback's context */
|
|
|
|
void *cb_ctx;
|
|
|
|
|
|
|
|
/* User callback function */
|
|
|
|
spdk_ftl_fn user_fn;
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
/* Flags */
|
|
|
|
int flags;
|
|
|
|
|
|
|
|
/* IO type */
|
|
|
|
enum ftl_io_type type;
|
|
|
|
|
2019-03-22 15:45:46 +00:00
|
|
|
/* Done flag */
|
|
|
|
bool done;
|
|
|
|
|
|
|
|
/* Parent request */
|
|
|
|
struct ftl_io *parent;
|
|
|
|
/* Child requests list */
|
|
|
|
LIST_HEAD(, ftl_io) children;
|
|
|
|
/* Child list link */
|
|
|
|
LIST_ENTRY(ftl_io) child_entry;
|
|
|
|
/* Children lock */
|
|
|
|
pthread_spinlock_t lock;
|
|
|
|
|
2018-10-29 11:29:03 +00:00
|
|
|
/* Trace group id */
|
2018-11-30 17:33:15 +00:00
|
|
|
uint64_t trace;
|
2019-03-19 15:29:34 +00:00
|
|
|
|
|
|
|
TAILQ_ENTRY(ftl_io) retry_entry;
|
2018-10-29 11:29:03 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Metadata IO */
|
|
|
|
struct ftl_md_io {
|
|
|
|
/* Parent IO structure */
|
|
|
|
struct ftl_io io;
|
|
|
|
|
|
|
|
/* Serialization/deserialization callback */
|
|
|
|
ftl_md_pack_fn pack_fn;
|
|
|
|
|
2019-05-27 12:56:48 +00:00
|
|
|
/* Callback's function */
|
|
|
|
ftl_io_fn cb_fn;
|
|
|
|
|
|
|
|
/* Callback's context */
|
|
|
|
void *cb_ctx;
|
2018-10-29 11:29:03 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
ftl_io_mode_ppa(const struct ftl_io *io)
|
|
|
|
{
|
|
|
|
return io->flags & FTL_IO_PPA_MODE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
ftl_io_mode_lba(const struct ftl_io *io)
|
|
|
|
{
|
|
|
|
return !ftl_io_mode_ppa(io);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline bool
|
|
|
|
ftl_io_done(const struct ftl_io *io)
|
|
|
|
{
|
2019-04-04 14:45:22 +00:00
|
|
|
return io->req_cnt == 0 &&
|
|
|
|
io->pos == io->lbk_cnt &&
|
|
|
|
!(io->flags & FTL_IO_RETRY);
|
2018-10-29 11:29:03 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch);
|
2019-03-22 15:45:46 +00:00
|
|
|
struct ftl_io *ftl_io_alloc_child(struct ftl_io *parent);
|
2019-04-04 14:45:22 +00:00
|
|
|
void ftl_io_fail(struct ftl_io *io, int status);
|
2018-10-29 11:29:03 +00:00
|
|
|
void ftl_io_free(struct ftl_io *io);
|
|
|
|
struct ftl_io *ftl_io_init_internal(const struct ftl_io_init_opts *opts);
|
2019-05-27 12:56:48 +00:00
|
|
|
void ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb,
|
2018-10-29 11:29:03 +00:00
|
|
|
void *ctx, int flags, int type);
|
|
|
|
void ftl_io_clear(struct ftl_io *io);
|
2019-03-26 10:12:18 +00:00
|
|
|
void ftl_io_inc_req(struct ftl_io *io);
|
|
|
|
void ftl_io_dec_req(struct ftl_io *io);
|
2018-10-29 11:29:03 +00:00
|
|
|
struct iovec *ftl_io_iovec(struct ftl_io *io);
|
2019-04-05 12:08:42 +00:00
|
|
|
uint64_t ftl_io_current_lba(const struct ftl_io *io);
|
|
|
|
uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset);
|
2019-04-04 14:45:22 +00:00
|
|
|
void ftl_io_advance(struct ftl_io *io, size_t lbk_cnt);
|
2018-10-29 11:29:03 +00:00
|
|
|
size_t ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt);
|
|
|
|
void *ftl_io_iovec_addr(struct ftl_io *io);
|
|
|
|
size_t ftl_io_iovec_len_left(struct ftl_io *io);
|
|
|
|
struct ftl_io *ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,
|
2019-05-27 12:56:48 +00:00
|
|
|
struct ftl_rwb_batch *entry, ftl_io_fn cb);
|
|
|
|
struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb);
|
2019-05-15 11:05:06 +00:00
|
|
|
struct ftl_io *ftl_io_user_init(struct spdk_io_channel *ioch, uint64_t lba, size_t lbk_cnt,
|
|
|
|
struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn,
|
|
|
|
void *cb_arg, int type);
|
2018-10-29 11:29:03 +00:00
|
|
|
void *ftl_io_get_md(const struct ftl_io *io);
|
|
|
|
void ftl_io_complete(struct ftl_io *io);
|
2019-05-09 14:04:26 +00:00
|
|
|
void ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt);
|
2018-10-29 11:29:03 +00:00
|
|
|
void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status);
|
2019-04-05 11:31:19 +00:00
|
|
|
void ftl_io_reset(struct ftl_io *io);
|
2019-05-15 09:18:10 +00:00
|
|
|
void ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *));
|
2018-10-29 11:29:03 +00:00
|
|
|
|
|
|
|
#endif /* FTL_IO_H */
|