d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ioat: Dedupe operation enqueue logic

Browse Source 
Add generic hw descriptor struct and generic control flags struct, in
preparation for other kinds of IOAT operation.

Sponsored by:	EMC / Isilon Storage Division
This commit is contained in:
Conrad Meyer 2015-10-26 19:34:00 +00:00
parent 6aa2fe8969
commit 9e3bbf26a9
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=290020
2 changed files with 86 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
sys/dev/ioat/ioat.c
View File

@ -648,15 +648,18 @@ ioat_release(bus_dmaengine_t dmaengine)
mtx_unlock(&ioat->submit_lock); mtx_unlock(&ioat->submit_lock);
} }
struct bus_dmadesc * static struct ioat_descriptor *
ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, ioat_op_generic(struct ioat_softc *ioat, uint8_t op,
void *callback_arg, uint32_t flags) uint32_t size, uint64_t src, uint64_t dst,
bus_dmaengine_callback_t callback_fn, void *callback_arg,
uint32_t flags)
{ {
struct ioat_softc *ioat; struct ioat_generic_hw_descriptor *hw_desc;
struct ioat_descriptor *desc; struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
int mflags; int mflags;
mtx_assert(&ioat->submit_lock, MA_OWNED);
KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x", KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
flags & ~DMA_ALL_FLAGS)); flags & ~DMA_ALL_FLAGS));
if ((flags & DMA_NO_WAIT) != 0) if ((flags & DMA_NO_WAIT) != 0)
@ -664,31 +667,52 @@ ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
else else
mflags = M_WAITOK; mflags = M_WAITOK;
ioat = to_ioat_softc(dmaengine); if (size > ioat->max_xfer_size) {
mtx_assert(&ioat->submit_lock, MA_OWNED); ioat_log_message(0, "%s: max_xfer_size = %d, requested = %u\n",
__func__, ioat->max_xfer_size, (unsigned)size);
return (NULL);
}
if (ioat_reserve_space(ioat, 1, mflags) != 0) if (ioat_reserve_space(ioat, 1, mflags) != 0)
return (NULL); return (NULL);
CTR0(KTR_IOAT, __func__);
desc = ioat_get_ring_entry(ioat, ioat->head); desc = ioat_get_ring_entry(ioat, ioat->head);
hw_desc = desc->u.dma; hw_desc = desc->u.generic;
hw_desc->u.control_raw = 0; hw_desc->u.control_raw = 0;
hw_desc->u.control.null = 1; hw_desc->u.control_generic.op = op;
hw_desc->u.control.completion_update = 1; hw_desc->u.control_generic.completion_update = 1;
if ((flags & DMA_INT_EN) != 0) if ((flags & DMA_INT_EN) != 0)
hw_desc->u.control.int_enable = 1; hw_desc->u.control_generic.int_enable = 1;
hw_desc->size = 8; hw_desc->size = size;
hw_desc->src_addr = 0; hw_desc->src_addr = src;
hw_desc->dest_addr = 0; hw_desc->dest_addr = dst;
desc->bus_dmadesc.callback_fn = callback_fn; desc->bus_dmadesc.callback_fn = callback_fn;
desc->bus_dmadesc.callback_arg = callback_arg; desc->bus_dmadesc.callback_arg = callback_arg;
return (desc);
}
struct bus_dmadesc *
ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn,
void *callback_arg, uint32_t flags)
{
struct ioat_dma_hw_descriptor *hw_desc;
struct ioat_descriptor *desc;
struct ioat_softc *ioat;
CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine);
desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn,
callback_arg, flags);
if (desc == NULL)
return (NULL);
hw_desc = desc->u.dma;
hw_desc->u.control.null = 1;
ioat_submit_single(ioat); ioat_submit_single(ioat);
return (&desc->bus_dmadesc); return (&desc->bus_dmadesc);
} }
@ -698,51 +722,28 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst,
bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn,
void *callback_arg, uint32_t flags) void *callback_arg, uint32_t flags)
{ {
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc; struct ioat_dma_hw_descriptor *hw_desc;
struct ioat_descriptor *desc;
struct ioat_softc *ioat; struct ioat_softc *ioat;
int mflags;
KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
flags & ~DMA_ALL_FLAGS));
if ((flags & DMA_NO_WAIT) != 0)
mflags = M_NOWAIT;
else
mflags = M_WAITOK;
CTR0(KTR_IOAT, __func__);
ioat = to_ioat_softc(dmaengine); ioat = to_ioat_softc(dmaengine);
mtx_assert(&ioat->submit_lock, MA_OWNED);
if (len > ioat->max_xfer_size) { if (((src | dst) & (0xffffull << 48)) != 0) {
ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n", ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n",
__func__, ioat->max_xfer_size, (int)len); __func__);
return (NULL); return (NULL);
} }
if (ioat_reserve_space(ioat, 1, mflags) != 0) desc = ioat_op_generic(ioat, IOAT_OP_COPY, len, src, dst, callback_fn,
callback_arg, flags);
if (desc == NULL)
return (NULL); return (NULL);
CTR0(KTR_IOAT, __func__);
desc = ioat_get_ring_entry(ioat, ioat->head);
hw_desc = desc->u.dma; hw_desc = desc->u.dma;
hw_desc->u.control_raw = 0;
hw_desc->u.control.completion_update = 1;
if ((flags & DMA_INT_EN) != 0)
hw_desc->u.control.int_enable = 1;
hw_desc->size = len;
hw_desc->src_addr = src;
hw_desc->dest_addr = dst;
if (g_ioat_debug_level >= 3) if (g_ioat_debug_level >= 3)
dump_descriptor(hw_desc); dump_descriptor(hw_desc);
desc->bus_dmadesc.callback_fn = callback_fn;
desc->bus_dmadesc.callback_arg = callback_arg;
ioat_submit_single(ioat); ioat_submit_single(ioat);
return (&desc->bus_dmadesc); return (&desc->bus_dmadesc);
} }
@ -767,7 +768,7 @@ ioat_get_ring_space(struct ioat_softc *ioat)
static struct ioat_descriptor * static struct ioat_descriptor *
ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags) ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
{ {
struct ioat_dma_hw_descriptor *hw_desc; struct ioat_generic_hw_descriptor *hw_desc;
struct ioat_descriptor *desc; struct ioat_descriptor *desc;
int error, busdmaflag; int error, busdmaflag;
@ -788,7 +789,7 @@ ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
if (hw_desc == NULL) if (hw_desc == NULL)
goto out; goto out;
desc->u.dma = hw_desc; desc->u.generic = hw_desc;
error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
@ -811,8 +812,8 @@ ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc)
if (desc == NULL) if (desc == NULL)
return; return;
if (desc->u.dma) if (desc->u.generic)
bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma, bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic,
ioat->hw_desc_map); ioat->hw_desc_map);
free(desc, M_IOAT); free(desc, M_IOAT);
} }

34
sys/dev/ioat/ioat_internal.h
View File

@ -122,10 +122,39 @@ SYSCTL_DECL(_hw_ioat);
extern int g_ioat_debug_level; extern int g_ioat_debug_level;
struct generic_dma_control {
uint32_t int_enable:1;
uint32_t src_snoop_disable:1;
uint32_t dest_snoop_disable:1;
uint32_t completion_update:1;
uint32_t fence:1;
uint32_t reserved1:1;
uint32_t src_page_break:1;
uint32_t dest_page_break:1;
uint32_t bundle:1;
uint32_t dest_dca:1;
uint32_t hint:1;
uint32_t reserved2:13;
uint32_t op:8;
};
struct ioat_generic_hw_descriptor {
uint32_t size;
union {
uint32_t control_raw;
struct generic_dma_control control_generic;
} u;
uint64_t src_addr;
uint64_t dest_addr;
uint64_t next;
uint64_t reserved[4];
};
struct ioat_dma_hw_descriptor { struct ioat_dma_hw_descriptor {
uint32_t size; uint32_t size;
union { union {
uint32_t control_raw; uint32_t control_raw;
struct generic_dma_control control_generic;
struct { struct {
uint32_t int_enable:1; uint32_t int_enable:1;
uint32_t src_snoop_disable:1; uint32_t src_snoop_disable:1;
@ -156,6 +185,7 @@ struct ioat_fill_hw_descriptor {
uint32_t size; uint32_t size;
union { union {
uint32_t control_raw; uint32_t control_raw;
struct generic_dma_control control_generic;
struct { struct {
uint32_t int_enable:1; uint32_t int_enable:1;
uint32_t reserved:1; uint32_t reserved:1;
@ -183,6 +213,7 @@ struct ioat_xor_hw_descriptor {
uint32_t size; uint32_t size;
union { union {
uint32_t control_raw; uint32_t control_raw;
struct generic_dma_control control_generic;
struct { struct {
uint32_t int_enable:1; uint32_t int_enable:1;
uint32_t src_snoop_disable:1; uint32_t src_snoop_disable:1;
@ -220,6 +251,7 @@ struct ioat_pq_hw_descriptor {
uint32_t size; uint32_t size;
union { union {
uint32_t control_raw; uint32_t control_raw;
struct generic_dma_control control_generic;
struct { struct {
uint32_t int_enable:1; uint32_t int_enable:1;
uint32_t src_snoop_disable:1; uint32_t src_snoop_disable:1;
@ -261,6 +293,7 @@ struct ioat_pq_update_hw_descriptor {
uint32_t size; uint32_t size;
union { union {
uint32_t control_raw; uint32_t control_raw;
struct generic_dma_control control_generic;
struct { struct {
uint32_t int_enable:1; uint32_t int_enable:1;
uint32_t src_snoop_disable:1; uint32_t src_snoop_disable:1;
@ -300,6 +333,7 @@ struct bus_dmadesc {
struct ioat_descriptor { struct ioat_descriptor {
struct bus_dmadesc bus_dmadesc; struct bus_dmadesc bus_dmadesc;
union { union {
struct ioat_generic_hw_descriptor *generic;
struct ioat_dma_hw_descriptor *dma; struct ioat_dma_hw_descriptor *dma;
struct ioat_fill_hw_descriptor *fill; struct ioat_fill_hw_descriptor *fill;
struct ioat_xor_hw_descriptor *xor; struct ioat_xor_hw_descriptor *xor;