d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cam: allocate CCBs from UMA for SCSI and ATA IO

Browse Source 
This patch makes it possible for CAM to use small CCBs allocated
from an periph-specific UMA zone instead of the usual, huge ones.
The end result is that CCBs issued via da(4) take 544B (size of
ccb_scsiio) instead of the usual 2kB (size of 'union ccb', ~1.5kB,
rounded up by malloc(9)).  For ATA it's 272B.  We waste less
memory, we avoid zeroing the unused 1kB, and it should be easier
to allocate those CCBs in low memory conditions.  It should also
be possible to use uma_zone_reserve(9) to improve behaviour
in low memory conditions even further.

Note that this does not change the size, or the layout, of CCBs
as such.  CCBs get allocated in various different ways, in particular
on the stack, and I don't want to redo all that.  Instead, this
provides an opt-in mechanism for the periph to declare "my start()
callback is fine with receiving a CCB allocated from this UMA zone".
In other words, most of the code works exactly as it used to; the
change only happens to IOs issued by xpt_run_allockq(), which
is - conveniently - pretty much all that matters for performance.

The reason for doing it this way is that it's pretty small, localized
change, and can be implemented gradually and iteratively: take a
periph, make sure its start() callback only casts the CCBs it takes
to a particular type of CCB, for example ccb_scsiio, and that it only
casts CCBs returned by cam_periph_getccb() to that type, then add UMA
zone for that size, and declare it safe to XPT.

This is disabled by default.  Set 'kern.cam.ada.enable_uma_ccbs=1'
and 'kern.cam.da.enable_uma_ccbs=1' tunables to enable it.  Testing
is welcome; I will flip the default to enable in two weeks from now.

Reviewed By:	imp
Sponsored by:	NetApp, Inc.
Sponsored by:	Klara, Inc.
Differential Revision:	https://reviews.freebsd.org/D28674
This commit is contained in:
Edward Tomasz Napierala 2021-05-15 11:17:22 +01:00
parent 189f8eea13
commit 3394d4239b
7 changed files with 95 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
sys/cam/ata/ata_da.c
View File

@ -297,6 +297,8 @@ struct ada_softc {
char announce_buffer[ADA_ANNOUNCE_SZ];
};
static uma_zone_t ada_ccb_zone;
struct ada_quirk_entry {
struct scsi_inquiry_pattern inq_pat;
ada_quirks quirks;
@ -902,6 +904,7 @@ static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
static int ada_enable_biospeedup = 1;
static int ada_enable_uma_ccbs = 0;
static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"CAM Direct Access Disk driver");
@ -921,6 +924,8 @@ SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
&ada_write_cache, 0, "Enable disk write cache");
SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
&ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
&ada_enable_uma_ccbs, 0, "Use UMA for CCBs");
/*
* ADA_ORDEREDTAG_INTERVAL determines how often, relative
@ -1178,6 +1183,10 @@ adainit(void)
{
cam_status status;
ada_ccb_zone = uma_zcreate("ada_ccb",
sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
/*
* Install a global async callback. This callback will
* receive async callbacks like "new device found".
@ -1855,6 +1864,15 @@ adaregister(struct cam_periph *periph, void *arg)
"kern.cam.ada.%d.write_cache", periph->unit_number);
TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
/*
* Let XPT know we can use UMA-allocated CCBs.
*/
if (ada_enable_uma_ccbs) {
KASSERT(ada_ccb_zone != NULL,
("%s: NULL ada_ccb_zone", __func__));
periph->ccb_zone = ada_ccb_zone;
}
/*
* Set support flags based on the Identify data and quirks.
*/

7
sys/cam/ata/ata_xpt.c
View File

@ -1795,6 +1795,13 @@ static void
ata_action(union ccb *start_ccb)
{
if (start_ccb->ccb_h.func_code != XPT_ATA_IO) {
KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
("%s: ccb %p, func_code %#x should not be allocated "
"from UMA zone\n",
__func__, start_ccb, start_ccb->ccb_h.func_code));
}
switch (start_ccb->ccb_h.func_code) {
case XPT_SET_TRAN_SETTINGS:
{

14
sys/cam/cam_ccb.h
View File

@ -58,6 +58,12 @@
/* Struct definitions for CAM control blocks */
/* Common CCB header */
/* CCB memory allocation flags */
typedef enum {
CAM_CCB_FROM_UMA = 0x00000001,/* CCB from a periph UMA zone */
} ccb_alloc_flags;
/* CAM CCB flags */
typedef enum {
CAM_CDB_POINTER = 0x00000001,/* The CDB field is a pointer */
@ -341,7 +347,13 @@ struct ccb_hdr {
camq_entry xpt_links; /* For chaining in the XPT layer */
camq_entry sim_links; /* For chaining in the SIM layer */
camq_entry periph_links; /* For chaining in the type driver */
u_int32_t retry_count;
#if BYTE_ORDER == LITTLE_ENDIAN
u_int16_t retry_count;
u_int16_t alloc_flags; /* ccb_alloc_flags */
#else
u_int16_t alloc_flags; /* ccb_alloc_flags */
u_int16_t retry_count;
#endif
void (*cbfcnp)(struct cam_periph *, union ccb *);
/* Callback on completion function */
xpt_opcode func_code; /* XPT function code */

3
sys/cam/cam_periph.h
View File

@ -42,6 +42,8 @@
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <vm/uma.h>
#include <cam/cam_xpt.h>
struct devstat;
@ -147,6 +149,7 @@ struct cam_periph {
ac_callback_t *deferred_callback;
ac_code deferred_ac;
struct task periph_run_task;
uma_zone_t ccb_zone;
};
#define CAM_PERIPH_MAXMAPS 2

32
sys/cam/cam_xpt.c
View File

@ -4693,7 +4693,17 @@ xpt_alloc_ccb_nowait(void)
void
xpt_free_ccb(union ccb *free_ccb)
{
free(free_ccb, M_CAMCCB);
struct cam_periph *periph;
if (free_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) {
/*
* Looks like a CCB allocated from a periph UMA zone.
*/
periph = free_ccb->ccb_h.path->periph;
uma_zfree(periph->ccb_zone, free_ccb);
} else {
free(free_ccb, M_CAMCCB);
}
}
/* Private XPT functions */
@ -4707,10 +4717,18 @@ static union ccb *
xpt_get_ccb_nowait(struct cam_periph *periph)
{
union ccb *new_ccb;
int alloc_flags;
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
if (periph->ccb_zone != NULL) {
alloc_flags = CAM_CCB_FROM_UMA;
new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_NOWAIT);
} else {
alloc_flags = 0;
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
}
if (new_ccb == NULL)
return (NULL);
new_ccb->ccb_h.alloc_flags = alloc_flags;
periph->periph_allocated++;
cam_ccbq_take_opening(&periph->path->device->ccbq);
return (new_ccb);
@ -4720,9 +4738,17 @@ static union ccb *
xpt_get_ccb(struct cam_periph *periph)
{
union ccb *new_ccb;
int alloc_flags;
cam_periph_unlock(periph);
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
if (periph->ccb_zone != NULL) {
alloc_flags = CAM_CCB_FROM_UMA;
new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_WAITOK);
} else {
alloc_flags = 0;
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
}
new_ccb->ccb_h.alloc_flags = alloc_flags;
cam_periph_lock(periph);
periph->periph_allocated++;
cam_ccbq_take_opening(&periph->path->device->ccbq);

18
sys/cam/scsi/scsi_da.c
View File

@ -403,6 +403,8 @@ struct da_softc {
softc->delete_available &= ~(1 << delete_method); \
}
static uma_zone_t da_ccb_zone;
struct da_quirk_entry {
struct scsi_inquiry_pattern inq_pat;
da_quirks quirks;
@ -1557,6 +1559,7 @@ static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
static int da_disable_wp_detection = 0;
static int da_enable_biospeedup = 1;
static int da_enable_uma_ccbs = 0;
static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"CAM Direct Access Disk driver");
@ -1573,6 +1576,8 @@ SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN,
"Disable detection of write-protected disks");
SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
&da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
&da_enable_uma_ccbs, 0, "Use UMA for CCBs");
SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
@ -2011,6 +2016,10 @@ dainit(void)
NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
printf("dainit: shutdown event registration failed!\n");
}
da_ccb_zone = uma_zcreate("da_ccb",
sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL,
UMA_ALIGN_PTR, 0);
}
/*
@ -2848,6 +2857,15 @@ daregister(struct cam_periph *periph, void *arg)
TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
/*
* Let XPT know we can use UMA-allocated CCBs.
*/
if (da_enable_uma_ccbs) {
KASSERT(da_ccb_zone != NULL,
("%s: NULL da_ccb_zone", __func__));
periph->ccb_zone = da_ccb_zone;
}
/*
* Take an exclusive section lock on the periph while dastart is called
* to finish the probe. The lock will be dropped in dadone at the end

7
sys/cam/scsi/scsi_xpt.c
View File

@ -2625,6 +2625,13 @@ static void
scsi_action(union ccb *start_ccb)
{
if (start_ccb->ccb_h.func_code != XPT_SCSI_IO) {
KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
("%s: ccb %p, func_code %#x should not be allocated "
"from UMA zone\n",
__func__, start_ccb, start_ccb->ccb_h.func_code));
}
switch (start_ccb->ccb_h.func_code) {
case XPT_SET_TRAN_SETTINGS:
{