cam: allocate CCBs from UMA for SCSI and ATA IO
This patch makes it possible for CAM to use small CCBs allocated from an periph-specific UMA zone instead of the usual, huge ones. The end result is that CCBs issued via da(4) take 544B (size of ccb_scsiio) instead of the usual 2kB (size of 'union ccb', ~1.5kB, rounded up by malloc(9)). For ATA it's 272B. We waste less memory, we avoid zeroing the unused 1kB, and it should be easier to allocate those CCBs in low memory conditions. It should also be possible to use uma_zone_reserve(9) to improve behaviour in low memory conditions even further. Note that this does not change the size, or the layout, of CCBs as such. CCBs get allocated in various different ways, in particular on the stack, and I don't want to redo all that. Instead, this provides an opt-in mechanism for the periph to declare "my start() callback is fine with receiving a CCB allocated from this UMA zone". In other words, most of the code works exactly as it used to; the change only happens to IOs issued by xpt_run_allockq(), which is - conveniently - pretty much all that matters for performance. The reason for doing it this way is that it's pretty small, localized change, and can be implemented gradually and iteratively: take a periph, make sure its start() callback only casts the CCBs it takes to a particular type of CCB, for example ccb_scsiio, and that it only casts CCBs returned by cam_periph_getccb() to that type, then add UMA zone for that size, and declare it safe to XPT. This is disabled by default. Set 'kern.cam.ada.enable_uma_ccbs=1' and 'kern.cam.da.enable_uma_ccbs=1' tunables to enable it. Testing is welcome; I will flip the default to enable in two weeks from now. Reviewed By: imp Sponsored by: NetApp, Inc. Sponsored by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D28674
This commit is contained in:
parent
189f8eea13
commit
3394d4239b
@ -297,6 +297,8 @@ struct ada_softc {
|
||||
char announce_buffer[ADA_ANNOUNCE_SZ];
|
||||
};
|
||||
|
||||
static uma_zone_t ada_ccb_zone;
|
||||
|
||||
struct ada_quirk_entry {
|
||||
struct scsi_inquiry_pattern inq_pat;
|
||||
ada_quirks quirks;
|
||||
@ -902,6 +904,7 @@ static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
|
||||
static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
|
||||
static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
|
||||
static int ada_enable_biospeedup = 1;
|
||||
static int ada_enable_uma_ccbs = 0;
|
||||
|
||||
static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
|
||||
"CAM Direct Access Disk driver");
|
||||
@ -921,6 +924,8 @@ SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
|
||||
&ada_write_cache, 0, "Enable disk write cache");
|
||||
SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
|
||||
&ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
|
||||
SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
|
||||
&ada_enable_uma_ccbs, 0, "Use UMA for CCBs");
|
||||
|
||||
/*
|
||||
* ADA_ORDEREDTAG_INTERVAL determines how often, relative
|
||||
@ -1178,6 +1183,10 @@ adainit(void)
|
||||
{
|
||||
cam_status status;
|
||||
|
||||
ada_ccb_zone = uma_zcreate("ada_ccb",
|
||||
sizeof(struct ccb_ataio), NULL, NULL, NULL, NULL,
|
||||
UMA_ALIGN_PTR, 0);
|
||||
|
||||
/*
|
||||
* Install a global async callback. This callback will
|
||||
* receive async callbacks like "new device found".
|
||||
@ -1855,6 +1864,15 @@ adaregister(struct cam_periph *periph, void *arg)
|
||||
"kern.cam.ada.%d.write_cache", periph->unit_number);
|
||||
TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
|
||||
|
||||
/*
|
||||
* Let XPT know we can use UMA-allocated CCBs.
|
||||
*/
|
||||
if (ada_enable_uma_ccbs) {
|
||||
KASSERT(ada_ccb_zone != NULL,
|
||||
("%s: NULL ada_ccb_zone", __func__));
|
||||
periph->ccb_zone = ada_ccb_zone;
|
||||
}
|
||||
|
||||
/*
|
||||
* Set support flags based on the Identify data and quirks.
|
||||
*/
|
||||
|
@ -1795,6 +1795,13 @@ static void
|
||||
ata_action(union ccb *start_ccb)
|
||||
{
|
||||
|
||||
if (start_ccb->ccb_h.func_code != XPT_ATA_IO) {
|
||||
KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
|
||||
("%s: ccb %p, func_code %#x should not be allocated "
|
||||
"from UMA zone\n",
|
||||
__func__, start_ccb, start_ccb->ccb_h.func_code));
|
||||
}
|
||||
|
||||
switch (start_ccb->ccb_h.func_code) {
|
||||
case XPT_SET_TRAN_SETTINGS:
|
||||
{
|
||||
|
@ -58,6 +58,12 @@
|
||||
/* Struct definitions for CAM control blocks */
|
||||
|
||||
/* Common CCB header */
|
||||
|
||||
/* CCB memory allocation flags */
|
||||
typedef enum {
|
||||
CAM_CCB_FROM_UMA = 0x00000001,/* CCB from a periph UMA zone */
|
||||
} ccb_alloc_flags;
|
||||
|
||||
/* CAM CCB flags */
|
||||
typedef enum {
|
||||
CAM_CDB_POINTER = 0x00000001,/* The CDB field is a pointer */
|
||||
@ -341,7 +347,13 @@ struct ccb_hdr {
|
||||
camq_entry xpt_links; /* For chaining in the XPT layer */
|
||||
camq_entry sim_links; /* For chaining in the SIM layer */
|
||||
camq_entry periph_links; /* For chaining in the type driver */
|
||||
u_int32_t retry_count;
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
u_int16_t retry_count;
|
||||
u_int16_t alloc_flags; /* ccb_alloc_flags */
|
||||
#else
|
||||
u_int16_t alloc_flags; /* ccb_alloc_flags */
|
||||
u_int16_t retry_count;
|
||||
#endif
|
||||
void (*cbfcnp)(struct cam_periph *, union ccb *);
|
||||
/* Callback on completion function */
|
||||
xpt_opcode func_code; /* XPT function code */
|
||||
|
@ -42,6 +42,8 @@
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/taskqueue.h>
|
||||
|
||||
#include <vm/uma.h>
|
||||
|
||||
#include <cam/cam_xpt.h>
|
||||
|
||||
struct devstat;
|
||||
@ -147,6 +149,7 @@ struct cam_periph {
|
||||
ac_callback_t *deferred_callback;
|
||||
ac_code deferred_ac;
|
||||
struct task periph_run_task;
|
||||
uma_zone_t ccb_zone;
|
||||
};
|
||||
|
||||
#define CAM_PERIPH_MAXMAPS 2
|
||||
|
@ -4693,7 +4693,17 @@ xpt_alloc_ccb_nowait(void)
|
||||
void
|
||||
xpt_free_ccb(union ccb *free_ccb)
|
||||
{
|
||||
free(free_ccb, M_CAMCCB);
|
||||
struct cam_periph *periph;
|
||||
|
||||
if (free_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) {
|
||||
/*
|
||||
* Looks like a CCB allocated from a periph UMA zone.
|
||||
*/
|
||||
periph = free_ccb->ccb_h.path->periph;
|
||||
uma_zfree(periph->ccb_zone, free_ccb);
|
||||
} else {
|
||||
free(free_ccb, M_CAMCCB);
|
||||
}
|
||||
}
|
||||
|
||||
/* Private XPT functions */
|
||||
@ -4707,10 +4717,18 @@ static union ccb *
|
||||
xpt_get_ccb_nowait(struct cam_periph *periph)
|
||||
{
|
||||
union ccb *new_ccb;
|
||||
int alloc_flags;
|
||||
|
||||
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
|
||||
if (periph->ccb_zone != NULL) {
|
||||
alloc_flags = CAM_CCB_FROM_UMA;
|
||||
new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_NOWAIT);
|
||||
} else {
|
||||
alloc_flags = 0;
|
||||
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
|
||||
}
|
||||
if (new_ccb == NULL)
|
||||
return (NULL);
|
||||
new_ccb->ccb_h.alloc_flags = alloc_flags;
|
||||
periph->periph_allocated++;
|
||||
cam_ccbq_take_opening(&periph->path->device->ccbq);
|
||||
return (new_ccb);
|
||||
@ -4720,9 +4738,17 @@ static union ccb *
|
||||
xpt_get_ccb(struct cam_periph *periph)
|
||||
{
|
||||
union ccb *new_ccb;
|
||||
int alloc_flags;
|
||||
|
||||
cam_periph_unlock(periph);
|
||||
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
|
||||
if (periph->ccb_zone != NULL) {
|
||||
alloc_flags = CAM_CCB_FROM_UMA;
|
||||
new_ccb = uma_zalloc(periph->ccb_zone, M_ZERO|M_WAITOK);
|
||||
} else {
|
||||
alloc_flags = 0;
|
||||
new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
|
||||
}
|
||||
new_ccb->ccb_h.alloc_flags = alloc_flags;
|
||||
cam_periph_lock(periph);
|
||||
periph->periph_allocated++;
|
||||
cam_ccbq_take_opening(&periph->path->device->ccbq);
|
||||
|
@ -403,6 +403,8 @@ struct da_softc {
|
||||
softc->delete_available &= ~(1 << delete_method); \
|
||||
}
|
||||
|
||||
static uma_zone_t da_ccb_zone;
|
||||
|
||||
struct da_quirk_entry {
|
||||
struct scsi_inquiry_pattern inq_pat;
|
||||
da_quirks quirks;
|
||||
@ -1557,6 +1559,7 @@ static sbintime_t da_default_softtimeout = DA_DEFAULT_SOFTTIMEOUT;
|
||||
static int da_send_ordered = DA_DEFAULT_SEND_ORDERED;
|
||||
static int da_disable_wp_detection = 0;
|
||||
static int da_enable_biospeedup = 1;
|
||||
static int da_enable_uma_ccbs = 0;
|
||||
|
||||
static SYSCTL_NODE(_kern_cam, OID_AUTO, da, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
|
||||
"CAM Direct Access Disk driver");
|
||||
@ -1573,6 +1576,8 @@ SYSCTL_INT(_kern_cam_da, OID_AUTO, disable_wp_detection, CTLFLAG_RWTUN,
|
||||
"Disable detection of write-protected disks");
|
||||
SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
|
||||
&da_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
|
||||
SYSCTL_INT(_kern_cam_da, OID_AUTO, enable_uma_ccbs, CTLFLAG_RWTUN,
|
||||
&da_enable_uma_ccbs, 0, "Use UMA for CCBs");
|
||||
|
||||
SYSCTL_PROC(_kern_cam_da, OID_AUTO, default_softtimeout,
|
||||
CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, NULL, 0,
|
||||
@ -2011,6 +2016,10 @@ dainit(void)
|
||||
NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
|
||||
printf("dainit: shutdown event registration failed!\n");
|
||||
}
|
||||
|
||||
da_ccb_zone = uma_zcreate("da_ccb",
|
||||
sizeof(struct ccb_scsiio), NULL, NULL, NULL, NULL,
|
||||
UMA_ALIGN_PTR, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -2848,6 +2857,15 @@ daregister(struct cam_periph *periph, void *arg)
|
||||
|
||||
TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
|
||||
|
||||
/*
|
||||
* Let XPT know we can use UMA-allocated CCBs.
|
||||
*/
|
||||
if (da_enable_uma_ccbs) {
|
||||
KASSERT(da_ccb_zone != NULL,
|
||||
("%s: NULL da_ccb_zone", __func__));
|
||||
periph->ccb_zone = da_ccb_zone;
|
||||
}
|
||||
|
||||
/*
|
||||
* Take an exclusive section lock on the periph while dastart is called
|
||||
* to finish the probe. The lock will be dropped in dadone at the end
|
||||
|
@ -2625,6 +2625,13 @@ static void
|
||||
scsi_action(union ccb *start_ccb)
|
||||
{
|
||||
|
||||
if (start_ccb->ccb_h.func_code != XPT_SCSI_IO) {
|
||||
KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
|
||||
("%s: ccb %p, func_code %#x should not be allocated "
|
||||
"from UMA zone\n",
|
||||
__func__, start_ccb, start_ccb->ccb_h.func_code));
|
||||
}
|
||||
|
||||
switch (start_ccb->ccb_h.func_code) {
|
||||
case XPT_SET_TRAN_SETTINGS:
|
||||
{
|
||||
|
Loading…
Reference in New Issue
Block a user