d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bunch of microoptimizations to reduce dereferences and cache collisions.

Browse Source
This commit is contained in:
mav 2014-09-08 12:11:49 +00:00
parent 7105bcbd70
commit 07b6763268
3 changed files with 137 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

277
sys/dev/ahci/ahci.c
View File

@ -61,31 +61,31 @@ static void ahci_ch_pm(void *arg);
static void ahci_ch_intr(void *arg);
static void ahci_ch_intr_direct(void *arg);
static void ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus);
static void ahci_begin_transaction(device_t dev, union ccb *ccb);
static void ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb);
static void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
static void ahci_execute_transaction(struct ahci_slot *slot);
static void ahci_timeout(struct ahci_slot *slot);
static void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et);
static int ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
static int ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
static void ahci_dmainit(device_t dev);
static void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
static void ahci_dmafini(device_t dev);
static void ahci_slotsalloc(device_t dev);
static void ahci_slotsfree(device_t dev);
static void ahci_reset(device_t dev);
static void ahci_start(device_t dev, int fbs);
static void ahci_stop(device_t dev);
static void ahci_clo(device_t dev);
static void ahci_start_fr(device_t dev);
static void ahci_stop_fr(device_t dev);
static void ahci_reset(struct ahci_channel *ch);
static void ahci_start(struct ahci_channel *ch, int fbs);
static void ahci_stop(struct ahci_channel *ch);
static void ahci_clo(struct ahci_channel *ch);
static void ahci_start_fr(struct ahci_channel *ch);
static void ahci_stop_fr(struct ahci_channel *ch);
static int ahci_sata_connect(struct ahci_channel *ch);
static int ahci_sata_phy_reset(device_t dev);
static int ahci_wait_ready(device_t dev, int t, int t0);
static int ahci_sata_phy_reset(struct ahci_channel *ch);
static int ahci_wait_ready(struct ahci_channel *ch, int t, int t0);
static void ahci_issue_recovery(device_t dev);
static void ahci_process_read_log(device_t dev, union ccb *ccb);
static void ahci_process_request_sense(device_t dev, union ccb *ccb);
static void ahci_issue_recovery(struct ahci_channel *ch);
static void ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb);
static void ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb);
static void ahciaction(struct cam_sim *sim, union ccb *ccb);
static void ahcipoll(struct cam_sim *sim);
@ -669,7 +669,7 @@ ahci_ch_attach(device_t dev)
}
if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
ctlr->direct ? ahci_ch_intr_direct : ahci_ch_intr,
dev, &ch->ih))) {
ch, &ch->ih))) {
device_printf(dev, "Unable to setup interrupt\n");
error = ENXIO;
goto err1;
@ -698,7 +698,7 @@ ahci_ch_attach(device_t dev)
}
/* Construct SIM entry */
ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch,
device_get_unit(dev), &ch->mtx,
device_get_unit(dev), (struct mtx *)&ch->mtx,
min(2, ch->numslots),
(ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0,
devq);
@ -722,7 +722,7 @@ ahci_ch_attach(device_t dev)
if (ch->pm_level > 3) {
callout_reset(&ch->pm_timer,
(ch->pm_level == 4) ? hz / 1000 : hz / 8,
ahci_ch_pm, dev);
ahci_ch_pm, ch);
}
mtx_unlock(&ch->mtx);
return (0);
@ -792,8 +792,8 @@ ahci_ch_init(device_t dev)
(AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD |
((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) |
((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 )));
ahci_start_fr(dev);
ahci_start(dev, 1);
ahci_start_fr(ch);
ahci_start(ch, 1);
return (0);
}
@ -805,8 +805,8 @@ ahci_ch_deinit(device_t dev)
/* Disable port interrupts. */
ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
/* Reset command register. */
ahci_stop(dev);
ahci_stop_fr(dev);
ahci_stop(ch);
ahci_stop_fr(ch);
ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0);
/* Allow everything, including partial and slumber modes. */
ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0);
@ -845,7 +845,7 @@ ahci_ch_resume(device_t dev)
mtx_lock(&ch->mtx);
ahci_ch_init(dev);
ahci_reset(dev);
ahci_reset(ch);
xpt_release_simq(ch->sim, TRUE);
mtx_unlock(&ch->mtx);
return (0);
@ -976,7 +976,7 @@ ahci_slotsalloc(device_t dev)
for (i = 0; i < ch->numslots; i++) {
struct ahci_slot *slot = &ch->slot[i];
slot->dev = dev;
slot->ch = ch;
slot->slot = i;
slot->state = AHCI_SLOT_EMPTY;
slot->ccb = NULL;
@ -1006,9 +1006,8 @@ ahci_slotsfree(device_t dev)
}
static int
ahci_phy_check_events(device_t dev, u_int32_t serr)
ahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr)
{
struct ahci_channel *ch = device_get_softc(dev);
if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
((ch->pm_level != 0 || ch->listening) && (serr & ATA_SE_EXCHANGED))) {
@ -1017,11 +1016,11 @@ ahci_phy_check_events(device_t dev, u_int32_t serr)
if (bootverbose) {
if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE)
device_printf(dev, "CONNECT requested\n");
device_printf(ch->dev, "CONNECT requested\n");
else
device_printf(dev, "DISCONNECT requested\n");
device_printf(ch->dev, "DISCONNECT requested\n");
}
ahci_reset(dev);
ahci_reset(ch);
if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
return (0);
if (xpt_create_path(&ccb->ccb_h.path, NULL,
@ -1037,11 +1036,11 @@ ahci_phy_check_events(device_t dev, u_int32_t serr)
}
static void
ahci_cpd_check_events(device_t dev)
ahci_cpd_check_events(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t status;
union ccb *ccb;
device_t dev;
if (ch->pm_level == 0)
return;
@ -1051,12 +1050,13 @@ ahci_cpd_check_events(device_t dev)
return;
if (bootverbose) {
dev = ch->dev;
if (status & AHCI_P_CMD_CPS) {
device_printf(dev, "COLD CONNECT requested\n");
} else
device_printf(dev, "COLD DISCONNECT requested\n");
}
ahci_reset(dev);
ahci_reset(ch);
if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
return;
if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(ch->sim),
@ -1068,16 +1068,15 @@ ahci_cpd_check_events(device_t dev)
}
static void
ahci_notify_events(device_t dev, u_int32_t status)
ahci_notify_events(struct ahci_channel *ch, u_int32_t status)
{
struct ahci_channel *ch = device_get_softc(dev);
struct cam_path *dpath;
int i;
if (ch->caps & AHCI_CAP_SSNTF)
ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status);
if (bootverbose)
device_printf(dev, "SNTF 0x%04x\n", status);
device_printf(ch->dev, "SNTF 0x%04x\n", status);
for (i = 0; i < 16; i++) {
if ((status & (1 << i)) == 0)
continue;
@ -1106,8 +1105,7 @@ ahci_done(struct ahci_channel *ch, union ccb *ccb)
static void
ahci_ch_intr(void *arg)
{
device_t dev = (device_t)arg;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = (struct ahci_channel *)arg;
uint32_t istatus;
/* Read interrupt statuses. */
@ -1123,8 +1121,7 @@ ahci_ch_intr(void *arg)
static void
ahci_ch_intr_direct(void *arg)
{
device_t dev = (device_t)arg;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = (struct ahci_channel *)arg;
struct ccb_hdr *ccb_h;
uint32_t istatus;
@ -1147,8 +1144,7 @@ ahci_ch_intr_direct(void *arg)
static void
ahci_ch_pm(void *arg)
{
device_t dev = (device_t)arg;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = (struct ahci_channel *)arg;
uint32_t work;
if (ch->numrslots != 0)
@ -1164,7 +1160,6 @@ ahci_ch_pm(void *arg)
static void
ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
{
device_t dev = ch->dev;
uint32_t cstatus, serr = 0, sntf = 0, ok, err;
enum ahci_err_type et;
int i, ccs, port, reset = 0;
@ -1206,12 +1201,12 @@ ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
serr = ATA_INL(ch->r_mem, AHCI_P_SERR);
if (serr) {
ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr);
reset = ahci_phy_check_events(dev, serr);
reset = ahci_phy_check_events(ch, serr);
}
}
/* Process cold presence detection events */
if ((istatus & AHCI_P_IX_CPD) && !reset)
ahci_cpd_check_events(dev);
ahci_cpd_check_events(ch);
/* Process command errors */
if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF |
AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) {
@ -1306,14 +1301,13 @@ ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
}
/* Process NOTIFY events */
if (sntf)
ahci_notify_events(dev, sntf);
ahci_notify_events(ch, sntf);
}
/* Must be called with channel locked. */
static int
ahci_check_collision(device_t dev, union ccb *ccb)
ahci_check_collision(struct ahci_channel *ch, union ccb *ccb)
{
struct ahci_channel *ch = device_get_softc(dev);
int t = ccb->ccb_h.target_id;
if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
@ -1362,9 +1356,8 @@ ahci_check_collision(device_t dev, union ccb *ccb)
/* Must be called with channel locked. */
static void
ahci_begin_transaction(device_t dev, union ccb *ccb)
ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb)
{
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_slot *slot;
int tag, tags;
@ -1373,14 +1366,14 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
(ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
tags = ch->curr[ccb->ccb_h.target_id].tags;
tag = ch->lastslot;
while (1) {
if (tag >= tags)
tag = 0;
if (ch->slot[tag].state == AHCI_SLOT_EMPTY)
break;
tag++;
};
if (ch->lastslot + 1 < tags)
tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
else
tag = 0;
if (tag == 0 || tag + ch->lastslot >= tags)
tag = ffs(~ch->oslots) - 1;
else
tag += ch->lastslot;
ch->lastslot = tag;
/* Occupy chosen slot. */
slot = &ch->slot[tag];
@ -1389,7 +1382,7 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
if (ch->numrslots == 0 && ch->pm_level > 3)
callout_stop(&ch->pm_timer);
/* Update channel stats. */
ch->oslots |= (1 << slot->slot);
ch->oslots |= (1 << tag);
ch->numrslots++;
ch->numrslotspd[ccb->ccb_h.target_id]++;
if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
@ -1400,14 +1393,15 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
}
if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
(ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
ch->aslots |= (1 << slot->slot);
slot->dma.nsegs = 0;
ch->aslots |= (1 << tag);
if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
slot->state = AHCI_SLOT_LOADING;
bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
ahci_dmasetprd, slot, 0);
} else
} else {
slot->dma.nsegs = 0;
ahci_execute_transaction(slot);
}
}
/* Locked by busdma engine. */
@ -1415,13 +1409,13 @@ static void
ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
{
struct ahci_slot *slot = arg;
struct ahci_channel *ch = device_get_softc(slot->dev);
struct ahci_channel *ch = slot->ch;
struct ahci_cmd_tab *ctp;
struct ahci_dma_prd *prd;
int i;
if (error) {
device_printf(slot->dev, "DMA load error\n");
device_printf(ch->dev, "DMA load error\n");
ahci_end_transaction(slot, AHCI_ERR_INVALID);
return;
}
@ -1446,8 +1440,7 @@ ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
static void
ahci_execute_transaction(struct ahci_slot *slot)
{
device_t dev = slot->dev;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = slot->ch;
struct ahci_cmd_tab *ctp;
struct ahci_cmd_list *clp;
union ccb *ccb = slot->ccb;
@ -1460,7 +1453,7 @@ ahci_execute_transaction(struct ahci_slot *slot)
ctp = (struct ahci_cmd_tab *)
(ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot));
/* Setup the FIS for this request */
if (!(fis_size = ahci_setup_fis(dev, ctp, ccb, slot->slot))) {
if (!(fis_size = ahci_setup_fis(ch, ctp, ccb, slot->slot))) {
device_printf(ch->dev, "Setting up SATA FIS failed\n");
ahci_end_transaction(slot, AHCI_ERR_INVALID);
return;
@ -1481,9 +1474,9 @@ ahci_execute_transaction(struct ahci_slot *slot)
if (ccb->ataio.cmd.control & ATA_A_RESET) {
softreset = 1;
/* Kick controller into sane state */
ahci_stop(dev);
ahci_clo(dev);
ahci_start(dev, 0);
ahci_stop(ch);
ahci_clo(ch);
ahci_start(ch, 0);
clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY;
} else {
softreset = 2;
@ -1561,9 +1554,9 @@ ahci_execute_transaction(struct ahci_slot *slot)
}
if (timeout && (count >= timeout)) {
device_printf(dev, "Poll timeout on slot %d port %d\n",
device_printf(ch->dev, "Poll timeout on slot %d port %d\n",
slot->slot, port);
device_printf(dev, "is %08x cs %08x ss %08x "
device_printf(ch->dev, "is %08x cs %08x ss %08x "
"rs %08x tfd %02x serr %08x cmd %08x\n",
ATA_INL(ch->r_mem, AHCI_P_IS),
ATA_INL(ch->r_mem, AHCI_P_CI),
@ -1588,9 +1581,8 @@ ahci_execute_transaction(struct ahci_slot *slot)
/* Must be called with channel locked. */
static void
ahci_process_timeout(device_t dev)
ahci_process_timeout(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
int i;
mtx_assert(&ch->mtx, MA_OWNED);
@ -1605,9 +1597,8 @@ ahci_process_timeout(device_t dev)
/* Must be called with channel locked. */
static void
ahci_rearm_timeout(device_t dev)
ahci_rearm_timeout(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
int i;
mtx_assert(&ch->mtx, MA_OWNED);
@ -1629,8 +1620,8 @@ ahci_rearm_timeout(device_t dev)
static void
ahci_timeout(struct ahci_slot *slot)
{
device_t dev = slot->dev;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = slot->ch;
device_t dev = ch->dev;
uint32_t sstatus;
int ccs;
int i;
@ -1697,7 +1688,7 @@ ahci_timeout(struct ahci_slot *slot)
xpt_freeze_simq(ch->sim, 1);
ch->toslots |= (1 << slot->slot);
if ((ch->rslots & ~ch->toslots) == 0)
ahci_process_timeout(dev);
ahci_process_timeout(ch);
else
device_printf(dev, " ... waiting for slots %08x\n",
ch->rslots & ~ch->toslots);
@ -1708,8 +1699,7 @@ ahci_timeout(struct ahci_slot *slot)
static void
ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
{
device_t dev = slot->dev;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = slot->ch;
union ccb *ccb = slot->ccb;
struct ahci_cmd_list *clp;
int lastto;
@ -1867,15 +1857,15 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
(ccb->ataio.cmd.control & ATA_A_RESET) &&
et == AHCI_ERR_NONE) {
ccb->ataio.cmd.control &= ~ATA_A_RESET;
ahci_begin_transaction(dev, ccb);
ahci_begin_transaction(ch, ccb);
return;
}
/* If it was our READ LOG command - process it. */
if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
ahci_process_read_log(dev, ccb);
ahci_process_read_log(ch, ccb);
/* If it was our REQUEST SENSE command - process it. */
} else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
ahci_process_request_sense(dev, ccb);
ahci_process_request_sense(ch, ccb);
/* If it was NCQ or ATAPI command error, put result on hold. */
} else if (et == AHCI_ERR_NCQ ||
((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
@ -1888,27 +1878,27 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
if (ch->rslots == 0) {
/* if there was fatal error - reset port. */
if (ch->toslots != 0 || ch->fatalerr) {
ahci_reset(dev);
ahci_reset(ch);
} else {
/* if we have slots in error, we can reinit port. */
if (ch->eslots != 0) {
ahci_stop(dev);
ahci_clo(dev);
ahci_start(dev, 1);
ahci_stop(ch);
ahci_clo(ch);
ahci_start(ch, 1);
}
/* if there commands on hold, we can do READ LOG. */
if (!ch->recoverycmd && ch->numhslots)
ahci_issue_recovery(dev);
ahci_issue_recovery(ch);
}
/* If all the rest of commands are in timeout - give them chance. */
} else if ((ch->rslots & ~ch->toslots) == 0 &&
et != AHCI_ERR_TIMEOUT)
ahci_rearm_timeout(dev);
ahci_rearm_timeout(ch);
/* Unfreeze frozen command. */
if (ch->frozen && !ahci_check_collision(dev, ch->frozen)) {
if (ch->frozen && !ahci_check_collision(ch, ch->frozen)) {
union ccb *fccb = ch->frozen;
ch->frozen = NULL;
ahci_begin_transaction(dev, fccb);
ahci_begin_transaction(ch, fccb);
xpt_release_simq(ch->sim, TRUE);
}
/* Start PM timer. */
@ -1920,9 +1910,8 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
}
static void
ahci_issue_recovery(device_t dev)
ahci_issue_recovery(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
union ccb *ccb;
struct ccb_ataio *ataio;
struct ccb_scsiio *csio;
@ -1935,7 +1924,7 @@ ahci_issue_recovery(device_t dev)
}
ccb = xpt_alloc_ccb_nowait();
if (ccb == NULL) {
device_printf(dev, "Unable to allocate recovery command\n");
device_printf(ch->dev, "Unable to allocate recovery command\n");
completeall:
/* We can't do anything -- complete held commands. */
for (i = 0; i < ch->numslots; i++) {
@ -1947,7 +1936,7 @@ ahci_issue_recovery(device_t dev)
ch->hold[i] = NULL;
ch->numhslots--;
}
ahci_reset(dev);
ahci_reset(ch);
return;
}
ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */
@ -1961,7 +1950,7 @@ ahci_issue_recovery(device_t dev)
ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT);
if (ataio->data_ptr == NULL) {
xpt_free_ccb(ccb);
device_printf(dev,
device_printf(ch->dev,
"Unable to allocate memory for READ LOG command\n");
goto completeall;
}
@ -1993,13 +1982,12 @@ ahci_issue_recovery(device_t dev)
/* Freeze SIM while doing recovery. */
ch->recoverycmd = 1;
xpt_freeze_simq(ch->sim, 1);
ahci_begin_transaction(dev, ccb);
ahci_begin_transaction(ch, ccb);
}
static void
ahci_process_read_log(device_t dev, union ccb *ccb)
ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb)
{
struct ahci_channel *ch = device_get_softc(dev);
uint8_t *data;
struct ata_res *res;
int i;
@ -2037,9 +2025,9 @@ ahci_process_read_log(device_t dev, union ccb *ccb)
}
} else {
if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
device_printf(dev, "Error while READ LOG EXT\n");
device_printf(ch->dev, "Error while READ LOG EXT\n");
else if ((data[0] & 0x80) == 0) {
device_printf(dev, "Non-queued command error in READ LOG EXT\n");
device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
}
for (i = 0; i < ch->numslots; i++) {
if (!ch->hold[i])
@ -2057,9 +2045,8 @@ ahci_process_read_log(device_t dev, union ccb *ccb)
}
static void
ahci_process_request_sense(device_t dev, union ccb *ccb)
ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb)
{
struct ahci_channel *ch = device_get_softc(dev);
int i;
ch->recoverycmd = 0;
@ -2079,9 +2066,8 @@ ahci_process_request_sense(device_t dev, union ccb *ccb)
}
static void
ahci_start(device_t dev, int fbs)
ahci_start(struct ahci_channel *ch, int fbs)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t cmd;
/* Clear SATA error register */
@ -2102,9 +2088,8 @@ ahci_start(device_t dev, int fbs)
}
static void
ahci_stop(device_t dev)
ahci_stop(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t cmd;
int timeout;
@ -2116,7 +2101,7 @@ ahci_stop(device_t dev)
do {
DELAY(10);
if (timeout++ > 50000) {
device_printf(dev, "stopping AHCI engine failed\n");
device_printf(ch->dev, "stopping AHCI engine failed\n");
break;
}
} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR);
@ -2124,9 +2109,8 @@ ahci_stop(device_t dev)
}
static void
ahci_clo(device_t dev)
ahci_clo(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t cmd;
int timeout;
@ -2139,7 +2123,7 @@ ahci_clo(device_t dev)
do {
DELAY(10);
if (timeout++ > 50000) {
device_printf(dev, "executing CLO failed\n");
device_printf(ch->dev, "executing CLO failed\n");
break;
}
} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO);
@ -2147,9 +2131,8 @@ ahci_clo(device_t dev)
}
static void
ahci_stop_fr(device_t dev)
ahci_stop_fr(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t cmd;
int timeout;
@ -2161,16 +2144,15 @@ ahci_stop_fr(device_t dev)
do {
DELAY(10);
if (timeout++ > 50000) {
device_printf(dev, "stopping AHCI FR engine failed\n");
device_printf(ch->dev, "stopping AHCI FR engine failed\n");
break;
}
} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR);
}
static void
ahci_start_fr(device_t dev)
ahci_start_fr(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int32_t cmd;
/* Start FIS reception on this channel */
@ -2179,9 +2161,8 @@ ahci_start_fr(device_t dev)
}
static int
ahci_wait_ready(device_t dev, int t, int t0)
ahci_wait_ready(struct ahci_channel *ch, int t, int t0)
{
struct ahci_channel *ch = device_get_softc(dev);
int timeout = 0;
uint32_t val;
@ -2189,7 +2170,7 @@ ahci_wait_ready(device_t dev, int t, int t0)
(ATA_S_BUSY | ATA_S_DRQ)) {
if (timeout > t) {
if (t != 0) {
device_printf(dev,
device_printf(ch->dev,
"AHCI reset: device not ready after %dms "
"(tfd = %08x)\n",
MAX(t, 0) + t0, val);
@ -2200,7 +2181,7 @@ ahci_wait_ready(device_t dev, int t, int t0)
timeout++;
}
if (bootverbose)
device_printf(dev, "AHCI reset: device ready after %dms\n",
device_printf(ch->dev, "AHCI reset: device ready after %dms\n",
timeout + t0);
return (0);
}
@ -2208,22 +2189,21 @@ ahci_wait_ready(device_t dev, int t, int t0)
static void
ahci_reset_to(void *arg)
{
device_t dev = arg;
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_channel *ch = arg;
if (ch->resetting == 0)
return;
ch->resetting--;
if (ahci_wait_ready(dev, ch->resetting == 0 ? -1 : 0,
if (ahci_wait_ready(ch, ch->resetting == 0 ? -1 : 0,
(310 - ch->resetting) * 100) == 0) {
ch->resetting = 0;
ahci_start(dev, 1);
ahci_start(ch, 1);
xpt_release_simq(ch->sim, TRUE);
return;
}
if (ch->resetting == 0) {
ahci_clo(dev);
ahci_start(dev, 1);
ahci_clo(ch);
ahci_start(ch, 1);
xpt_release_simq(ch->sim, TRUE);
return;
}
@ -2231,15 +2211,14 @@ ahci_reset_to(void *arg)
}
static void
ahci_reset(device_t dev)
ahci_reset(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev));
struct ahci_controller *ctlr = device_get_softc(device_get_parent(ch->dev));
int i;
xpt_freeze_simq(ch->sim, 1);
if (bootverbose)
device_printf(dev, "AHCI reset...\n");
device_printf(ch->dev, "AHCI reset...\n");
/* Forget about previous reset. */
if (ch->resetting) {
ch->resetting = 0;
@ -2258,7 +2237,7 @@ ahci_reset(device_t dev)
ahci_done(ch, fccb);
}
/* Kill the engine and requeue all running commands. */
ahci_stop(dev);
ahci_stop(ch);
for (i = 0; i < ch->numslots; i++) {
/* Do we have a running request on slot? */
if (ch->slot[i].state < AHCI_SLOT_RUNNING)
@ -2284,9 +2263,9 @@ ahci_reset(device_t dev)
/* Disable port interrupts */
ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
/* Reset and reconnect PHY, */
if (!ahci_sata_phy_reset(dev)) {
if (!ahci_sata_phy_reset(ch)) {
if (bootverbose)
device_printf(dev,
device_printf(ch->dev,
"AHCI reset: device not found\n");
ch->devices = 0;
/* Enable wanted port interrupts */
@ -2297,11 +2276,11 @@ ahci_reset(device_t dev)
return;
}
if (bootverbose)
device_printf(dev, "AHCI reset: device found\n");
device_printf(ch->dev, "AHCI reset: device found\n");
/* Wait for clearing busy status. */
if (ahci_wait_ready(dev, dumping ? 31000 : 0, 0)) {
if (ahci_wait_ready(ch, dumping ? 31000 : 0, 0)) {
if (dumping)
ahci_clo(dev);
ahci_clo(ch);
else
ch->resetting = 310;
}
@ -2315,17 +2294,16 @@ ahci_reset(device_t dev)
AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) |
AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR)));
if (ch->resetting)
callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, dev);
callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, ch);
else {
ahci_start(dev, 1);
ahci_start(ch, 1);
xpt_release_simq(ch->sim, TRUE);
}
}
static int
ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
{
struct ahci_channel *ch = device_get_softc(dev);
u_int8_t *fis = &ctp->cfis[0];
bzero(fis, 20);
@ -2417,9 +2395,8 @@ ahci_sata_connect(struct ahci_channel *ch)
}
static int
ahci_sata_phy_reset(device_t dev)
ahci_sata_phy_reset(struct ahci_channel *ch)
{
struct ahci_channel *ch = device_get_softc(dev);
int sata_rev;
uint32_t val;
@ -2459,9 +2436,8 @@ ahci_sata_phy_reset(device_t dev)
}
static int
ahci_check_ids(device_t dev, union ccb *ccb)
ahci_check_ids(struct ahci_channel *ch, union ccb *ccb)
{
struct ahci_channel *ch = device_get_softc(dev);
if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) {
ccb->ccb_h.status = CAM_TID_INVALID;
@ -2479,19 +2455,17 @@ ahci_check_ids(device_t dev, union ccb *ccb)
static void
ahciaction(struct cam_sim *sim, union ccb *ccb)
{
device_t dev, parent;
struct ahci_channel *ch;
CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n",
ccb->ccb_h.func_code));
ch = (struct ahci_channel *)cam_sim_softc(sim);
dev = ch->dev;
switch (ccb->ccb_h.func_code) {
/* Common cases first */
case XPT_ATA_IO: /* Execute the requested I/O operation */
case XPT_SCSI_IO:
if (ahci_check_ids(dev, ccb))
if (ahci_check_ids(ch, ccb))
return;
if (ch->devices == 0 ||
(ch->pm_present == 0 &&
@ -2501,14 +2475,14 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
}
ccb->ccb_h.recovery_type = RECOVERY_NONE;
/* Check for command collision. */
if (ahci_check_collision(dev, ccb)) {
if (ahci_check_collision(ch, ccb)) {
/* Freeze command. */
ch->frozen = ccb;
/* We have only one frozen slot, so freeze simq also. */
xpt_freeze_simq(ch->sim, 1);
return;
}
ahci_begin_transaction(dev, ccb);
ahci_begin_transaction(ch, ccb);
return;
case XPT_EN_LUN: /* Enable LUN as a target */
case XPT_TARGET_IO: /* Execute target I/O request */
@ -2523,7 +2497,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
struct ccb_trans_settings *cts = &ccb->cts;
struct ahci_device *d;
if (ahci_check_ids(dev, ccb))
if (ahci_check_ids(ch, ccb))
return;
if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
d = &ch->curr[ccb->ccb_h.target_id];
@ -2553,7 +2527,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
struct ahci_device *d;
uint32_t status;
if (ahci_check_ids(dev, ccb))
if (ahci_check_ids(ch, ccb))
return;
if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
d = &ch->curr[ccb->ccb_h.target_id];
@ -2610,7 +2584,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
}
case XPT_RESET_BUS: /* Reset the specified SCSI bus */
case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */
ahci_reset(dev);
ahci_reset(ch);
ccb->ccb_h.status = CAM_REQ_CMP;
break;
case XPT_TERM_IO: /* Terminate the I/O process */
@ -2621,7 +2595,6 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
{
struct ccb_pathinq *cpi = &ccb->cpi;
parent = device_get_parent(dev);
cpi->version_num = 1; /* XXX??? */
cpi->hba_inquiry = PI_SDTR_ABLE;
if (ch->caps & AHCI_CAP_SNCQ)
@ -2678,7 +2651,7 @@ ahcipoll(struct cam_sim *sim)
if (ch->resetting != 0 &&
(--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) {
ch->resetpolldiv = 1000;
ahci_reset_to(ch->dev);
ahci_reset_to(ch);
}
}
MODULE_VERSION(ahci, 1);

19
sys/dev/ahci/ahci.h
View File

@ -375,7 +375,7 @@ enum ahci_slot_states {
};
struct ahci_slot {
device_t dev; /* Device handle */
struct ahci_channel *ch; /* Channel */
u_int8_t slot; /* Number of this slot */
enum ahci_slot_states state; /* Slot state */
union ccb *ccb; /* CCB occupying slot */
@ -422,20 +422,19 @@ struct ahci_channel {
int quirks;
int numslots; /* Number of present slots */
int pm_level; /* power management level */
struct ahci_slot slot[AHCI_MAX_SLOTS];
union ccb *hold[AHCI_MAX_SLOTS];
struct mtx mtx; /* state lock */
STAILQ_HEAD(, ccb_hdr) doneq; /* queue of completed CCBs */
int batch; /* doneq is in use */
int devices; /* What is present */
int pm_present; /* PM presence reported */
int fbs_enabled; /* FIS-based switching enabled */
union ccb *hold[AHCI_MAX_SLOTS];
struct ahci_slot slot[AHCI_MAX_SLOTS];
uint32_t oslots; /* Occupied slots */
uint32_t rslots; /* Running slots */
uint32_t aslots; /* Slots with atomic commands */
uint32_t eslots; /* Slots in error */
uint32_t toslots; /* Slots in timeout */
int lastslot; /* Last used slot */
int taggedtarget; /* Last tagged target */
int numrslots; /* Number of running slots */
int numrslotspd[16];/* Number of running slots per dev */
int numtslots; /* Number of tagged slots */
@ -443,8 +442,6 @@ struct ahci_channel {
int numhslots; /* Number of held slots */
int recoverycmd; /* Our READ LOG active */
int fatalerr; /* Fatal error happend */
int lastslot; /* Last used slot */
int taggedtarget; /* Last tagged target */
int resetting; /* Hard-reset in progress. */
int resetpolldiv; /* Hard-reset poll divider. */
int listening; /* SUD bit is cleared. */
@ -455,6 +452,10 @@ struct ahci_channel {
struct ahci_device user[16]; /* User-specified settings */
struct ahci_device curr[16]; /* Current settings */
struct mtx_padalign mtx; /* state lock */
STAILQ_HEAD(, ccb_hdr) doneq; /* queue of completed CCBs */
int batch; /* doneq is in use */
};
struct ahci_enclosure {

4
sys/dev/ahci/ahciem.c
View File

@ -344,7 +344,7 @@ ahci_em_led(void *priv, int onoff)
}
static int
ahci_check_ids(device_t dev, union ccb *ccb)
ahci_check_ids(union ccb *ccb)
{
if (ccb->ccb_h.target_id != 0) {
@ -554,7 +554,7 @@ ahciemaction(struct cam_sim *sim, union ccb *ccb)
dev = enc->dev;
switch (ccb->ccb_h.func_code) {
case XPT_ATA_IO: /* Execute the requested I/O operation */
if (ahci_check_ids(dev, ccb))
if (ahci_check_ids(ccb))
return;
ahci_em_begin_transaction(dev, ccb);
return;