Fix batched unload for DMAR busdma in qi mode.
Do not queue dmar_map_entries with zeroed gseq to dmar_qi_invalidate_locked(). Zero gseq stops the processing in the qi task. Do not assign possibly uninitialized on-stack gseq to map entries when requeuing them on unit tlb_flush queue. Random garbage in gsec is interpreted as too high invalidation sequence number and again stop the processing in the task. Make the sequence numbers generation completely contained in dmar_qi_invalidate_locked() and dmar_qi_emit_wait_seq(). Upper code directly passes boolean requesting emiting wait command instead of trying to provide hint to avoid it by passing NULL gseq pointer. Microoptimize the requeueing to tlb_flush queue by doing it for the whole queue. Diagnosed and tested by: Brett Gutstein <bgutstein@rice.edu> Discussed with: alc Sponsored by: The FreeBSD Foundation MFC after: 1 week
This commit is contained in:
parent
704cb42f2a
commit
cf619a92d2
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=320125
@ -703,7 +703,7 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free)
|
|||||||
if (unit->qi_enabled) {
|
if (unit->qi_enabled) {
|
||||||
DMAR_LOCK(unit);
|
DMAR_LOCK(unit);
|
||||||
dmar_qi_invalidate_locked(entry->domain, entry->start,
|
dmar_qi_invalidate_locked(entry->domain, entry->start,
|
||||||
entry->end - entry->start, &entry->gseq);
|
entry->end - entry->start, &entry->gseq, true);
|
||||||
if (!free)
|
if (!free)
|
||||||
entry->flags |= DMAR_MAP_ENTRY_QI_NF;
|
entry->flags |= DMAR_MAP_ENTRY_QI_NF;
|
||||||
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
|
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
|
||||||
@ -715,16 +715,14 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct dmar_qi_genseq *
|
static bool
|
||||||
dmar_domain_unload_gseq(struct dmar_domain *domain,
|
dmar_domain_unload_emit_wait(struct dmar_domain *domain,
|
||||||
struct dmar_map_entry *entry, struct dmar_qi_genseq *gseq)
|
struct dmar_map_entry *entry)
|
||||||
{
|
{
|
||||||
|
|
||||||
if (TAILQ_NEXT(entry, dmamap_link) != NULL)
|
if (TAILQ_NEXT(entry, dmamap_link) == NULL)
|
||||||
return (NULL);
|
return (true);
|
||||||
if (domain->batch_no++ % dmar_batch_coalesce != 0)
|
return (domain->batch_no++ % dmar_batch_coalesce == 0);
|
||||||
return (NULL);
|
|
||||||
return (gseq);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
@ -733,7 +731,6 @@ dmar_domain_unload(struct dmar_domain *domain,
|
|||||||
{
|
{
|
||||||
struct dmar_unit *unit;
|
struct dmar_unit *unit;
|
||||||
struct dmar_map_entry *entry, *entry1;
|
struct dmar_map_entry *entry, *entry1;
|
||||||
struct dmar_qi_genseq gseq;
|
|
||||||
int error;
|
int error;
|
||||||
|
|
||||||
unit = domain->dmar;
|
unit = domain->dmar;
|
||||||
@ -757,17 +754,11 @@ dmar_domain_unload(struct dmar_domain *domain,
|
|||||||
KASSERT(unit->qi_enabled, ("loaded entry left"));
|
KASSERT(unit->qi_enabled, ("loaded entry left"));
|
||||||
DMAR_LOCK(unit);
|
DMAR_LOCK(unit);
|
||||||
TAILQ_FOREACH(entry, entries, dmamap_link) {
|
TAILQ_FOREACH(entry, entries, dmamap_link) {
|
||||||
entry->gseq.gen = 0;
|
|
||||||
entry->gseq.seq = 0;
|
|
||||||
dmar_qi_invalidate_locked(domain, entry->start, entry->end -
|
dmar_qi_invalidate_locked(domain, entry->start, entry->end -
|
||||||
entry->start, dmar_domain_unload_gseq(domain, entry,
|
entry->start, &entry->gseq,
|
||||||
&gseq));
|
dmar_domain_unload_emit_wait(domain, entry));
|
||||||
}
|
|
||||||
TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
|
|
||||||
entry->gseq = gseq;
|
|
||||||
TAILQ_REMOVE(entries, entry, dmamap_link);
|
|
||||||
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
|
|
||||||
}
|
}
|
||||||
|
TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
|
||||||
DMAR_UNLOCK(unit);
|
DMAR_UNLOCK(unit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -305,7 +305,7 @@ void dmar_disable_qi_intr(struct dmar_unit *unit);
|
|||||||
int dmar_init_qi(struct dmar_unit *unit);
|
int dmar_init_qi(struct dmar_unit *unit);
|
||||||
void dmar_fini_qi(struct dmar_unit *unit);
|
void dmar_fini_qi(struct dmar_unit *unit);
|
||||||
void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start,
|
void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start,
|
||||||
dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
|
dmar_gaddr_t size, struct dmar_qi_genseq *psec, bool emit_wait);
|
||||||
void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
|
void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
|
||||||
void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
|
void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
|
||||||
void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
|
void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
|
||||||
|
@ -171,7 +171,8 @@ dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
|
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
|
||||||
|
bool emit_wait)
|
||||||
{
|
{
|
||||||
struct dmar_qi_genseq gsec;
|
struct dmar_qi_genseq gsec;
|
||||||
uint32_t seq;
|
uint32_t seq;
|
||||||
@ -192,7 +193,10 @@ dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
|
|||||||
seq = unit->inv_waitd_seq++;
|
seq = unit->inv_waitd_seq++;
|
||||||
pseq->gen = unit->inv_waitd_gen;
|
pseq->gen = unit->inv_waitd_gen;
|
||||||
pseq->seq = seq;
|
pseq->seq = seq;
|
||||||
dmar_qi_emit_wait_descr(unit, seq, true, true, false);
|
if (emit_wait) {
|
||||||
|
dmar_qi_ensure(unit, 1);
|
||||||
|
dmar_qi_emit_wait_descr(unit, seq, true, true, false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
@ -215,7 +219,7 @@ dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
|
|||||||
|
|
||||||
void
|
void
|
||||||
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
|
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
|
||||||
dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
|
dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)
|
||||||
{
|
{
|
||||||
struct dmar_unit *unit;
|
struct dmar_unit *unit;
|
||||||
dmar_gaddr_t isize;
|
dmar_gaddr_t isize;
|
||||||
@ -232,10 +236,7 @@ dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
|
|||||||
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
|
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
|
||||||
base | am);
|
base | am);
|
||||||
}
|
}
|
||||||
if (pseq != NULL) {
|
dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
|
||||||
dmar_qi_ensure(unit, 1);
|
|
||||||
dmar_qi_emit_wait_seq(unit, pseq);
|
|
||||||
}
|
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -247,7 +248,7 @@ dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
|
|||||||
DMAR_ASSERT_LOCKED(unit);
|
DMAR_ASSERT_LOCKED(unit);
|
||||||
dmar_qi_ensure(unit, 2);
|
dmar_qi_ensure(unit, 2);
|
||||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
|
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
|
||||||
dmar_qi_emit_wait_seq(unit, &gseq);
|
dmar_qi_emit_wait_seq(unit, &gseq, true);
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
dmar_qi_wait_for_seq(unit, &gseq, false);
|
dmar_qi_wait_for_seq(unit, &gseq, false);
|
||||||
}
|
}
|
||||||
@ -261,7 +262,7 @@ dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
|
|||||||
dmar_qi_ensure(unit, 2);
|
dmar_qi_ensure(unit, 2);
|
||||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
|
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
|
||||||
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
|
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
|
||||||
dmar_qi_emit_wait_seq(unit, &gseq);
|
dmar_qi_emit_wait_seq(unit, &gseq, true);
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
dmar_qi_wait_for_seq(unit, &gseq, false);
|
dmar_qi_wait_for_seq(unit, &gseq, false);
|
||||||
}
|
}
|
||||||
@ -274,7 +275,7 @@ dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
|
|||||||
DMAR_ASSERT_LOCKED(unit);
|
DMAR_ASSERT_LOCKED(unit);
|
||||||
dmar_qi_ensure(unit, 2);
|
dmar_qi_ensure(unit, 2);
|
||||||
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
|
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
|
||||||
dmar_qi_emit_wait_seq(unit, &gseq);
|
dmar_qi_emit_wait_seq(unit, &gseq, true);
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
dmar_qi_wait_for_seq(unit, &gseq, false);
|
dmar_qi_wait_for_seq(unit, &gseq, false);
|
||||||
}
|
}
|
||||||
@ -298,7 +299,7 @@ dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
|
|||||||
DMAR_IQ_DESCR_IEC_IM(l), 0);
|
DMAR_IQ_DESCR_IEC_IM(l), 0);
|
||||||
}
|
}
|
||||||
dmar_qi_ensure(unit, 1);
|
dmar_qi_ensure(unit, 1);
|
||||||
dmar_qi_emit_wait_seq(unit, &gseq);
|
dmar_qi_emit_wait_seq(unit, &gseq, true);
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -344,8 +345,7 @@ dmar_qi_task(void *arg, int pending __unused)
|
|||||||
entry = TAILQ_FIRST(&unit->tlb_flush_entries);
|
entry = TAILQ_FIRST(&unit->tlb_flush_entries);
|
||||||
if (entry == NULL)
|
if (entry == NULL)
|
||||||
break;
|
break;
|
||||||
if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
|
if (!dmar_qi_seq_processed(unit, &entry->gseq))
|
||||||
!dmar_qi_seq_processed(unit, &entry->gseq))
|
|
||||||
break;
|
break;
|
||||||
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
|
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
|
||||||
DMAR_UNLOCK(unit);
|
DMAR_UNLOCK(unit);
|
||||||
@ -432,7 +432,7 @@ dmar_fini_qi(struct dmar_unit *unit)
|
|||||||
DMAR_LOCK(unit);
|
DMAR_LOCK(unit);
|
||||||
/* quisce */
|
/* quisce */
|
||||||
dmar_qi_ensure(unit, 1);
|
dmar_qi_ensure(unit, 1);
|
||||||
dmar_qi_emit_wait_seq(unit, &gseq);
|
dmar_qi_emit_wait_seq(unit, &gseq, true);
|
||||||
dmar_qi_advance_tail(unit);
|
dmar_qi_advance_tail(unit);
|
||||||
dmar_qi_wait_for_seq(unit, &gseq, false);
|
dmar_qi_wait_for_seq(unit, &gseq, false);
|
||||||
/* only after the quisce, disable queue */
|
/* only after the quisce, disable queue */
|
||||||
|
Loading…
Reference in New Issue
Block a user