d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix batched unload for DMAR busdma in qi mode.

Browse Source 
Do not queue dmar_map_entries with zeroed gseq to
dmar_qi_invalidate_locked().  Zero gseq stops the processing in the qi
task.  Do not assign possibly uninitialized on-stack gseq to map
entries when requeuing them on unit tlb_flush queue.  Random garbage
in gsec is interpreted as too high invalidation sequence number and
again stop the processing in the task.

Make the sequence numbers generation completely contained in
dmar_qi_invalidate_locked() and dmar_qi_emit_wait_seq().  Upper code
directly passes boolean requesting emiting wait command instead of
trying to provide hint to avoid it by passing NULL gseq pointer.

Microoptimize the requeueing to tlb_flush queue by doing it for the
whole queue.

Diagnosed and tested by:	Brett Gutstein <bgutstein@rice.edu>
Discussed with:	alc
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
This commit is contained in:
Konstantin Belousov 2017-06-19 21:48:52 +00:00
parent 704cb42f2a
commit cf619a92d2
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=320125
3 changed files with 25 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
sys/x86/iommu/intel_ctx.c
View File

@ -703,7 +703,7 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free)
if (unit->qi_enabled) {
DMAR_LOCK(unit);
dmar_qi_invalidate_locked(entry->domain, entry->start,
entry->end - entry->start, &entry->gseq);
entry->end - entry->start, &entry->gseq, true);
if (!free)
entry->flags |= DMAR_MAP_ENTRY_QI_NF;
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
@ -715,16 +715,14 @@ dmar_domain_unload_entry(struct dmar_map_entry *entry, bool free)
}
}
static struct dmar_qi_genseq *
dmar_domain_unload_gseq(struct dmar_domain *domain,
struct dmar_map_entry *entry, struct dmar_qi_genseq *gseq)
static bool
dmar_domain_unload_emit_wait(struct dmar_domain *domain,
struct dmar_map_entry *entry)
{
if (TAILQ_NEXT(entry, dmamap_link) != NULL)
return (NULL);
if (domain->batch_no++ % dmar_batch_coalesce != 0)
return (NULL);
return (gseq);
if (TAILQ_NEXT(entry, dmamap_link) == NULL)
return (true);
return (domain->batch_no++ % dmar_batch_coalesce == 0);
}
void
@ -733,7 +731,6 @@ dmar_domain_unload(struct dmar_domain *domain,
{
struct dmar_unit *unit;
struct dmar_map_entry *entry, *entry1;
struct dmar_qi_genseq gseq;
int error;
unit = domain->dmar;
@ -757,17 +754,11 @@ dmar_domain_unload(struct dmar_domain *domain,
KASSERT(unit->qi_enabled, ("loaded entry left"));
DMAR_LOCK(unit);
TAILQ_FOREACH(entry, entries, dmamap_link) {
entry->gseq.gen = 0;
entry->gseq.seq = 0;
dmar_qi_invalidate_locked(domain, entry->start, entry->end -
entry->start, dmar_domain_unload_gseq(domain, entry,
&gseq));
}
TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
entry->gseq = gseq;
TAILQ_REMOVE(entries, entry, dmamap_link);
TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
entry->start, &entry->gseq,
dmar_domain_unload_emit_wait(domain, entry));
}
TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link);
DMAR_UNLOCK(unit);
}

2
sys/x86/iommu/intel_dmar.h
View File

@ -305,7 +305,7 @@ void dmar_disable_qi_intr(struct dmar_unit *unit);
int dmar_init_qi(struct dmar_unit *unit);
void dmar_fini_qi(struct dmar_unit *unit);
void dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t start,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
dmar_gaddr_t size, struct dmar_qi_genseq *psec, bool emit_wait);
void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);

28
sys/x86/iommu/intel_qi.c
View File

@ -171,7 +171,8 @@ dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
}
static void
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq,
bool emit_wait)
{
struct dmar_qi_genseq gsec;
uint32_t seq;
@ -192,7 +193,10 @@ dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
seq = unit->inv_waitd_seq++;
pseq->gen = unit->inv_waitd_gen;
pseq->seq = seq;
dmar_qi_emit_wait_descr(unit, seq, true, true, false);
if (emit_wait) {
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_descr(unit, seq, true, true, false);
}
}
static void
@ -215,7 +219,7 @@ dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
void
dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait)
{
struct dmar_unit *unit;
dmar_gaddr_t isize;
@ -232,10 +236,7 @@ dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
base | am);
}
if (pseq != NULL) {
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, pseq);
}
dmar_qi_emit_wait_seq(unit, pseq, emit_wait);
dmar_qi_advance_tail(unit);
}
@ -247,7 +248,7 @@ dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@ -261,7 +262,7 @@ dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@ -274,7 +275,7 @@ dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
DMAR_ASSERT_LOCKED(unit);
dmar_qi_ensure(unit, 2);
dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
}
@ -298,7 +299,7 @@ dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
DMAR_IQ_DESCR_IEC_IM(l), 0);
}
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
/*
@ -344,8 +345,7 @@ dmar_qi_task(void *arg, int pending __unused)
entry = TAILQ_FIRST(&unit->tlb_flush_entries);
if (entry == NULL)
break;
if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
!dmar_qi_seq_processed(unit, &entry->gseq))
if (!dmar_qi_seq_processed(unit, &entry->gseq))
break;
TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
DMAR_UNLOCK(unit);
@ -432,7 +432,7 @@ dmar_fini_qi(struct dmar_unit *unit)
DMAR_LOCK(unit);
/* quisce */
dmar_qi_ensure(unit, 1);
dmar_qi_emit_wait_seq(unit, &gseq);
dmar_qi_emit_wait_seq(unit, &gseq, true);
dmar_qi_advance_tail(unit);
dmar_qi_wait_for_seq(unit, &gseq, false);
/* only after the quisce, disable queue */