82896fc74f
- Many drivers on amd64 are picking system uptime, interrupt time and ticks via global data structure instead of calling functions for performance reasons. For now just patch such address so driver will not trigger page fault when trying to access such data. In future, additional callout may be added to update data in periodic intervals. - On amd64 we need to allocate "shadow space" on stack before calling any function. Submitted by: Paul B Mahol <onemda at gmail.com>
4318 lines
99 KiB
C
4318 lines
99 KiB
C
/*-
|
|
* Copyright (c) 2003
|
|
* Bill Paul <wpaul@windriver.com>. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. All advertising materials mentioning features or use of this software
|
|
* must display the following acknowledgement:
|
|
* This product includes software developed by Bill Paul.
|
|
* 4. Neither the name of the author nor the names of any co-contributors
|
|
* may be used to endorse or promote products derived from this software
|
|
* without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
|
|
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include <sys/ctype.h>
|
|
#include <sys/unistd.h>
|
|
#include <sys/param.h>
|
|
#include <sys/types.h>
|
|
#include <sys/errno.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/malloc.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
|
|
#include <sys/callout.h>
|
|
#include <sys/kdb.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/condvar.h>
|
|
#include <sys/kthread.h>
|
|
#include <sys/module.h>
|
|
#include <sys/smp.h>
|
|
#include <sys/sched.h>
|
|
#include <sys/sysctl.h>
|
|
|
|
#include <machine/atomic.h>
|
|
#include <machine/bus.h>
|
|
#include <machine/stdarg.h>
|
|
#include <machine/resource.h>
|
|
|
|
#include <sys/bus.h>
|
|
#include <sys/rman.h>
|
|
|
|
#include <vm/vm.h>
|
|
#include <vm/vm_param.h>
|
|
#include <vm/pmap.h>
|
|
#include <vm/uma.h>
|
|
#include <vm/vm_kern.h>
|
|
#include <vm/vm_map.h>
|
|
#include <vm/vm_extern.h>
|
|
|
|
#include <compat/ndis/pe_var.h>
|
|
#include <compat/ndis/cfg_var.h>
|
|
#include <compat/ndis/resource_var.h>
|
|
#include <compat/ndis/ntoskrnl_var.h>
|
|
#include <compat/ndis/hal_var.h>
|
|
#include <compat/ndis/ndis_var.h>
|
|
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
static int sysctl_show_timers(SYSCTL_HANDLER_ARGS);
|
|
|
|
SYSCTL_PROC(_debug, OID_AUTO, ntoskrnl_timers, CTLFLAG_RW, 0, 0,
|
|
sysctl_show_timers, "I", "Show ntoskrnl timer stats");
|
|
#endif
|
|
|
|
struct kdpc_queue {
|
|
list_entry kq_disp;
|
|
struct thread *kq_td;
|
|
int kq_cpu;
|
|
int kq_exit;
|
|
int kq_running;
|
|
kspin_lock kq_lock;
|
|
nt_kevent kq_proc;
|
|
nt_kevent kq_done;
|
|
};
|
|
|
|
typedef struct kdpc_queue kdpc_queue;
|
|
|
|
struct wb_ext {
|
|
struct cv we_cv;
|
|
struct thread *we_td;
|
|
};
|
|
|
|
typedef struct wb_ext wb_ext;
|
|
|
|
#define NTOSKRNL_TIMEOUTS 256
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
static uint64_t ntoskrnl_timer_fires;
|
|
static uint64_t ntoskrnl_timer_sets;
|
|
static uint64_t ntoskrnl_timer_reloads;
|
|
static uint64_t ntoskrnl_timer_cancels;
|
|
#endif
|
|
|
|
struct callout_entry {
|
|
struct callout ce_callout;
|
|
list_entry ce_list;
|
|
};
|
|
|
|
typedef struct callout_entry callout_entry;
|
|
|
|
static struct list_entry ntoskrnl_calllist;
|
|
static struct mtx ntoskrnl_calllock;
|
|
struct kuser_shared_data kuser_shared_data;
|
|
|
|
static struct list_entry ntoskrnl_intlist;
|
|
static kspin_lock ntoskrnl_intlock;
|
|
|
|
static uint8_t RtlEqualUnicodeString(unicode_string *,
|
|
unicode_string *, uint8_t);
|
|
static void RtlCopyUnicodeString(unicode_string *,
|
|
unicode_string *);
|
|
static irp *IoBuildSynchronousFsdRequest(uint32_t, device_object *,
|
|
void *, uint32_t, uint64_t *, nt_kevent *, io_status_block *);
|
|
static irp *IoBuildAsynchronousFsdRequest(uint32_t,
|
|
device_object *, void *, uint32_t, uint64_t *, io_status_block *);
|
|
static irp *IoBuildDeviceIoControlRequest(uint32_t,
|
|
device_object *, void *, uint32_t, void *, uint32_t,
|
|
uint8_t, nt_kevent *, io_status_block *);
|
|
static irp *IoAllocateIrp(uint8_t, uint8_t);
|
|
static void IoReuseIrp(irp *, uint32_t);
|
|
static void IoFreeIrp(irp *);
|
|
static void IoInitializeIrp(irp *, uint16_t, uint8_t);
|
|
static irp *IoMakeAssociatedIrp(irp *, uint8_t);
|
|
static uint32_t KeWaitForMultipleObjects(uint32_t,
|
|
nt_dispatch_header **, uint32_t, uint32_t, uint32_t, uint8_t,
|
|
int64_t *, wait_block *);
|
|
static void ntoskrnl_waittest(nt_dispatch_header *, uint32_t);
|
|
static void ntoskrnl_satisfy_wait(nt_dispatch_header *, struct thread *);
|
|
static void ntoskrnl_satisfy_multiple_waits(wait_block *);
|
|
static int ntoskrnl_is_signalled(nt_dispatch_header *, struct thread *);
|
|
static void ntoskrnl_insert_timer(ktimer *, int);
|
|
static void ntoskrnl_remove_timer(ktimer *);
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
static void ntoskrnl_show_timers(void);
|
|
#endif
|
|
static void ntoskrnl_timercall(void *);
|
|
static void ntoskrnl_dpc_thread(void *);
|
|
static void ntoskrnl_destroy_dpc_threads(void);
|
|
static void ntoskrnl_destroy_workitem_threads(void);
|
|
static void ntoskrnl_workitem_thread(void *);
|
|
static void ntoskrnl_workitem(device_object *, void *);
|
|
static void ntoskrnl_unicode_to_ascii(uint16_t *, char *, int);
|
|
static void ntoskrnl_ascii_to_unicode(char *, uint16_t *, int);
|
|
static uint8_t ntoskrnl_insert_dpc(list_entry *, kdpc *);
|
|
static void WRITE_REGISTER_USHORT(uint16_t *, uint16_t);
|
|
static uint16_t READ_REGISTER_USHORT(uint16_t *);
|
|
static void WRITE_REGISTER_ULONG(uint32_t *, uint32_t);
|
|
static uint32_t READ_REGISTER_ULONG(uint32_t *);
|
|
static void WRITE_REGISTER_UCHAR(uint8_t *, uint8_t);
|
|
static uint8_t READ_REGISTER_UCHAR(uint8_t *);
|
|
static int64_t _allmul(int64_t, int64_t);
|
|
static int64_t _alldiv(int64_t, int64_t);
|
|
static int64_t _allrem(int64_t, int64_t);
|
|
static int64_t _allshr(int64_t, uint8_t);
|
|
static int64_t _allshl(int64_t, uint8_t);
|
|
static uint64_t _aullmul(uint64_t, uint64_t);
|
|
static uint64_t _aulldiv(uint64_t, uint64_t);
|
|
static uint64_t _aullrem(uint64_t, uint64_t);
|
|
static uint64_t _aullshr(uint64_t, uint8_t);
|
|
static uint64_t _aullshl(uint64_t, uint8_t);
|
|
static slist_entry *ntoskrnl_pushsl(slist_header *, slist_entry *);
|
|
static slist_entry *ntoskrnl_popsl(slist_header *);
|
|
static void ExInitializePagedLookasideList(paged_lookaside_list *,
|
|
lookaside_alloc_func *, lookaside_free_func *,
|
|
uint32_t, size_t, uint32_t, uint16_t);
|
|
static void ExDeletePagedLookasideList(paged_lookaside_list *);
|
|
static void ExInitializeNPagedLookasideList(npaged_lookaside_list *,
|
|
lookaside_alloc_func *, lookaside_free_func *,
|
|
uint32_t, size_t, uint32_t, uint16_t);
|
|
static void ExDeleteNPagedLookasideList(npaged_lookaside_list *);
|
|
static slist_entry
|
|
*ExInterlockedPushEntrySList(slist_header *,
|
|
slist_entry *, kspin_lock *);
|
|
static slist_entry
|
|
*ExInterlockedPopEntrySList(slist_header *, kspin_lock *);
|
|
static uint32_t InterlockedIncrement(volatile uint32_t *);
|
|
static uint32_t InterlockedDecrement(volatile uint32_t *);
|
|
static void ExInterlockedAddLargeStatistic(uint64_t *, uint32_t);
|
|
static void *MmAllocateContiguousMemory(uint32_t, uint64_t);
|
|
static void *MmAllocateContiguousMemorySpecifyCache(uint32_t,
|
|
uint64_t, uint64_t, uint64_t, enum nt_caching_type);
|
|
static void MmFreeContiguousMemory(void *);
|
|
static void MmFreeContiguousMemorySpecifyCache(void *, uint32_t,
|
|
enum nt_caching_type);
|
|
static uint32_t MmSizeOfMdl(void *, size_t);
|
|
static void *MmMapLockedPages(mdl *, uint8_t);
|
|
static void *MmMapLockedPagesSpecifyCache(mdl *,
|
|
uint8_t, uint32_t, void *, uint32_t, uint32_t);
|
|
static void MmUnmapLockedPages(void *, mdl *);
|
|
static device_t ntoskrnl_finddev(device_t, uint64_t, struct resource **);
|
|
static void RtlZeroMemory(void *, size_t);
|
|
static void RtlCopyMemory(void *, const void *, size_t);
|
|
static size_t RtlCompareMemory(const void *, const void *, size_t);
|
|
static ndis_status RtlUnicodeStringToInteger(unicode_string *,
|
|
uint32_t, uint32_t *);
|
|
static int atoi (const char *);
|
|
static long atol (const char *);
|
|
static int rand(void);
|
|
static void srand(unsigned int);
|
|
static void KeQuerySystemTime(uint64_t *);
|
|
static uint32_t KeTickCount(void);
|
|
static uint8_t IoIsWdmVersionAvailable(uint8_t, uint8_t);
|
|
static void ntoskrnl_thrfunc(void *);
|
|
static ndis_status PsCreateSystemThread(ndis_handle *,
|
|
uint32_t, void *, ndis_handle, void *, void *, void *);
|
|
static ndis_status PsTerminateSystemThread(ndis_status);
|
|
static ndis_status IoGetDeviceObjectPointer(unicode_string *,
|
|
uint32_t, void *, device_object *);
|
|
static ndis_status IoGetDeviceProperty(device_object *, uint32_t,
|
|
uint32_t, void *, uint32_t *);
|
|
static void KeInitializeMutex(kmutant *, uint32_t);
|
|
static uint32_t KeReleaseMutex(kmutant *, uint8_t);
|
|
static uint32_t KeReadStateMutex(kmutant *);
|
|
static ndis_status ObReferenceObjectByHandle(ndis_handle,
|
|
uint32_t, void *, uint8_t, void **, void **);
|
|
static void ObfDereferenceObject(void *);
|
|
static uint32_t ZwClose(ndis_handle);
|
|
static uint32_t WmiQueryTraceInformation(uint32_t, void *, uint32_t,
|
|
uint32_t, void *);
|
|
static uint32_t WmiTraceMessage(uint64_t, uint32_t, void *, uint16_t, ...);
|
|
static uint32_t IoWMIRegistrationControl(device_object *, uint32_t);
|
|
static void *ntoskrnl_memset(void *, int, size_t);
|
|
static void *ntoskrnl_memmove(void *, void *, size_t);
|
|
static void *ntoskrnl_memchr(void *, unsigned char, size_t);
|
|
static char *ntoskrnl_strstr(char *, char *);
|
|
static char *ntoskrnl_strncat(char *, char *, size_t);
|
|
static int ntoskrnl_toupper(int);
|
|
static int ntoskrnl_tolower(int);
|
|
static funcptr ntoskrnl_findwrap(funcptr);
|
|
static uint32_t DbgPrint(char *, ...);
|
|
static void DbgBreakPoint(void);
|
|
static void KeBugCheckEx(uint32_t, u_long, u_long, u_long, u_long);
|
|
static int32_t KeDelayExecutionThread(uint8_t, uint8_t, int64_t *);
|
|
static int32_t KeSetPriorityThread(struct thread *, int32_t);
|
|
static void dummy(void);
|
|
|
|
static struct mtx ntoskrnl_dispatchlock;
|
|
static struct mtx ntoskrnl_interlock;
|
|
static kspin_lock ntoskrnl_cancellock;
|
|
static int ntoskrnl_kth = 0;
|
|
static struct nt_objref_head ntoskrnl_reflist;
|
|
static uma_zone_t mdl_zone;
|
|
static uma_zone_t iw_zone;
|
|
static struct kdpc_queue *kq_queues;
|
|
static struct kdpc_queue *wq_queues;
|
|
static int wq_idx = 0;
|
|
|
|
int
|
|
ntoskrnl_libinit()
|
|
{
|
|
image_patch_table *patch;
|
|
int error;
|
|
struct proc *p;
|
|
kdpc_queue *kq;
|
|
callout_entry *e;
|
|
int i;
|
|
|
|
mtx_init(&ntoskrnl_dispatchlock,
|
|
"ntoskrnl dispatch lock", MTX_NDIS_LOCK, MTX_DEF|MTX_RECURSE);
|
|
mtx_init(&ntoskrnl_interlock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
|
|
KeInitializeSpinLock(&ntoskrnl_cancellock);
|
|
KeInitializeSpinLock(&ntoskrnl_intlock);
|
|
TAILQ_INIT(&ntoskrnl_reflist);
|
|
|
|
InitializeListHead(&ntoskrnl_calllist);
|
|
InitializeListHead(&ntoskrnl_intlist);
|
|
mtx_init(&ntoskrnl_calllock, MTX_NTOSKRNL_SPIN_LOCK, NULL, MTX_SPIN);
|
|
|
|
kq_queues = ExAllocatePoolWithTag(NonPagedPool,
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
sizeof(kdpc_queue) * mp_ncpus, 0);
|
|
#else
|
|
sizeof(kdpc_queue), 0);
|
|
#endif
|
|
|
|
if (kq_queues == NULL)
|
|
return (ENOMEM);
|
|
|
|
wq_queues = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(kdpc_queue) * WORKITEM_THREADS, 0);
|
|
|
|
if (wq_queues == NULL)
|
|
return (ENOMEM);
|
|
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
bzero((char *)kq_queues, sizeof(kdpc_queue) * mp_ncpus);
|
|
#else
|
|
bzero((char *)kq_queues, sizeof(kdpc_queue));
|
|
#endif
|
|
bzero((char *)wq_queues, sizeof(kdpc_queue) * WORKITEM_THREADS);
|
|
|
|
/*
|
|
* Launch the DPC threads.
|
|
*/
|
|
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
for (i = 0; i < mp_ncpus; i++) {
|
|
#else
|
|
for (i = 0; i < 1; i++) {
|
|
#endif
|
|
kq = kq_queues + i;
|
|
kq->kq_cpu = i;
|
|
error = kproc_create(ntoskrnl_dpc_thread, kq, &p,
|
|
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows DPC %d", i);
|
|
if (error)
|
|
panic("failed to launch DPC thread");
|
|
}
|
|
|
|
/*
|
|
* Launch the workitem threads.
|
|
*/
|
|
|
|
for (i = 0; i < WORKITEM_THREADS; i++) {
|
|
kq = wq_queues + i;
|
|
error = kproc_create(ntoskrnl_workitem_thread, kq, &p,
|
|
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Workitem %d", i);
|
|
if (error)
|
|
panic("failed to launch workitem thread");
|
|
}
|
|
|
|
patch = ntoskrnl_functbl;
|
|
while (patch->ipt_func != NULL) {
|
|
windrv_wrap((funcptr)patch->ipt_func,
|
|
(funcptr *)&patch->ipt_wrap,
|
|
patch->ipt_argcnt, patch->ipt_ftype);
|
|
patch++;
|
|
}
|
|
|
|
for (i = 0; i < NTOSKRNL_TIMEOUTS; i++) {
|
|
e = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(callout_entry), 0);
|
|
if (e == NULL)
|
|
panic("failed to allocate timeouts");
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
}
|
|
|
|
/*
|
|
* MDLs are supposed to be variable size (they describe
|
|
* buffers containing some number of pages, but we don't
|
|
* know ahead of time how many pages that will be). But
|
|
* always allocating them off the heap is very slow. As
|
|
* a compromise, we create an MDL UMA zone big enough to
|
|
* handle any buffer requiring up to 16 pages, and we
|
|
* use those for any MDLs for buffers of 16 pages or less
|
|
* in size. For buffers larger than that (which we assume
|
|
* will be few and far between, we allocate the MDLs off
|
|
* the heap.
|
|
*/
|
|
|
|
mdl_zone = uma_zcreate("Windows MDL", MDL_ZONE_SIZE,
|
|
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
|
|
|
|
iw_zone = uma_zcreate("Windows WorkItem", sizeof(io_workitem),
|
|
NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
|
|
|
|
return (0);
|
|
}
|
|
|
|
int
|
|
ntoskrnl_libfini()
|
|
{
|
|
image_patch_table *patch;
|
|
callout_entry *e;
|
|
list_entry *l;
|
|
|
|
patch = ntoskrnl_functbl;
|
|
while (patch->ipt_func != NULL) {
|
|
windrv_unwrap(patch->ipt_wrap);
|
|
patch++;
|
|
}
|
|
|
|
/* Stop the workitem queues. */
|
|
ntoskrnl_destroy_workitem_threads();
|
|
/* Stop the DPC queues. */
|
|
ntoskrnl_destroy_dpc_threads();
|
|
|
|
ExFreePool(kq_queues);
|
|
ExFreePool(wq_queues);
|
|
|
|
uma_zdestroy(mdl_zone);
|
|
uma_zdestroy(iw_zone);
|
|
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
while(!IsListEmpty(&ntoskrnl_calllist)) {
|
|
l = RemoveHeadList(&ntoskrnl_calllist);
|
|
e = CONTAINING_RECORD(l, callout_entry, ce_list);
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
ExFreePool(e);
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
}
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
|
|
mtx_destroy(&ntoskrnl_dispatchlock);
|
|
mtx_destroy(&ntoskrnl_interlock);
|
|
mtx_destroy(&ntoskrnl_calllock);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* We need to be able to reference this externally from the wrapper;
|
|
* GCC only generates a local implementation of memset.
|
|
*/
|
|
static void *
|
|
ntoskrnl_memset(buf, ch, size)
|
|
void *buf;
|
|
int ch;
|
|
size_t size;
|
|
{
|
|
return (memset(buf, ch, size));
|
|
}
|
|
|
|
static void *
|
|
ntoskrnl_memmove(dst, src, size)
|
|
void *src;
|
|
void *dst;
|
|
size_t size;
|
|
{
|
|
bcopy(src, dst, size);
|
|
return (dst);
|
|
}
|
|
|
|
static void *
|
|
ntoskrnl_memchr(void *buf, unsigned char ch, size_t len)
|
|
{
|
|
if (len != 0) {
|
|
unsigned char *p = buf;
|
|
|
|
do {
|
|
if (*p++ == ch)
|
|
return (p - 1);
|
|
} while (--len != 0);
|
|
}
|
|
return (NULL);
|
|
}
|
|
|
|
static char *
|
|
ntoskrnl_strstr(s, find)
|
|
char *s, *find;
|
|
{
|
|
char c, sc;
|
|
size_t len;
|
|
|
|
if ((c = *find++) != 0) {
|
|
len = strlen(find);
|
|
do {
|
|
do {
|
|
if ((sc = *s++) == 0)
|
|
return (NULL);
|
|
} while (sc != c);
|
|
} while (strncmp(s, find, len) != 0);
|
|
s--;
|
|
}
|
|
return ((char *)s);
|
|
}
|
|
|
|
/* Taken from libc */
|
|
static char *
|
|
ntoskrnl_strncat(dst, src, n)
|
|
char *dst;
|
|
char *src;
|
|
size_t n;
|
|
{
|
|
if (n != 0) {
|
|
char *d = dst;
|
|
const char *s = src;
|
|
|
|
while (*d != 0)
|
|
d++;
|
|
do {
|
|
if ((*d = *s++) == 0)
|
|
break;
|
|
d++;
|
|
} while (--n != 0);
|
|
*d = 0;
|
|
}
|
|
return (dst);
|
|
}
|
|
|
|
static int
|
|
ntoskrnl_toupper(c)
|
|
int c;
|
|
{
|
|
return (toupper(c));
|
|
}
|
|
|
|
static int
|
|
ntoskrnl_tolower(c)
|
|
int c;
|
|
{
|
|
return (tolower(c));
|
|
}
|
|
|
|
static uint8_t
|
|
RtlEqualUnicodeString(unicode_string *str1, unicode_string *str2,
|
|
uint8_t caseinsensitive)
|
|
{
|
|
int i;
|
|
|
|
if (str1->us_len != str2->us_len)
|
|
return (FALSE);
|
|
|
|
for (i = 0; i < str1->us_len; i++) {
|
|
if (caseinsensitive == TRUE) {
|
|
if (toupper((char)(str1->us_buf[i] & 0xFF)) !=
|
|
toupper((char)(str2->us_buf[i] & 0xFF)))
|
|
return (FALSE);
|
|
} else {
|
|
if (str1->us_buf[i] != str2->us_buf[i])
|
|
return (FALSE);
|
|
}
|
|
}
|
|
|
|
return (TRUE);
|
|
}
|
|
|
|
static void
|
|
RtlCopyUnicodeString(dest, src)
|
|
unicode_string *dest;
|
|
unicode_string *src;
|
|
{
|
|
|
|
if (dest->us_maxlen >= src->us_len)
|
|
dest->us_len = src->us_len;
|
|
else
|
|
dest->us_len = dest->us_maxlen;
|
|
memcpy(dest->us_buf, src->us_buf, dest->us_len);
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_ascii_to_unicode(ascii, unicode, len)
|
|
char *ascii;
|
|
uint16_t *unicode;
|
|
int len;
|
|
{
|
|
int i;
|
|
uint16_t *ustr;
|
|
|
|
ustr = unicode;
|
|
for (i = 0; i < len; i++) {
|
|
*ustr = (uint16_t)ascii[i];
|
|
ustr++;
|
|
}
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_unicode_to_ascii(unicode, ascii, len)
|
|
uint16_t *unicode;
|
|
char *ascii;
|
|
int len;
|
|
{
|
|
int i;
|
|
uint8_t *astr;
|
|
|
|
astr = ascii;
|
|
for (i = 0; i < len / 2; i++) {
|
|
*astr = (uint8_t)unicode[i];
|
|
astr++;
|
|
}
|
|
}
|
|
|
|
uint32_t
|
|
RtlUnicodeStringToAnsiString(ansi_string *dest, unicode_string *src, uint8_t allocate)
|
|
{
|
|
if (dest == NULL || src == NULL)
|
|
return (STATUS_INVALID_PARAMETER);
|
|
|
|
dest->as_len = src->us_len / 2;
|
|
if (dest->as_maxlen < dest->as_len)
|
|
dest->as_len = dest->as_maxlen;
|
|
|
|
if (allocate == TRUE) {
|
|
dest->as_buf = ExAllocatePoolWithTag(NonPagedPool,
|
|
(src->us_len / 2) + 1, 0);
|
|
if (dest->as_buf == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
dest->as_len = dest->as_maxlen = src->us_len / 2;
|
|
} else {
|
|
dest->as_len = src->us_len / 2; /* XXX */
|
|
if (dest->as_maxlen < dest->as_len)
|
|
dest->as_len = dest->as_maxlen;
|
|
}
|
|
|
|
ntoskrnl_unicode_to_ascii(src->us_buf, dest->as_buf,
|
|
dest->as_len * 2);
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
uint32_t
|
|
RtlAnsiStringToUnicodeString(unicode_string *dest, ansi_string *src,
|
|
uint8_t allocate)
|
|
{
|
|
if (dest == NULL || src == NULL)
|
|
return (STATUS_INVALID_PARAMETER);
|
|
|
|
if (allocate == TRUE) {
|
|
dest->us_buf = ExAllocatePoolWithTag(NonPagedPool,
|
|
src->as_len * 2, 0);
|
|
if (dest->us_buf == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
dest->us_len = dest->us_maxlen = strlen(src->as_buf) * 2;
|
|
} else {
|
|
dest->us_len = src->as_len * 2; /* XXX */
|
|
if (dest->us_maxlen < dest->us_len)
|
|
dest->us_len = dest->us_maxlen;
|
|
}
|
|
|
|
ntoskrnl_ascii_to_unicode(src->as_buf, dest->us_buf,
|
|
dest->us_len / 2);
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
void *
|
|
ExAllocatePoolWithTag(pooltype, len, tag)
|
|
uint32_t pooltype;
|
|
size_t len;
|
|
uint32_t tag;
|
|
{
|
|
void *buf;
|
|
|
|
buf = malloc(len, M_DEVBUF, M_NOWAIT|M_ZERO);
|
|
if (buf == NULL)
|
|
return (NULL);
|
|
|
|
return (buf);
|
|
}
|
|
|
|
void
|
|
ExFreePool(buf)
|
|
void *buf;
|
|
{
|
|
free(buf, M_DEVBUF);
|
|
}
|
|
|
|
uint32_t
|
|
IoAllocateDriverObjectExtension(drv, clid, extlen, ext)
|
|
driver_object *drv;
|
|
void *clid;
|
|
uint32_t extlen;
|
|
void **ext;
|
|
{
|
|
custom_extension *ce;
|
|
|
|
ce = ExAllocatePoolWithTag(NonPagedPool, sizeof(custom_extension)
|
|
+ extlen, 0);
|
|
|
|
if (ce == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
|
|
ce->ce_clid = clid;
|
|
InsertTailList((&drv->dro_driverext->dre_usrext), (&ce->ce_list));
|
|
|
|
*ext = (void *)(ce + 1);
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
void *
|
|
IoGetDriverObjectExtension(drv, clid)
|
|
driver_object *drv;
|
|
void *clid;
|
|
{
|
|
list_entry *e;
|
|
custom_extension *ce;
|
|
|
|
/*
|
|
* Sanity check. Our dummy bus drivers don't have
|
|
* any driver extentions.
|
|
*/
|
|
|
|
if (drv->dro_driverext == NULL)
|
|
return (NULL);
|
|
|
|
e = drv->dro_driverext->dre_usrext.nle_flink;
|
|
while (e != &drv->dro_driverext->dre_usrext) {
|
|
ce = (custom_extension *)e;
|
|
if (ce->ce_clid == clid)
|
|
return ((void *)(ce + 1));
|
|
e = e->nle_flink;
|
|
}
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
|
|
uint32_t
|
|
IoCreateDevice(driver_object *drv, uint32_t devextlen, unicode_string *devname,
|
|
uint32_t devtype, uint32_t devchars, uint8_t exclusive,
|
|
device_object **newdev)
|
|
{
|
|
device_object *dev;
|
|
|
|
dev = ExAllocatePoolWithTag(NonPagedPool, sizeof(device_object), 0);
|
|
if (dev == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
|
|
dev->do_type = devtype;
|
|
dev->do_drvobj = drv;
|
|
dev->do_currirp = NULL;
|
|
dev->do_flags = 0;
|
|
|
|
if (devextlen) {
|
|
dev->do_devext = ExAllocatePoolWithTag(NonPagedPool,
|
|
devextlen, 0);
|
|
|
|
if (dev->do_devext == NULL) {
|
|
ExFreePool(dev);
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
}
|
|
|
|
bzero(dev->do_devext, devextlen);
|
|
} else
|
|
dev->do_devext = NULL;
|
|
|
|
dev->do_size = sizeof(device_object) + devextlen;
|
|
dev->do_refcnt = 1;
|
|
dev->do_attacheddev = NULL;
|
|
dev->do_nextdev = NULL;
|
|
dev->do_devtype = devtype;
|
|
dev->do_stacksize = 1;
|
|
dev->do_alignreq = 1;
|
|
dev->do_characteristics = devchars;
|
|
dev->do_iotimer = NULL;
|
|
KeInitializeEvent(&dev->do_devlock, EVENT_TYPE_SYNC, TRUE);
|
|
|
|
/*
|
|
* Vpd is used for disk/tape devices,
|
|
* but we don't support those. (Yet.)
|
|
*/
|
|
dev->do_vpb = NULL;
|
|
|
|
dev->do_devobj_ext = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(devobj_extension), 0);
|
|
|
|
if (dev->do_devobj_ext == NULL) {
|
|
if (dev->do_devext != NULL)
|
|
ExFreePool(dev->do_devext);
|
|
ExFreePool(dev);
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
}
|
|
|
|
dev->do_devobj_ext->dve_type = 0;
|
|
dev->do_devobj_ext->dve_size = sizeof(devobj_extension);
|
|
dev->do_devobj_ext->dve_devobj = dev;
|
|
|
|
/*
|
|
* Attach this device to the driver object's list
|
|
* of devices. Note: this is not the same as attaching
|
|
* the device to the device stack. The driver's AddDevice
|
|
* routine must explicitly call IoAddDeviceToDeviceStack()
|
|
* to do that.
|
|
*/
|
|
|
|
if (drv->dro_devobj == NULL) {
|
|
drv->dro_devobj = dev;
|
|
dev->do_nextdev = NULL;
|
|
} else {
|
|
dev->do_nextdev = drv->dro_devobj;
|
|
drv->dro_devobj = dev;
|
|
}
|
|
|
|
*newdev = dev;
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
void
|
|
IoDeleteDevice(dev)
|
|
device_object *dev;
|
|
{
|
|
device_object *prev;
|
|
|
|
if (dev == NULL)
|
|
return;
|
|
|
|
if (dev->do_devobj_ext != NULL)
|
|
ExFreePool(dev->do_devobj_ext);
|
|
|
|
if (dev->do_devext != NULL)
|
|
ExFreePool(dev->do_devext);
|
|
|
|
/* Unlink the device from the driver's device list. */
|
|
|
|
prev = dev->do_drvobj->dro_devobj;
|
|
if (prev == dev)
|
|
dev->do_drvobj->dro_devobj = dev->do_nextdev;
|
|
else {
|
|
while (prev->do_nextdev != dev)
|
|
prev = prev->do_nextdev;
|
|
prev->do_nextdev = dev->do_nextdev;
|
|
}
|
|
|
|
ExFreePool(dev);
|
|
}
|
|
|
|
device_object *
|
|
IoGetAttachedDevice(dev)
|
|
device_object *dev;
|
|
{
|
|
device_object *d;
|
|
|
|
if (dev == NULL)
|
|
return (NULL);
|
|
|
|
d = dev;
|
|
|
|
while (d->do_attacheddev != NULL)
|
|
d = d->do_attacheddev;
|
|
|
|
return (d);
|
|
}
|
|
|
|
static irp *
|
|
IoBuildSynchronousFsdRequest(func, dobj, buf, len, off, event, status)
|
|
uint32_t func;
|
|
device_object *dobj;
|
|
void *buf;
|
|
uint32_t len;
|
|
uint64_t *off;
|
|
nt_kevent *event;
|
|
io_status_block *status;
|
|
{
|
|
irp *ip;
|
|
|
|
ip = IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status);
|
|
if (ip == NULL)
|
|
return (NULL);
|
|
ip->irp_usrevent = event;
|
|
|
|
return (ip);
|
|
}
|
|
|
|
static irp *
|
|
IoBuildAsynchronousFsdRequest(func, dobj, buf, len, off, status)
|
|
uint32_t func;
|
|
device_object *dobj;
|
|
void *buf;
|
|
uint32_t len;
|
|
uint64_t *off;
|
|
io_status_block *status;
|
|
{
|
|
irp *ip;
|
|
io_stack_location *sl;
|
|
|
|
ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
|
|
if (ip == NULL)
|
|
return (NULL);
|
|
|
|
ip->irp_usriostat = status;
|
|
ip->irp_tail.irp_overlay.irp_thread = NULL;
|
|
|
|
sl = IoGetNextIrpStackLocation(ip);
|
|
sl->isl_major = func;
|
|
sl->isl_minor = 0;
|
|
sl->isl_flags = 0;
|
|
sl->isl_ctl = 0;
|
|
sl->isl_devobj = dobj;
|
|
sl->isl_fileobj = NULL;
|
|
sl->isl_completionfunc = NULL;
|
|
|
|
ip->irp_userbuf = buf;
|
|
|
|
if (dobj->do_flags & DO_BUFFERED_IO) {
|
|
ip->irp_assoc.irp_sysbuf =
|
|
ExAllocatePoolWithTag(NonPagedPool, len, 0);
|
|
if (ip->irp_assoc.irp_sysbuf == NULL) {
|
|
IoFreeIrp(ip);
|
|
return (NULL);
|
|
}
|
|
bcopy(buf, ip->irp_assoc.irp_sysbuf, len);
|
|
}
|
|
|
|
if (dobj->do_flags & DO_DIRECT_IO) {
|
|
ip->irp_mdl = IoAllocateMdl(buf, len, FALSE, FALSE, ip);
|
|
if (ip->irp_mdl == NULL) {
|
|
if (ip->irp_assoc.irp_sysbuf != NULL)
|
|
ExFreePool(ip->irp_assoc.irp_sysbuf);
|
|
IoFreeIrp(ip);
|
|
return (NULL);
|
|
}
|
|
ip->irp_userbuf = NULL;
|
|
ip->irp_assoc.irp_sysbuf = NULL;
|
|
}
|
|
|
|
if (func == IRP_MJ_READ) {
|
|
sl->isl_parameters.isl_read.isl_len = len;
|
|
if (off != NULL)
|
|
sl->isl_parameters.isl_read.isl_byteoff = *off;
|
|
else
|
|
sl->isl_parameters.isl_read.isl_byteoff = 0;
|
|
}
|
|
|
|
if (func == IRP_MJ_WRITE) {
|
|
sl->isl_parameters.isl_write.isl_len = len;
|
|
if (off != NULL)
|
|
sl->isl_parameters.isl_write.isl_byteoff = *off;
|
|
else
|
|
sl->isl_parameters.isl_write.isl_byteoff = 0;
|
|
}
|
|
|
|
return (ip);
|
|
}
|
|
|
|
static irp *
|
|
IoBuildDeviceIoControlRequest(uint32_t iocode, device_object *dobj, void *ibuf,
|
|
uint32_t ilen, void *obuf, uint32_t olen, uint8_t isinternal,
|
|
nt_kevent *event, io_status_block *status)
|
|
{
|
|
irp *ip;
|
|
io_stack_location *sl;
|
|
uint32_t buflen;
|
|
|
|
ip = IoAllocateIrp(dobj->do_stacksize, TRUE);
|
|
if (ip == NULL)
|
|
return (NULL);
|
|
ip->irp_usrevent = event;
|
|
ip->irp_usriostat = status;
|
|
ip->irp_tail.irp_overlay.irp_thread = NULL;
|
|
|
|
sl = IoGetNextIrpStackLocation(ip);
|
|
sl->isl_major = isinternal == TRUE ?
|
|
IRP_MJ_INTERNAL_DEVICE_CONTROL : IRP_MJ_DEVICE_CONTROL;
|
|
sl->isl_minor = 0;
|
|
sl->isl_flags = 0;
|
|
sl->isl_ctl = 0;
|
|
sl->isl_devobj = dobj;
|
|
sl->isl_fileobj = NULL;
|
|
sl->isl_completionfunc = NULL;
|
|
sl->isl_parameters.isl_ioctl.isl_iocode = iocode;
|
|
sl->isl_parameters.isl_ioctl.isl_ibuflen = ilen;
|
|
sl->isl_parameters.isl_ioctl.isl_obuflen = olen;
|
|
|
|
switch(IO_METHOD(iocode)) {
|
|
case METHOD_BUFFERED:
|
|
if (ilen > olen)
|
|
buflen = ilen;
|
|
else
|
|
buflen = olen;
|
|
if (buflen) {
|
|
ip->irp_assoc.irp_sysbuf =
|
|
ExAllocatePoolWithTag(NonPagedPool, buflen, 0);
|
|
if (ip->irp_assoc.irp_sysbuf == NULL) {
|
|
IoFreeIrp(ip);
|
|
return (NULL);
|
|
}
|
|
}
|
|
if (ilen && ibuf != NULL) {
|
|
bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
|
|
bzero((char *)ip->irp_assoc.irp_sysbuf + ilen,
|
|
buflen - ilen);
|
|
} else
|
|
bzero(ip->irp_assoc.irp_sysbuf, ilen);
|
|
ip->irp_userbuf = obuf;
|
|
break;
|
|
case METHOD_IN_DIRECT:
|
|
case METHOD_OUT_DIRECT:
|
|
if (ilen && ibuf != NULL) {
|
|
ip->irp_assoc.irp_sysbuf =
|
|
ExAllocatePoolWithTag(NonPagedPool, ilen, 0);
|
|
if (ip->irp_assoc.irp_sysbuf == NULL) {
|
|
IoFreeIrp(ip);
|
|
return (NULL);
|
|
}
|
|
bcopy(ibuf, ip->irp_assoc.irp_sysbuf, ilen);
|
|
}
|
|
if (olen && obuf != NULL) {
|
|
ip->irp_mdl = IoAllocateMdl(obuf, olen,
|
|
FALSE, FALSE, ip);
|
|
/*
|
|
* Normally we would MmProbeAndLockPages()
|
|
* here, but we don't have to in our
|
|
* imlementation.
|
|
*/
|
|
}
|
|
break;
|
|
case METHOD_NEITHER:
|
|
ip->irp_userbuf = obuf;
|
|
sl->isl_parameters.isl_ioctl.isl_type3ibuf = ibuf;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
/*
|
|
* Ideally, we should associate this IRP with the calling
|
|
* thread here.
|
|
*/
|
|
|
|
return (ip);
|
|
}
|
|
|
|
static irp *
|
|
IoAllocateIrp(uint8_t stsize, uint8_t chargequota)
|
|
{
|
|
irp *i;
|
|
|
|
i = ExAllocatePoolWithTag(NonPagedPool, IoSizeOfIrp(stsize), 0);
|
|
if (i == NULL)
|
|
return (NULL);
|
|
|
|
IoInitializeIrp(i, IoSizeOfIrp(stsize), stsize);
|
|
|
|
return (i);
|
|
}
|
|
|
|
static irp *
|
|
IoMakeAssociatedIrp(irp *ip, uint8_t stsize)
|
|
{
|
|
irp *associrp;
|
|
|
|
associrp = IoAllocateIrp(stsize, FALSE);
|
|
if (associrp == NULL)
|
|
return (NULL);
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
associrp->irp_flags |= IRP_ASSOCIATED_IRP;
|
|
associrp->irp_tail.irp_overlay.irp_thread =
|
|
ip->irp_tail.irp_overlay.irp_thread;
|
|
associrp->irp_assoc.irp_master = ip;
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (associrp);
|
|
}
|
|
|
|
static void
|
|
IoFreeIrp(ip)
|
|
irp *ip;
|
|
{
|
|
ExFreePool(ip);
|
|
}
|
|
|
|
static void
|
|
IoInitializeIrp(irp *io, uint16_t psize, uint8_t ssize)
|
|
{
|
|
bzero((char *)io, IoSizeOfIrp(ssize));
|
|
io->irp_size = psize;
|
|
io->irp_stackcnt = ssize;
|
|
io->irp_currentstackloc = ssize;
|
|
InitializeListHead(&io->irp_thlist);
|
|
io->irp_tail.irp_overlay.irp_csl =
|
|
(io_stack_location *)(io + 1) + ssize;
|
|
}
|
|
|
|
static void
|
|
IoReuseIrp(ip, status)
|
|
irp *ip;
|
|
uint32_t status;
|
|
{
|
|
uint8_t allocflags;
|
|
|
|
allocflags = ip->irp_allocflags;
|
|
IoInitializeIrp(ip, ip->irp_size, ip->irp_stackcnt);
|
|
ip->irp_iostat.isb_status = status;
|
|
ip->irp_allocflags = allocflags;
|
|
}
|
|
|
|
void
|
|
IoAcquireCancelSpinLock(uint8_t *irql)
|
|
{
|
|
KeAcquireSpinLock(&ntoskrnl_cancellock, irql);
|
|
}
|
|
|
|
void
|
|
IoReleaseCancelSpinLock(uint8_t irql)
|
|
{
|
|
KeReleaseSpinLock(&ntoskrnl_cancellock, irql);
|
|
}
|
|
|
|
uint8_t
|
|
IoCancelIrp(irp *ip)
|
|
{
|
|
cancel_func cfunc;
|
|
uint8_t cancelirql;
|
|
|
|
IoAcquireCancelSpinLock(&cancelirql);
|
|
cfunc = IoSetCancelRoutine(ip, NULL);
|
|
ip->irp_cancel = TRUE;
|
|
if (cfunc == NULL) {
|
|
IoReleaseCancelSpinLock(cancelirql);
|
|
return (FALSE);
|
|
}
|
|
ip->irp_cancelirql = cancelirql;
|
|
MSCALL2(cfunc, IoGetCurrentIrpStackLocation(ip)->isl_devobj, ip);
|
|
return (uint8_t)IoSetCancelValue(ip, TRUE);
|
|
}
|
|
|
|
uint32_t
|
|
IofCallDriver(dobj, ip)
|
|
device_object *dobj;
|
|
irp *ip;
|
|
{
|
|
driver_object *drvobj;
|
|
io_stack_location *sl;
|
|
uint32_t status;
|
|
driver_dispatch disp;
|
|
|
|
drvobj = dobj->do_drvobj;
|
|
|
|
if (ip->irp_currentstackloc <= 0)
|
|
panic("IoCallDriver(): out of stack locations");
|
|
|
|
IoSetNextIrpStackLocation(ip);
|
|
sl = IoGetCurrentIrpStackLocation(ip);
|
|
|
|
sl->isl_devobj = dobj;
|
|
|
|
disp = drvobj->dro_dispatch[sl->isl_major];
|
|
status = MSCALL2(disp, dobj, ip);
|
|
|
|
return (status);
|
|
}
|
|
|
|
void
|
|
IofCompleteRequest(irp *ip, uint8_t prioboost)
|
|
{
|
|
uint32_t status;
|
|
device_object *dobj;
|
|
io_stack_location *sl;
|
|
completion_func cf;
|
|
|
|
KASSERT(ip->irp_iostat.isb_status != STATUS_PENDING,
|
|
("incorrect IRP(%p) status (STATUS_PENDING)", ip));
|
|
|
|
sl = IoGetCurrentIrpStackLocation(ip);
|
|
IoSkipCurrentIrpStackLocation(ip);
|
|
|
|
do {
|
|
if (sl->isl_ctl & SL_PENDING_RETURNED)
|
|
ip->irp_pendingreturned = TRUE;
|
|
|
|
if (ip->irp_currentstackloc != (ip->irp_stackcnt + 1))
|
|
dobj = IoGetCurrentIrpStackLocation(ip)->isl_devobj;
|
|
else
|
|
dobj = NULL;
|
|
|
|
if (sl->isl_completionfunc != NULL &&
|
|
((ip->irp_iostat.isb_status == STATUS_SUCCESS &&
|
|
sl->isl_ctl & SL_INVOKE_ON_SUCCESS) ||
|
|
(ip->irp_iostat.isb_status != STATUS_SUCCESS &&
|
|
sl->isl_ctl & SL_INVOKE_ON_ERROR) ||
|
|
(ip->irp_cancel == TRUE &&
|
|
sl->isl_ctl & SL_INVOKE_ON_CANCEL))) {
|
|
cf = sl->isl_completionfunc;
|
|
status = MSCALL3(cf, dobj, ip, sl->isl_completionctx);
|
|
if (status == STATUS_MORE_PROCESSING_REQUIRED)
|
|
return;
|
|
} else {
|
|
if ((ip->irp_currentstackloc <= ip->irp_stackcnt) &&
|
|
(ip->irp_pendingreturned == TRUE))
|
|
IoMarkIrpPending(ip);
|
|
}
|
|
|
|
/* move to the next. */
|
|
IoSkipCurrentIrpStackLocation(ip);
|
|
sl++;
|
|
} while (ip->irp_currentstackloc <= (ip->irp_stackcnt + 1));
|
|
|
|
if (ip->irp_usriostat != NULL)
|
|
*ip->irp_usriostat = ip->irp_iostat;
|
|
if (ip->irp_usrevent != NULL)
|
|
KeSetEvent(ip->irp_usrevent, prioboost, FALSE);
|
|
|
|
/* Handle any associated IRPs. */
|
|
|
|
if (ip->irp_flags & IRP_ASSOCIATED_IRP) {
|
|
uint32_t masterirpcnt;
|
|
irp *masterirp;
|
|
mdl *m;
|
|
|
|
masterirp = ip->irp_assoc.irp_master;
|
|
masterirpcnt =
|
|
InterlockedDecrement(&masterirp->irp_assoc.irp_irpcnt);
|
|
|
|
while ((m = ip->irp_mdl) != NULL) {
|
|
ip->irp_mdl = m->mdl_next;
|
|
IoFreeMdl(m);
|
|
}
|
|
IoFreeIrp(ip);
|
|
if (masterirpcnt == 0)
|
|
IoCompleteRequest(masterirp, IO_NO_INCREMENT);
|
|
return;
|
|
}
|
|
|
|
/* With any luck, these conditions will never arise. */
|
|
|
|
if (ip->irp_flags & IRP_PAGING_IO) {
|
|
if (ip->irp_mdl != NULL)
|
|
IoFreeMdl(ip->irp_mdl);
|
|
IoFreeIrp(ip);
|
|
}
|
|
}
|
|
|
|
void
|
|
ntoskrnl_intr(arg)
|
|
void *arg;
|
|
{
|
|
kinterrupt *iobj;
|
|
uint8_t irql;
|
|
uint8_t claimed;
|
|
list_entry *l;
|
|
|
|
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
|
|
l = ntoskrnl_intlist.nle_flink;
|
|
while (l != &ntoskrnl_intlist) {
|
|
iobj = CONTAINING_RECORD(l, kinterrupt, ki_list);
|
|
claimed = MSCALL2(iobj->ki_svcfunc, iobj, iobj->ki_svcctx);
|
|
if (claimed == TRUE)
|
|
break;
|
|
l = l->nle_flink;
|
|
}
|
|
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
|
|
}
|
|
|
|
uint8_t
|
|
KeAcquireInterruptSpinLock(iobj)
|
|
kinterrupt *iobj;
|
|
{
|
|
uint8_t irql;
|
|
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
|
|
return (irql);
|
|
}
|
|
|
|
void
|
|
KeReleaseInterruptSpinLock(kinterrupt *iobj, uint8_t irql)
|
|
{
|
|
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
|
|
}
|
|
|
|
uint8_t
|
|
KeSynchronizeExecution(iobj, syncfunc, syncctx)
|
|
kinterrupt *iobj;
|
|
void *syncfunc;
|
|
void *syncctx;
|
|
{
|
|
uint8_t irql;
|
|
|
|
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
|
|
MSCALL1(syncfunc, syncctx);
|
|
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
|
|
|
|
return (TRUE);
|
|
}
|
|
|
|
/*
|
|
* IoConnectInterrupt() is passed only the interrupt vector and
|
|
* irql that a device wants to use, but no device-specific tag
|
|
* of any kind. This conflicts rather badly with FreeBSD's
|
|
* bus_setup_intr(), which needs the device_t for the device
|
|
* requesting interrupt delivery. In order to bypass this
|
|
* inconsistency, we implement a second level of interrupt
|
|
* dispatching on top of bus_setup_intr(). All devices use
|
|
* ntoskrnl_intr() as their ISR, and any device requesting
|
|
* interrupts will be registered with ntoskrnl_intr()'s interrupt
|
|
* dispatch list. When an interrupt arrives, we walk the list
|
|
* and invoke all the registered ISRs. This effectively makes all
|
|
* interrupts shared, but it's the only way to duplicate the
|
|
* semantics of IoConnectInterrupt() and IoDisconnectInterrupt() properly.
|
|
*/
|
|
|
|
uint32_t
|
|
IoConnectInterrupt(kinterrupt **iobj, void *svcfunc, void *svcctx,
|
|
kspin_lock *lock, uint32_t vector, uint8_t irql, uint8_t syncirql,
|
|
uint8_t imode, uint8_t shared, uint32_t affinity, uint8_t savefloat)
|
|
{
|
|
uint8_t curirql;
|
|
|
|
*iobj = ExAllocatePoolWithTag(NonPagedPool, sizeof(kinterrupt), 0);
|
|
if (*iobj == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
|
|
(*iobj)->ki_svcfunc = svcfunc;
|
|
(*iobj)->ki_svcctx = svcctx;
|
|
|
|
if (lock == NULL) {
|
|
KeInitializeSpinLock(&(*iobj)->ki_lock_priv);
|
|
(*iobj)->ki_lock = &(*iobj)->ki_lock_priv;
|
|
} else
|
|
(*iobj)->ki_lock = lock;
|
|
|
|
KeAcquireSpinLock(&ntoskrnl_intlock, &curirql);
|
|
InsertHeadList((&ntoskrnl_intlist), (&(*iobj)->ki_list));
|
|
KeReleaseSpinLock(&ntoskrnl_intlock, curirql);
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
void
|
|
IoDisconnectInterrupt(iobj)
|
|
kinterrupt *iobj;
|
|
{
|
|
uint8_t irql;
|
|
|
|
if (iobj == NULL)
|
|
return;
|
|
|
|
KeAcquireSpinLock(&ntoskrnl_intlock, &irql);
|
|
RemoveEntryList((&iobj->ki_list));
|
|
KeReleaseSpinLock(&ntoskrnl_intlock, irql);
|
|
|
|
ExFreePool(iobj);
|
|
}
|
|
|
|
device_object *
|
|
IoAttachDeviceToDeviceStack(src, dst)
|
|
device_object *src;
|
|
device_object *dst;
|
|
{
|
|
device_object *attached;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
attached = IoGetAttachedDevice(dst);
|
|
attached->do_attacheddev = src;
|
|
src->do_attacheddev = NULL;
|
|
src->do_stacksize = attached->do_stacksize + 1;
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (attached);
|
|
}
|
|
|
|
void
|
|
IoDetachDevice(topdev)
|
|
device_object *topdev;
|
|
{
|
|
device_object *tail;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
/* First, break the chain. */
|
|
tail = topdev->do_attacheddev;
|
|
if (tail == NULL) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
return;
|
|
}
|
|
topdev->do_attacheddev = tail->do_attacheddev;
|
|
topdev->do_refcnt--;
|
|
|
|
/* Now reduce the stacksize count for the takm_il objects. */
|
|
|
|
tail = topdev->do_attacheddev;
|
|
while (tail != NULL) {
|
|
tail->do_stacksize--;
|
|
tail = tail->do_attacheddev;
|
|
}
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
}
|
|
|
|
/*
|
|
* For the most part, an object is considered signalled if
|
|
* dh_sigstate == TRUE. The exception is for mutant objects
|
|
* (mutexes), where the logic works like this:
|
|
*
|
|
* - If the thread already owns the object and sigstate is
|
|
* less than or equal to 0, then the object is considered
|
|
* signalled (recursive acquisition).
|
|
* - If dh_sigstate == 1, the object is also considered
|
|
* signalled.
|
|
*/
|
|
|
|
static int
|
|
ntoskrnl_is_signalled(obj, td)
|
|
nt_dispatch_header *obj;
|
|
struct thread *td;
|
|
{
|
|
kmutant *km;
|
|
|
|
if (obj->dh_type == DISP_TYPE_MUTANT) {
|
|
km = (kmutant *)obj;
|
|
if ((obj->dh_sigstate <= 0 && km->km_ownerthread == td) ||
|
|
obj->dh_sigstate == 1)
|
|
return (TRUE);
|
|
return (FALSE);
|
|
}
|
|
|
|
if (obj->dh_sigstate > 0)
|
|
return (TRUE);
|
|
return (FALSE);
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_satisfy_wait(obj, td)
|
|
nt_dispatch_header *obj;
|
|
struct thread *td;
|
|
{
|
|
kmutant *km;
|
|
|
|
switch (obj->dh_type) {
|
|
case DISP_TYPE_MUTANT:
|
|
km = (struct kmutant *)obj;
|
|
obj->dh_sigstate--;
|
|
/*
|
|
* If sigstate reaches 0, the mutex is now
|
|
* non-signalled (the new thread owns it).
|
|
*/
|
|
if (obj->dh_sigstate == 0) {
|
|
km->km_ownerthread = td;
|
|
if (km->km_abandoned == TRUE)
|
|
km->km_abandoned = FALSE;
|
|
}
|
|
break;
|
|
/* Synchronization objects get reset to unsignalled. */
|
|
case DISP_TYPE_SYNCHRONIZATION_EVENT:
|
|
case DISP_TYPE_SYNCHRONIZATION_TIMER:
|
|
obj->dh_sigstate = 0;
|
|
break;
|
|
case DISP_TYPE_SEMAPHORE:
|
|
obj->dh_sigstate--;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_satisfy_multiple_waits(wb)
|
|
wait_block *wb;
|
|
{
|
|
wait_block *cur;
|
|
struct thread *td;
|
|
|
|
cur = wb;
|
|
td = wb->wb_kthread;
|
|
|
|
do {
|
|
ntoskrnl_satisfy_wait(wb->wb_object, td);
|
|
cur->wb_awakened = TRUE;
|
|
cur = cur->wb_next;
|
|
} while (cur != wb);
|
|
}
|
|
|
|
/* Always called with dispatcher lock held. */
|
|
static void
|
|
ntoskrnl_waittest(obj, increment)
|
|
nt_dispatch_header *obj;
|
|
uint32_t increment;
|
|
{
|
|
wait_block *w, *next;
|
|
list_entry *e;
|
|
struct thread *td;
|
|
wb_ext *we;
|
|
int satisfied;
|
|
|
|
/*
|
|
* Once an object has been signalled, we walk its list of
|
|
* wait blocks. If a wait block can be awakened, then satisfy
|
|
* waits as necessary and wake the thread.
|
|
*
|
|
* The rules work like this:
|
|
*
|
|
* If a wait block is marked as WAITTYPE_ANY, then
|
|
* we can satisfy the wait conditions on the current
|
|
* object and wake the thread right away. Satisfying
|
|
* the wait also has the effect of breaking us out
|
|
* of the search loop.
|
|
*
|
|
* If the object is marked as WAITTYLE_ALL, then the
|
|
* wait block will be part of a circularly linked
|
|
* list of wait blocks belonging to a waiting thread
|
|
* that's sleeping in KeWaitForMultipleObjects(). In
|
|
* order to wake the thread, all the objects in the
|
|
* wait list must be in the signalled state. If they
|
|
* are, we then satisfy all of them and wake the
|
|
* thread.
|
|
*
|
|
*/
|
|
|
|
e = obj->dh_waitlisthead.nle_flink;
|
|
|
|
while (e != &obj->dh_waitlisthead && obj->dh_sigstate > 0) {
|
|
w = CONTAINING_RECORD(e, wait_block, wb_waitlist);
|
|
we = w->wb_ext;
|
|
td = we->we_td;
|
|
satisfied = FALSE;
|
|
if (w->wb_waittype == WAITTYPE_ANY) {
|
|
/*
|
|
* Thread can be awakened if
|
|
* any wait is satisfied.
|
|
*/
|
|
ntoskrnl_satisfy_wait(obj, td);
|
|
satisfied = TRUE;
|
|
w->wb_awakened = TRUE;
|
|
} else {
|
|
/*
|
|
* Thread can only be woken up
|
|
* if all waits are satisfied.
|
|
* If the thread is waiting on multiple
|
|
* objects, they should all be linked
|
|
* through the wb_next pointers in the
|
|
* wait blocks.
|
|
*/
|
|
satisfied = TRUE;
|
|
next = w->wb_next;
|
|
while (next != w) {
|
|
if (ntoskrnl_is_signalled(obj, td) == FALSE) {
|
|
satisfied = FALSE;
|
|
break;
|
|
}
|
|
next = next->wb_next;
|
|
}
|
|
ntoskrnl_satisfy_multiple_waits(w);
|
|
}
|
|
|
|
if (satisfied == TRUE)
|
|
cv_broadcastpri(&we->we_cv,
|
|
(w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
|
|
w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
|
|
|
|
e = e->nle_flink;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Return the number of 100 nanosecond intervals since
|
|
* January 1, 1601. (?!?!)
|
|
*/
|
|
void
|
|
ntoskrnl_time(tval)
|
|
uint64_t *tval;
|
|
{
|
|
struct timespec ts;
|
|
|
|
nanotime(&ts);
|
|
*tval = (uint64_t)ts.tv_nsec / 100 + (uint64_t)ts.tv_sec * 10000000 +
|
|
11644473600 * 10000000; /* 100ns ticks from 1601 to 1970 */
|
|
}
|
|
|
|
static void
|
|
KeQuerySystemTime(current_time)
|
|
uint64_t *current_time;
|
|
{
|
|
ntoskrnl_time(current_time);
|
|
}
|
|
|
|
static uint32_t
|
|
KeTickCount(void)
|
|
{
|
|
struct timeval tv;
|
|
getmicrouptime(&tv);
|
|
return tvtohz(&tv);
|
|
}
|
|
|
|
|
|
/*
|
|
* KeWaitForSingleObject() is a tricky beast, because it can be used
|
|
* with several different object types: semaphores, timers, events,
|
|
* mutexes and threads. Semaphores don't appear very often, but the
|
|
* other object types are quite common. KeWaitForSingleObject() is
|
|
* what's normally used to acquire a mutex, and it can be used to
|
|
* wait for a thread termination.
|
|
*
|
|
* The Windows NDIS API is implemented in terms of Windows kernel
|
|
* primitives, and some of the object manipulation is duplicated in
|
|
* NDIS. For example, NDIS has timers and events, which are actually
|
|
* Windows kevents and ktimers. Now, you're supposed to only use the
|
|
* NDIS variants of these objects within the confines of the NDIS API,
|
|
* but there are some naughty developers out there who will use
|
|
* KeWaitForSingleObject() on NDIS timer and event objects, so we
|
|
* have to support that as well. Conseqently, our NDIS timer and event
|
|
* code has to be closely tied into our ntoskrnl timer and event code,
|
|
* just as it is in Windows.
|
|
*
|
|
* KeWaitForSingleObject() may do different things for different kinds
|
|
* of objects:
|
|
*
|
|
* - For events, we check if the event has been signalled. If the
|
|
* event is already in the signalled state, we just return immediately,
|
|
* otherwise we wait for it to be set to the signalled state by someone
|
|
* else calling KeSetEvent(). Events can be either synchronization or
|
|
* notification events.
|
|
*
|
|
* - For timers, if the timer has already fired and the timer is in
|
|
* the signalled state, we just return, otherwise we wait on the
|
|
* timer. Unlike an event, timers get signalled automatically when
|
|
* they expire rather than someone having to trip them manually.
|
|
* Timers initialized with KeInitializeTimer() are always notification
|
|
* events: KeInitializeTimerEx() lets you initialize a timer as
|
|
* either a notification or synchronization event.
|
|
*
|
|
* - For mutexes, we try to acquire the mutex and if we can't, we wait
|
|
* on the mutex until it's available and then grab it. When a mutex is
|
|
* released, it enters the signalled state, which wakes up one of the
|
|
* threads waiting to acquire it. Mutexes are always synchronization
|
|
* events.
|
|
*
|
|
* - For threads, the only thing we do is wait until the thread object
|
|
* enters a signalled state, which occurs when the thread terminates.
|
|
* Threads are always notification events.
|
|
*
|
|
* A notification event wakes up all threads waiting on an object. A
|
|
* synchronization event wakes up just one. Also, a synchronization event
|
|
* is auto-clearing, which means we automatically set the event back to
|
|
* the non-signalled state once the wakeup is done.
|
|
*/
|
|
|
|
uint32_t
|
|
KeWaitForSingleObject(void *arg, uint32_t reason, uint32_t mode,
|
|
uint8_t alertable, int64_t *duetime)
|
|
{
|
|
wait_block w;
|
|
struct thread *td = curthread;
|
|
struct timeval tv;
|
|
int error = 0;
|
|
uint64_t curtime;
|
|
wb_ext we;
|
|
nt_dispatch_header *obj;
|
|
|
|
obj = arg;
|
|
|
|
if (obj == NULL)
|
|
return (STATUS_INVALID_PARAMETER);
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
cv_init(&we.we_cv, "KeWFS");
|
|
we.we_td = td;
|
|
|
|
/*
|
|
* Check to see if this object is already signalled,
|
|
* and just return without waiting if it is.
|
|
*/
|
|
if (ntoskrnl_is_signalled(obj, td) == TRUE) {
|
|
/* Sanity check the signal state value. */
|
|
if (obj->dh_sigstate != INT32_MIN) {
|
|
ntoskrnl_satisfy_wait(obj, curthread);
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
return (STATUS_SUCCESS);
|
|
} else {
|
|
/*
|
|
* There's a limit to how many times we can
|
|
* recursively acquire a mutant. If we hit
|
|
* the limit, something is very wrong.
|
|
*/
|
|
if (obj->dh_type == DISP_TYPE_MUTANT) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
panic("mutant limit exceeded");
|
|
}
|
|
}
|
|
}
|
|
|
|
bzero((char *)&w, sizeof(wait_block));
|
|
w.wb_object = obj;
|
|
w.wb_ext = &we;
|
|
w.wb_waittype = WAITTYPE_ANY;
|
|
w.wb_next = &w;
|
|
w.wb_waitkey = 0;
|
|
w.wb_awakened = FALSE;
|
|
w.wb_oldpri = td->td_priority;
|
|
|
|
InsertTailList((&obj->dh_waitlisthead), (&w.wb_waitlist));
|
|
|
|
/*
|
|
* The timeout value is specified in 100 nanosecond units
|
|
* and can be a positive or negative number. If it's positive,
|
|
* then the duetime is absolute, and we need to convert it
|
|
* to an absolute offset relative to now in order to use it.
|
|
* If it's negative, then the duetime is relative and we
|
|
* just have to convert the units.
|
|
*/
|
|
|
|
if (duetime != NULL) {
|
|
if (*duetime < 0) {
|
|
tv.tv_sec = - (*duetime) / 10000000;
|
|
tv.tv_usec = (- (*duetime) / 10) -
|
|
(tv.tv_sec * 1000000);
|
|
} else {
|
|
ntoskrnl_time(&curtime);
|
|
if (*duetime < curtime)
|
|
tv.tv_sec = tv.tv_usec = 0;
|
|
else {
|
|
tv.tv_sec = ((*duetime) - curtime) / 10000000;
|
|
tv.tv_usec = ((*duetime) - curtime) / 10 -
|
|
(tv.tv_sec * 1000000);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (duetime == NULL)
|
|
cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
|
|
else
|
|
error = cv_timedwait(&we.we_cv,
|
|
&ntoskrnl_dispatchlock, tvtohz(&tv));
|
|
|
|
RemoveEntryList(&w.wb_waitlist);
|
|
|
|
cv_destroy(&we.we_cv);
|
|
|
|
/* We timed out. Leave the object alone and return status. */
|
|
|
|
if (error == EWOULDBLOCK) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
return (STATUS_TIMEOUT);
|
|
}
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (STATUS_SUCCESS);
|
|
/*
|
|
return (KeWaitForMultipleObjects(1, &obj, WAITTYPE_ALL, reason,
|
|
mode, alertable, duetime, &w));
|
|
*/
|
|
}
|
|
|
|
static uint32_t
|
|
KeWaitForMultipleObjects(uint32_t cnt, nt_dispatch_header *obj[], uint32_t wtype,
|
|
uint32_t reason, uint32_t mode, uint8_t alertable, int64_t *duetime,
|
|
wait_block *wb_array)
|
|
{
|
|
struct thread *td = curthread;
|
|
wait_block *whead, *w;
|
|
wait_block _wb_array[MAX_WAIT_OBJECTS];
|
|
nt_dispatch_header *cur;
|
|
struct timeval tv;
|
|
int i, wcnt = 0, error = 0;
|
|
uint64_t curtime;
|
|
struct timespec t1, t2;
|
|
uint32_t status = STATUS_SUCCESS;
|
|
wb_ext we;
|
|
|
|
if (cnt > MAX_WAIT_OBJECTS)
|
|
return (STATUS_INVALID_PARAMETER);
|
|
if (cnt > THREAD_WAIT_OBJECTS && wb_array == NULL)
|
|
return (STATUS_INVALID_PARAMETER);
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
cv_init(&we.we_cv, "KeWFM");
|
|
we.we_td = td;
|
|
|
|
if (wb_array == NULL)
|
|
whead = _wb_array;
|
|
else
|
|
whead = wb_array;
|
|
|
|
bzero((char *)whead, sizeof(wait_block) * cnt);
|
|
|
|
/* First pass: see if we can satisfy any waits immediately. */
|
|
|
|
wcnt = 0;
|
|
w = whead;
|
|
|
|
for (i = 0; i < cnt; i++) {
|
|
InsertTailList((&obj[i]->dh_waitlisthead),
|
|
(&w->wb_waitlist));
|
|
w->wb_ext = &we;
|
|
w->wb_object = obj[i];
|
|
w->wb_waittype = wtype;
|
|
w->wb_waitkey = i;
|
|
w->wb_awakened = FALSE;
|
|
w->wb_oldpri = td->td_priority;
|
|
w->wb_next = w + 1;
|
|
w++;
|
|
wcnt++;
|
|
if (ntoskrnl_is_signalled(obj[i], td)) {
|
|
/*
|
|
* There's a limit to how many times
|
|
* we can recursively acquire a mutant.
|
|
* If we hit the limit, something
|
|
* is very wrong.
|
|
*/
|
|
if (obj[i]->dh_sigstate == INT32_MIN &&
|
|
obj[i]->dh_type == DISP_TYPE_MUTANT) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
panic("mutant limit exceeded");
|
|
}
|
|
|
|
/*
|
|
* If this is a WAITTYPE_ANY wait, then
|
|
* satisfy the waited object and exit
|
|
* right now.
|
|
*/
|
|
|
|
if (wtype == WAITTYPE_ANY) {
|
|
ntoskrnl_satisfy_wait(obj[i], td);
|
|
status = STATUS_WAIT_0 + i;
|
|
goto wait_done;
|
|
} else {
|
|
w--;
|
|
wcnt--;
|
|
w->wb_object = NULL;
|
|
RemoveEntryList(&w->wb_waitlist);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If this is a WAITTYPE_ALL wait and all objects are
|
|
* already signalled, satisfy the waits and exit now.
|
|
*/
|
|
|
|
if (wtype == WAITTYPE_ALL && wcnt == 0) {
|
|
for (i = 0; i < cnt; i++)
|
|
ntoskrnl_satisfy_wait(obj[i], td);
|
|
status = STATUS_SUCCESS;
|
|
goto wait_done;
|
|
}
|
|
|
|
/*
|
|
* Create a circular waitblock list. The waitcount
|
|
* must always be non-zero when we get here.
|
|
*/
|
|
|
|
(w - 1)->wb_next = whead;
|
|
|
|
/* Wait on any objects that aren't yet signalled. */
|
|
|
|
/* Calculate timeout, if any. */
|
|
|
|
if (duetime != NULL) {
|
|
if (*duetime < 0) {
|
|
tv.tv_sec = - (*duetime) / 10000000;
|
|
tv.tv_usec = (- (*duetime) / 10) -
|
|
(tv.tv_sec * 1000000);
|
|
} else {
|
|
ntoskrnl_time(&curtime);
|
|
if (*duetime < curtime)
|
|
tv.tv_sec = tv.tv_usec = 0;
|
|
else {
|
|
tv.tv_sec = ((*duetime) - curtime) / 10000000;
|
|
tv.tv_usec = ((*duetime) - curtime) / 10 -
|
|
(tv.tv_sec * 1000000);
|
|
}
|
|
}
|
|
}
|
|
|
|
while (wcnt) {
|
|
nanotime(&t1);
|
|
|
|
if (duetime == NULL)
|
|
cv_wait(&we.we_cv, &ntoskrnl_dispatchlock);
|
|
else
|
|
error = cv_timedwait(&we.we_cv,
|
|
&ntoskrnl_dispatchlock, tvtohz(&tv));
|
|
|
|
/* Wait with timeout expired. */
|
|
|
|
if (error) {
|
|
status = STATUS_TIMEOUT;
|
|
goto wait_done;
|
|
}
|
|
|
|
nanotime(&t2);
|
|
|
|
/* See what's been signalled. */
|
|
|
|
w = whead;
|
|
do {
|
|
cur = w->wb_object;
|
|
if (ntoskrnl_is_signalled(cur, td) == TRUE ||
|
|
w->wb_awakened == TRUE) {
|
|
/* Sanity check the signal state value. */
|
|
if (cur->dh_sigstate == INT32_MIN &&
|
|
cur->dh_type == DISP_TYPE_MUTANT) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
panic("mutant limit exceeded");
|
|
}
|
|
wcnt--;
|
|
if (wtype == WAITTYPE_ANY) {
|
|
status = w->wb_waitkey &
|
|
STATUS_WAIT_0;
|
|
goto wait_done;
|
|
}
|
|
}
|
|
w = w->wb_next;
|
|
} while (w != whead);
|
|
|
|
/*
|
|
* If all objects have been signalled, or if this
|
|
* is a WAITTYPE_ANY wait and we were woke up by
|
|
* someone, we can bail.
|
|
*/
|
|
|
|
if (wcnt == 0) {
|
|
status = STATUS_SUCCESS;
|
|
goto wait_done;
|
|
}
|
|
|
|
/*
|
|
* If this is WAITTYPE_ALL wait, and there's still
|
|
* objects that haven't been signalled, deduct the
|
|
* time that's elapsed so far from the timeout and
|
|
* wait again (or continue waiting indefinitely if
|
|
* there's no timeout).
|
|
*/
|
|
|
|
if (duetime != NULL) {
|
|
tv.tv_sec -= (t2.tv_sec - t1.tv_sec);
|
|
tv.tv_usec -= (t2.tv_nsec - t1.tv_nsec) / 1000;
|
|
}
|
|
}
|
|
|
|
|
|
wait_done:
|
|
|
|
cv_destroy(&we.we_cv);
|
|
|
|
for (i = 0; i < cnt; i++) {
|
|
if (whead[i].wb_object != NULL)
|
|
RemoveEntryList(&whead[i].wb_waitlist);
|
|
|
|
}
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (status);
|
|
}
|
|
|
|
static void
|
|
WRITE_REGISTER_USHORT(uint16_t *reg, uint16_t val)
|
|
{
|
|
bus_space_write_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
|
|
}
|
|
|
|
static uint16_t
|
|
READ_REGISTER_USHORT(reg)
|
|
uint16_t *reg;
|
|
{
|
|
return (bus_space_read_2(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
|
|
}
|
|
|
|
static void
|
|
WRITE_REGISTER_ULONG(reg, val)
|
|
uint32_t *reg;
|
|
uint32_t val;
|
|
{
|
|
bus_space_write_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
|
|
}
|
|
|
|
static uint32_t
|
|
READ_REGISTER_ULONG(reg)
|
|
uint32_t *reg;
|
|
{
|
|
return (bus_space_read_4(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
|
|
}
|
|
|
|
static uint8_t
|
|
READ_REGISTER_UCHAR(uint8_t *reg)
|
|
{
|
|
return (bus_space_read_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg));
|
|
}
|
|
|
|
static void
|
|
WRITE_REGISTER_UCHAR(uint8_t *reg, uint8_t val)
|
|
{
|
|
bus_space_write_1(NDIS_BUS_SPACE_MEM, 0x0, (bus_size_t)reg, val);
|
|
}
|
|
|
|
static int64_t
|
|
_allmul(a, b)
|
|
int64_t a;
|
|
int64_t b;
|
|
{
|
|
return (a * b);
|
|
}
|
|
|
|
static int64_t
|
|
_alldiv(a, b)
|
|
int64_t a;
|
|
int64_t b;
|
|
{
|
|
return (a / b);
|
|
}
|
|
|
|
static int64_t
|
|
_allrem(a, b)
|
|
int64_t a;
|
|
int64_t b;
|
|
{
|
|
return (a % b);
|
|
}
|
|
|
|
static uint64_t
|
|
_aullmul(a, b)
|
|
uint64_t a;
|
|
uint64_t b;
|
|
{
|
|
return (a * b);
|
|
}
|
|
|
|
static uint64_t
|
|
_aulldiv(a, b)
|
|
uint64_t a;
|
|
uint64_t b;
|
|
{
|
|
return (a / b);
|
|
}
|
|
|
|
static uint64_t
|
|
_aullrem(a, b)
|
|
uint64_t a;
|
|
uint64_t b;
|
|
{
|
|
return (a % b);
|
|
}
|
|
|
|
static int64_t
|
|
_allshl(int64_t a, uint8_t b)
|
|
{
|
|
return (a << b);
|
|
}
|
|
|
|
static uint64_t
|
|
_aullshl(uint64_t a, uint8_t b)
|
|
{
|
|
return (a << b);
|
|
}
|
|
|
|
static int64_t
|
|
_allshr(int64_t a, uint8_t b)
|
|
{
|
|
return (a >> b);
|
|
}
|
|
|
|
static uint64_t
|
|
_aullshr(uint64_t a, uint8_t b)
|
|
{
|
|
return (a >> b);
|
|
}
|
|
|
|
static slist_entry *
|
|
ntoskrnl_pushsl(head, entry)
|
|
slist_header *head;
|
|
slist_entry *entry;
|
|
{
|
|
slist_entry *oldhead;
|
|
|
|
oldhead = head->slh_list.slh_next;
|
|
entry->sl_next = head->slh_list.slh_next;
|
|
head->slh_list.slh_next = entry;
|
|
head->slh_list.slh_depth++;
|
|
head->slh_list.slh_seq++;
|
|
|
|
return (oldhead);
|
|
}
|
|
|
|
static slist_entry *
|
|
ntoskrnl_popsl(head)
|
|
slist_header *head;
|
|
{
|
|
slist_entry *first;
|
|
|
|
first = head->slh_list.slh_next;
|
|
if (first != NULL) {
|
|
head->slh_list.slh_next = first->sl_next;
|
|
head->slh_list.slh_depth--;
|
|
head->slh_list.slh_seq++;
|
|
}
|
|
|
|
return (first);
|
|
}
|
|
|
|
/*
|
|
* We need this to make lookaside lists work for amd64.
|
|
* We pass a pointer to ExAllocatePoolWithTag() the lookaside
|
|
* list structure. For amd64 to work right, this has to be a
|
|
* pointer to the wrapped version of the routine, not the
|
|
* original. Letting the Windows driver invoke the original
|
|
* function directly will result in a convention calling
|
|
* mismatch and a pretty crash. On x86, this effectively
|
|
* becomes a no-op since ipt_func and ipt_wrap are the same.
|
|
*/
|
|
|
|
static funcptr
|
|
ntoskrnl_findwrap(func)
|
|
funcptr func;
|
|
{
|
|
image_patch_table *patch;
|
|
|
|
patch = ntoskrnl_functbl;
|
|
while (patch->ipt_func != NULL) {
|
|
if ((funcptr)patch->ipt_func == func)
|
|
return ((funcptr)patch->ipt_wrap);
|
|
patch++;
|
|
}
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
static void
|
|
ExInitializePagedLookasideList(paged_lookaside_list *lookaside,
|
|
lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
|
|
uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
|
|
{
|
|
bzero((char *)lookaside, sizeof(paged_lookaside_list));
|
|
|
|
if (size < sizeof(slist_entry))
|
|
lookaside->nll_l.gl_size = sizeof(slist_entry);
|
|
else
|
|
lookaside->nll_l.gl_size = size;
|
|
lookaside->nll_l.gl_tag = tag;
|
|
if (allocfunc == NULL)
|
|
lookaside->nll_l.gl_allocfunc =
|
|
ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
|
|
else
|
|
lookaside->nll_l.gl_allocfunc = allocfunc;
|
|
|
|
if (freefunc == NULL)
|
|
lookaside->nll_l.gl_freefunc =
|
|
ntoskrnl_findwrap((funcptr)ExFreePool);
|
|
else
|
|
lookaside->nll_l.gl_freefunc = freefunc;
|
|
|
|
#ifdef __i386__
|
|
KeInitializeSpinLock(&lookaside->nll_obsoletelock);
|
|
#endif
|
|
|
|
lookaside->nll_l.gl_type = NonPagedPool;
|
|
lookaside->nll_l.gl_depth = depth;
|
|
lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
|
|
}
|
|
|
|
static void
|
|
ExDeletePagedLookasideList(lookaside)
|
|
paged_lookaside_list *lookaside;
|
|
{
|
|
void *buf;
|
|
void (*freefunc)(void *);
|
|
|
|
freefunc = lookaside->nll_l.gl_freefunc;
|
|
while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
|
|
MSCALL1(freefunc, buf);
|
|
}
|
|
|
|
static void
|
|
ExInitializeNPagedLookasideList(npaged_lookaside_list *lookaside,
|
|
lookaside_alloc_func *allocfunc, lookaside_free_func *freefunc,
|
|
uint32_t flags, size_t size, uint32_t tag, uint16_t depth)
|
|
{
|
|
bzero((char *)lookaside, sizeof(npaged_lookaside_list));
|
|
|
|
if (size < sizeof(slist_entry))
|
|
lookaside->nll_l.gl_size = sizeof(slist_entry);
|
|
else
|
|
lookaside->nll_l.gl_size = size;
|
|
lookaside->nll_l.gl_tag = tag;
|
|
if (allocfunc == NULL)
|
|
lookaside->nll_l.gl_allocfunc =
|
|
ntoskrnl_findwrap((funcptr)ExAllocatePoolWithTag);
|
|
else
|
|
lookaside->nll_l.gl_allocfunc = allocfunc;
|
|
|
|
if (freefunc == NULL)
|
|
lookaside->nll_l.gl_freefunc =
|
|
ntoskrnl_findwrap((funcptr)ExFreePool);
|
|
else
|
|
lookaside->nll_l.gl_freefunc = freefunc;
|
|
|
|
#ifdef __i386__
|
|
KeInitializeSpinLock(&lookaside->nll_obsoletelock);
|
|
#endif
|
|
|
|
lookaside->nll_l.gl_type = NonPagedPool;
|
|
lookaside->nll_l.gl_depth = depth;
|
|
lookaside->nll_l.gl_maxdepth = LOOKASIDE_DEPTH;
|
|
}
|
|
|
|
static void
|
|
ExDeleteNPagedLookasideList(lookaside)
|
|
npaged_lookaside_list *lookaside;
|
|
{
|
|
void *buf;
|
|
void (*freefunc)(void *);
|
|
|
|
freefunc = lookaside->nll_l.gl_freefunc;
|
|
while((buf = ntoskrnl_popsl(&lookaside->nll_l.gl_listhead)) != NULL)
|
|
MSCALL1(freefunc, buf);
|
|
}
|
|
|
|
slist_entry *
|
|
InterlockedPushEntrySList(head, entry)
|
|
slist_header *head;
|
|
slist_entry *entry;
|
|
{
|
|
slist_entry *oldhead;
|
|
|
|
mtx_lock_spin(&ntoskrnl_interlock);
|
|
oldhead = ntoskrnl_pushsl(head, entry);
|
|
mtx_unlock_spin(&ntoskrnl_interlock);
|
|
|
|
return (oldhead);
|
|
}
|
|
|
|
slist_entry *
|
|
InterlockedPopEntrySList(head)
|
|
slist_header *head;
|
|
{
|
|
slist_entry *first;
|
|
|
|
mtx_lock_spin(&ntoskrnl_interlock);
|
|
first = ntoskrnl_popsl(head);
|
|
mtx_unlock_spin(&ntoskrnl_interlock);
|
|
|
|
return (first);
|
|
}
|
|
|
|
static slist_entry *
|
|
ExInterlockedPushEntrySList(head, entry, lock)
|
|
slist_header *head;
|
|
slist_entry *entry;
|
|
kspin_lock *lock;
|
|
{
|
|
return (InterlockedPushEntrySList(head, entry));
|
|
}
|
|
|
|
static slist_entry *
|
|
ExInterlockedPopEntrySList(head, lock)
|
|
slist_header *head;
|
|
kspin_lock *lock;
|
|
{
|
|
return (InterlockedPopEntrySList(head));
|
|
}
|
|
|
|
uint16_t
|
|
ExQueryDepthSList(head)
|
|
slist_header *head;
|
|
{
|
|
uint16_t depth;
|
|
|
|
mtx_lock_spin(&ntoskrnl_interlock);
|
|
depth = head->slh_list.slh_depth;
|
|
mtx_unlock_spin(&ntoskrnl_interlock);
|
|
|
|
return (depth);
|
|
}
|
|
|
|
void
|
|
KeInitializeSpinLock(lock)
|
|
kspin_lock *lock;
|
|
{
|
|
*lock = 0;
|
|
}
|
|
|
|
#ifdef __i386__
|
|
void
|
|
KefAcquireSpinLockAtDpcLevel(lock)
|
|
kspin_lock *lock;
|
|
{
|
|
#ifdef NTOSKRNL_DEBUG_SPINLOCKS
|
|
int i = 0;
|
|
#endif
|
|
|
|
while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0) {
|
|
/* sit and spin */;
|
|
#ifdef NTOSKRNL_DEBUG_SPINLOCKS
|
|
i++;
|
|
if (i > 200000000)
|
|
panic("DEADLOCK!");
|
|
#endif
|
|
}
|
|
}
|
|
|
|
void
|
|
KefReleaseSpinLockFromDpcLevel(lock)
|
|
kspin_lock *lock;
|
|
{
|
|
atomic_store_rel_int((volatile u_int *)lock, 0);
|
|
}
|
|
|
|
uint8_t
|
|
KeAcquireSpinLockRaiseToDpc(kspin_lock *lock)
|
|
{
|
|
uint8_t oldirql;
|
|
|
|
if (KeGetCurrentIrql() > DISPATCH_LEVEL)
|
|
panic("IRQL_NOT_LESS_THAN_OR_EQUAL");
|
|
|
|
KeRaiseIrql(DISPATCH_LEVEL, &oldirql);
|
|
KeAcquireSpinLockAtDpcLevel(lock);
|
|
|
|
return (oldirql);
|
|
}
|
|
#else
|
|
void
|
|
KeAcquireSpinLockAtDpcLevel(kspin_lock *lock)
|
|
{
|
|
while (atomic_cmpset_acq_int((volatile u_int *)lock, 0, 1) == 0)
|
|
/* sit and spin */;
|
|
}
|
|
|
|
void
|
|
KeReleaseSpinLockFromDpcLevel(kspin_lock *lock)
|
|
{
|
|
atomic_store_rel_int((volatile u_int *)lock, 0);
|
|
}
|
|
#endif /* __i386__ */
|
|
|
|
uintptr_t
|
|
InterlockedExchange(dst, val)
|
|
volatile uint32_t *dst;
|
|
uintptr_t val;
|
|
{
|
|
uintptr_t r;
|
|
|
|
mtx_lock_spin(&ntoskrnl_interlock);
|
|
r = *dst;
|
|
*dst = val;
|
|
mtx_unlock_spin(&ntoskrnl_interlock);
|
|
|
|
return (r);
|
|
}
|
|
|
|
static uint32_t
|
|
InterlockedIncrement(addend)
|
|
volatile uint32_t *addend;
|
|
{
|
|
atomic_add_long((volatile u_long *)addend, 1);
|
|
return (*addend);
|
|
}
|
|
|
|
static uint32_t
|
|
InterlockedDecrement(addend)
|
|
volatile uint32_t *addend;
|
|
{
|
|
atomic_subtract_long((volatile u_long *)addend, 1);
|
|
return (*addend);
|
|
}
|
|
|
|
static void
|
|
ExInterlockedAddLargeStatistic(addend, inc)
|
|
uint64_t *addend;
|
|
uint32_t inc;
|
|
{
|
|
mtx_lock_spin(&ntoskrnl_interlock);
|
|
*addend += inc;
|
|
mtx_unlock_spin(&ntoskrnl_interlock);
|
|
};
|
|
|
|
mdl *
|
|
IoAllocateMdl(void *vaddr, uint32_t len, uint8_t secondarybuf,
|
|
uint8_t chargequota, irp *iopkt)
|
|
{
|
|
mdl *m;
|
|
int zone = 0;
|
|
|
|
if (MmSizeOfMdl(vaddr, len) > MDL_ZONE_SIZE)
|
|
m = ExAllocatePoolWithTag(NonPagedPool,
|
|
MmSizeOfMdl(vaddr, len), 0);
|
|
else {
|
|
m = uma_zalloc(mdl_zone, M_NOWAIT | M_ZERO);
|
|
zone++;
|
|
}
|
|
|
|
if (m == NULL)
|
|
return (NULL);
|
|
|
|
MmInitializeMdl(m, vaddr, len);
|
|
|
|
/*
|
|
* MmInitializMdl() clears the flags field, so we
|
|
* have to set this here. If the MDL came from the
|
|
* MDL UMA zone, tag it so we can release it to
|
|
* the right place later.
|
|
*/
|
|
if (zone)
|
|
m->mdl_flags = MDL_ZONE_ALLOCED;
|
|
|
|
if (iopkt != NULL) {
|
|
if (secondarybuf == TRUE) {
|
|
mdl *last;
|
|
last = iopkt->irp_mdl;
|
|
while (last->mdl_next != NULL)
|
|
last = last->mdl_next;
|
|
last->mdl_next = m;
|
|
} else {
|
|
if (iopkt->irp_mdl != NULL)
|
|
panic("leaking an MDL in IoAllocateMdl()");
|
|
iopkt->irp_mdl = m;
|
|
}
|
|
}
|
|
|
|
return (m);
|
|
}
|
|
|
|
void
|
|
IoFreeMdl(m)
|
|
mdl *m;
|
|
{
|
|
if (m == NULL)
|
|
return;
|
|
|
|
if (m->mdl_flags & MDL_ZONE_ALLOCED)
|
|
uma_zfree(mdl_zone, m);
|
|
else
|
|
ExFreePool(m);
|
|
}
|
|
|
|
static void *
|
|
MmAllocateContiguousMemory(size, highest)
|
|
uint32_t size;
|
|
uint64_t highest;
|
|
{
|
|
void *addr;
|
|
size_t pagelength = roundup(size, PAGE_SIZE);
|
|
|
|
addr = ExAllocatePoolWithTag(NonPagedPool, pagelength, 0);
|
|
|
|
return (addr);
|
|
}
|
|
|
|
static void *
|
|
MmAllocateContiguousMemorySpecifyCache(size, lowest, highest,
|
|
boundary, cachetype)
|
|
uint32_t size;
|
|
uint64_t lowest;
|
|
uint64_t highest;
|
|
uint64_t boundary;
|
|
enum nt_caching_type cachetype;
|
|
{
|
|
vm_memattr_t memattr;
|
|
void *ret;
|
|
|
|
switch (cachetype) {
|
|
case MmNonCached:
|
|
memattr = VM_MEMATTR_UNCACHEABLE;
|
|
break;
|
|
case MmWriteCombined:
|
|
memattr = VM_MEMATTR_WRITE_COMBINING;
|
|
break;
|
|
case MmNonCachedUnordered:
|
|
memattr = VM_MEMATTR_UNCACHEABLE;
|
|
break;
|
|
case MmCached:
|
|
case MmHardwareCoherentCached:
|
|
case MmUSWCCached:
|
|
default:
|
|
memattr = VM_MEMATTR_DEFAULT;
|
|
break;
|
|
}
|
|
|
|
ret = (void *)kmem_alloc_contig(kernel_map, size, M_ZERO | M_NOWAIT,
|
|
lowest, highest, PAGE_SIZE, boundary, memattr);
|
|
if (ret != NULL)
|
|
malloc_type_allocated(M_DEVBUF, round_page(size));
|
|
return (ret);
|
|
}
|
|
|
|
static void
|
|
MmFreeContiguousMemory(base)
|
|
void *base;
|
|
{
|
|
ExFreePool(base);
|
|
}
|
|
|
|
static void
|
|
MmFreeContiguousMemorySpecifyCache(base, size, cachetype)
|
|
void *base;
|
|
uint32_t size;
|
|
enum nt_caching_type cachetype;
|
|
{
|
|
contigfree(base, size, M_DEVBUF);
|
|
}
|
|
|
|
static uint32_t
|
|
MmSizeOfMdl(vaddr, len)
|
|
void *vaddr;
|
|
size_t len;
|
|
{
|
|
uint32_t l;
|
|
|
|
l = sizeof(struct mdl) +
|
|
(sizeof(vm_offset_t *) * SPAN_PAGES(vaddr, len));
|
|
|
|
return (l);
|
|
}
|
|
|
|
/*
|
|
* The Microsoft documentation says this routine fills in the
|
|
* page array of an MDL with the _physical_ page addresses that
|
|
* comprise the buffer, but we don't really want to do that here.
|
|
* Instead, we just fill in the page array with the kernel virtual
|
|
* addresses of the buffers.
|
|
*/
|
|
void
|
|
MmBuildMdlForNonPagedPool(m)
|
|
mdl *m;
|
|
{
|
|
vm_offset_t *mdl_pages;
|
|
int pagecnt, i;
|
|
|
|
pagecnt = SPAN_PAGES(m->mdl_byteoffset, m->mdl_bytecount);
|
|
|
|
if (pagecnt > (m->mdl_size - sizeof(mdl)) / sizeof(vm_offset_t *))
|
|
panic("not enough pages in MDL to describe buffer");
|
|
|
|
mdl_pages = MmGetMdlPfnArray(m);
|
|
|
|
for (i = 0; i < pagecnt; i++)
|
|
*mdl_pages = (vm_offset_t)m->mdl_startva + (i * PAGE_SIZE);
|
|
|
|
m->mdl_flags |= MDL_SOURCE_IS_NONPAGED_POOL;
|
|
m->mdl_mappedsystemva = MmGetMdlVirtualAddress(m);
|
|
}
|
|
|
|
static void *
|
|
MmMapLockedPages(mdl *buf, uint8_t accessmode)
|
|
{
|
|
buf->mdl_flags |= MDL_MAPPED_TO_SYSTEM_VA;
|
|
return (MmGetMdlVirtualAddress(buf));
|
|
}
|
|
|
|
static void *
|
|
MmMapLockedPagesSpecifyCache(mdl *buf, uint8_t accessmode, uint32_t cachetype,
|
|
void *vaddr, uint32_t bugcheck, uint32_t prio)
|
|
{
|
|
return (MmMapLockedPages(buf, accessmode));
|
|
}
|
|
|
|
static void
|
|
MmUnmapLockedPages(vaddr, buf)
|
|
void *vaddr;
|
|
mdl *buf;
|
|
{
|
|
buf->mdl_flags &= ~MDL_MAPPED_TO_SYSTEM_VA;
|
|
}
|
|
|
|
/*
|
|
* This function has a problem in that it will break if you
|
|
* compile this module without PAE and try to use it on a PAE
|
|
* kernel. Unfortunately, there's no way around this at the
|
|
* moment. It's slightly less broken that using pmap_kextract().
|
|
* You'd think the virtual memory subsystem would help us out
|
|
* here, but it doesn't.
|
|
*/
|
|
|
|
static uint64_t
|
|
MmGetPhysicalAddress(void *base)
|
|
{
|
|
return (pmap_extract(kernel_map->pmap, (vm_offset_t)base));
|
|
}
|
|
|
|
uint8_t
|
|
MmIsAddressValid(vaddr)
|
|
void *vaddr;
|
|
{
|
|
if (pmap_extract(kernel_map->pmap, (vm_offset_t)vaddr))
|
|
return (TRUE);
|
|
|
|
return (FALSE);
|
|
}
|
|
|
|
void *
|
|
MmMapIoSpace(paddr, len, cachetype)
|
|
uint64_t paddr;
|
|
uint32_t len;
|
|
uint32_t cachetype;
|
|
{
|
|
devclass_t nexus_class;
|
|
device_t *nexus_devs, devp;
|
|
int nexus_count = 0;
|
|
device_t matching_dev = NULL;
|
|
struct resource *res;
|
|
int i;
|
|
vm_offset_t v;
|
|
|
|
/* There will always be at least one nexus. */
|
|
|
|
nexus_class = devclass_find("nexus");
|
|
devclass_get_devices(nexus_class, &nexus_devs, &nexus_count);
|
|
|
|
for (i = 0; i < nexus_count; i++) {
|
|
devp = nexus_devs[i];
|
|
matching_dev = ntoskrnl_finddev(devp, paddr, &res);
|
|
if (matching_dev)
|
|
break;
|
|
}
|
|
|
|
free(nexus_devs, M_TEMP);
|
|
|
|
if (matching_dev == NULL)
|
|
return (NULL);
|
|
|
|
v = (vm_offset_t)rman_get_virtual(res);
|
|
if (paddr > rman_get_start(res))
|
|
v += paddr - rman_get_start(res);
|
|
|
|
return ((void *)v);
|
|
}
|
|
|
|
void
|
|
MmUnmapIoSpace(vaddr, len)
|
|
void *vaddr;
|
|
size_t len;
|
|
{
|
|
}
|
|
|
|
|
|
static device_t
|
|
ntoskrnl_finddev(dev, paddr, res)
|
|
device_t dev;
|
|
uint64_t paddr;
|
|
struct resource **res;
|
|
{
|
|
device_t *children = NULL;
|
|
device_t matching_dev;
|
|
int childcnt;
|
|
struct resource *r;
|
|
struct resource_list *rl;
|
|
struct resource_list_entry *rle;
|
|
uint32_t flags;
|
|
int i;
|
|
|
|
/* We only want devices that have been successfully probed. */
|
|
|
|
if (device_is_alive(dev) == FALSE)
|
|
return (NULL);
|
|
|
|
rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev);
|
|
if (rl != NULL) {
|
|
STAILQ_FOREACH(rle, rl, link) {
|
|
r = rle->res;
|
|
|
|
if (r == NULL)
|
|
continue;
|
|
|
|
flags = rman_get_flags(r);
|
|
|
|
if (rle->type == SYS_RES_MEMORY &&
|
|
paddr >= rman_get_start(r) &&
|
|
paddr <= rman_get_end(r)) {
|
|
if (!(flags & RF_ACTIVE))
|
|
bus_activate_resource(dev,
|
|
SYS_RES_MEMORY, 0, r);
|
|
*res = r;
|
|
return (dev);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* If this device has children, do another
|
|
* level of recursion to inspect them.
|
|
*/
|
|
|
|
device_get_children(dev, &children, &childcnt);
|
|
|
|
for (i = 0; i < childcnt; i++) {
|
|
matching_dev = ntoskrnl_finddev(children[i], paddr, res);
|
|
if (matching_dev != NULL) {
|
|
free(children, M_TEMP);
|
|
return (matching_dev);
|
|
}
|
|
}
|
|
|
|
|
|
/* Won't somebody please think of the children! */
|
|
|
|
if (children != NULL)
|
|
free(children, M_TEMP);
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Workitems are unlike DPCs, in that they run in a user-mode thread
|
|
* context rather than at DISPATCH_LEVEL in kernel context. In our
|
|
* case we run them in kernel context anyway.
|
|
*/
|
|
static void
|
|
ntoskrnl_workitem_thread(arg)
|
|
void *arg;
|
|
{
|
|
kdpc_queue *kq;
|
|
list_entry *l;
|
|
io_workitem *iw;
|
|
uint8_t irql;
|
|
|
|
kq = arg;
|
|
|
|
InitializeListHead(&kq->kq_disp);
|
|
kq->kq_td = curthread;
|
|
kq->kq_exit = 0;
|
|
KeInitializeSpinLock(&kq->kq_lock);
|
|
KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
|
|
|
|
while (1) {
|
|
KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
|
|
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
|
|
if (kq->kq_exit) {
|
|
kq->kq_exit = 0;
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
break;
|
|
}
|
|
|
|
while (!IsListEmpty(&kq->kq_disp)) {
|
|
l = RemoveHeadList(&kq->kq_disp);
|
|
iw = CONTAINING_RECORD(l,
|
|
io_workitem, iw_listentry);
|
|
InitializeListHead((&iw->iw_listentry));
|
|
if (iw->iw_func == NULL)
|
|
continue;
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
MSCALL2(iw->iw_func, iw->iw_dobj, iw->iw_ctx);
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
}
|
|
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
}
|
|
|
|
kproc_exit(0);
|
|
return; /* notreached */
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_destroy_workitem_threads(void)
|
|
{
|
|
kdpc_queue *kq;
|
|
int i;
|
|
|
|
for (i = 0; i < WORKITEM_THREADS; i++) {
|
|
kq = wq_queues + i;
|
|
kq->kq_exit = 1;
|
|
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
|
|
while (kq->kq_exit)
|
|
tsleep(kq->kq_td->td_proc, PWAIT, "waitiw", hz/10);
|
|
}
|
|
}
|
|
|
|
io_workitem *
|
|
IoAllocateWorkItem(dobj)
|
|
device_object *dobj;
|
|
{
|
|
io_workitem *iw;
|
|
|
|
iw = uma_zalloc(iw_zone, M_NOWAIT);
|
|
if (iw == NULL)
|
|
return (NULL);
|
|
|
|
InitializeListHead(&iw->iw_listentry);
|
|
iw->iw_dobj = dobj;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
iw->iw_idx = wq_idx;
|
|
WORKIDX_INC(wq_idx);
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (iw);
|
|
}
|
|
|
|
void
|
|
IoFreeWorkItem(iw)
|
|
io_workitem *iw;
|
|
{
|
|
uma_zfree(iw_zone, iw);
|
|
}
|
|
|
|
void
|
|
IoQueueWorkItem(iw, iw_func, qtype, ctx)
|
|
io_workitem *iw;
|
|
io_workitem_func iw_func;
|
|
uint32_t qtype;
|
|
void *ctx;
|
|
{
|
|
kdpc_queue *kq;
|
|
list_entry *l;
|
|
io_workitem *cur;
|
|
uint8_t irql;
|
|
|
|
kq = wq_queues + iw->iw_idx;
|
|
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
|
|
/*
|
|
* Traverse the list and make sure this workitem hasn't
|
|
* already been inserted. Queuing the same workitem
|
|
* twice will hose the list but good.
|
|
*/
|
|
|
|
l = kq->kq_disp.nle_flink;
|
|
while (l != &kq->kq_disp) {
|
|
cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
|
|
if (cur == iw) {
|
|
/* Already queued -- do nothing. */
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
return;
|
|
}
|
|
l = l->nle_flink;
|
|
}
|
|
|
|
iw->iw_func = iw_func;
|
|
iw->iw_ctx = ctx;
|
|
|
|
InsertTailList((&kq->kq_disp), (&iw->iw_listentry));
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
|
|
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_workitem(dobj, arg)
|
|
device_object *dobj;
|
|
void *arg;
|
|
{
|
|
io_workitem *iw;
|
|
work_queue_item *w;
|
|
work_item_func f;
|
|
|
|
iw = arg;
|
|
w = (work_queue_item *)dobj;
|
|
f = (work_item_func)w->wqi_func;
|
|
uma_zfree(iw_zone, iw);
|
|
MSCALL2(f, w, w->wqi_ctx);
|
|
}
|
|
|
|
/*
|
|
* The ExQueueWorkItem() API is deprecated in Windows XP. Microsoft
|
|
* warns that it's unsafe and to use IoQueueWorkItem() instead. The
|
|
* problem with ExQueueWorkItem() is that it can't guard against
|
|
* the condition where a driver submits a job to the work queue and
|
|
* is then unloaded before the job is able to run. IoQueueWorkItem()
|
|
* acquires a reference to the device's device_object via the
|
|
* object manager and retains it until after the job has completed,
|
|
* which prevents the driver from being unloaded before the job
|
|
* runs. (We don't currently support this behavior, though hopefully
|
|
* that will change once the object manager API is fleshed out a bit.)
|
|
*
|
|
* Having said all that, the ExQueueWorkItem() API remains, because
|
|
* there are still other parts of Windows that use it, including
|
|
* NDIS itself: NdisScheduleWorkItem() calls ExQueueWorkItem().
|
|
* We fake up the ExQueueWorkItem() API on top of our implementation
|
|
* of IoQueueWorkItem(). Workitem thread #3 is reserved exclusively
|
|
* for ExQueueWorkItem() jobs, and we pass a pointer to the work
|
|
* queue item (provided by the caller) in to IoAllocateWorkItem()
|
|
* instead of the device_object. We need to save this pointer so
|
|
* we can apply a sanity check: as with the DPC queue and other
|
|
* workitem queues, we can't allow the same work queue item to
|
|
* be queued twice. If it's already pending, we silently return
|
|
*/
|
|
|
|
void
|
|
ExQueueWorkItem(w, qtype)
|
|
work_queue_item *w;
|
|
uint32_t qtype;
|
|
{
|
|
io_workitem *iw;
|
|
io_workitem_func iwf;
|
|
kdpc_queue *kq;
|
|
list_entry *l;
|
|
io_workitem *cur;
|
|
uint8_t irql;
|
|
|
|
|
|
/*
|
|
* We need to do a special sanity test to make sure
|
|
* the ExQueueWorkItem() API isn't used to queue
|
|
* the same workitem twice. Rather than checking the
|
|
* io_workitem pointer itself, we test the attached
|
|
* device object, which is really a pointer to the
|
|
* legacy work queue item structure.
|
|
*/
|
|
|
|
kq = wq_queues + WORKITEM_LEGACY_THREAD;
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
l = kq->kq_disp.nle_flink;
|
|
while (l != &kq->kq_disp) {
|
|
cur = CONTAINING_RECORD(l, io_workitem, iw_listentry);
|
|
if (cur->iw_dobj == (device_object *)w) {
|
|
/* Already queued -- do nothing. */
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
return;
|
|
}
|
|
l = l->nle_flink;
|
|
}
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
|
|
iw = IoAllocateWorkItem((device_object *)w);
|
|
if (iw == NULL)
|
|
return;
|
|
|
|
iw->iw_idx = WORKITEM_LEGACY_THREAD;
|
|
iwf = (io_workitem_func)ntoskrnl_findwrap((funcptr)ntoskrnl_workitem);
|
|
IoQueueWorkItem(iw, iwf, qtype, iw);
|
|
}
|
|
|
|
static void
|
|
RtlZeroMemory(dst, len)
|
|
void *dst;
|
|
size_t len;
|
|
{
|
|
bzero(dst, len);
|
|
}
|
|
|
|
static void
|
|
RtlCopyMemory(dst, src, len)
|
|
void *dst;
|
|
const void *src;
|
|
size_t len;
|
|
{
|
|
bcopy(src, dst, len);
|
|
}
|
|
|
|
static size_t
|
|
RtlCompareMemory(s1, s2, len)
|
|
const void *s1;
|
|
const void *s2;
|
|
size_t len;
|
|
{
|
|
size_t i, total = 0;
|
|
uint8_t *m1, *m2;
|
|
|
|
m1 = __DECONST(char *, s1);
|
|
m2 = __DECONST(char *, s2);
|
|
|
|
for (i = 0; i < len; i++) {
|
|
if (m1[i] == m2[i])
|
|
total++;
|
|
}
|
|
return (total);
|
|
}
|
|
|
|
void
|
|
RtlInitAnsiString(dst, src)
|
|
ansi_string *dst;
|
|
char *src;
|
|
{
|
|
ansi_string *a;
|
|
|
|
a = dst;
|
|
if (a == NULL)
|
|
return;
|
|
if (src == NULL) {
|
|
a->as_len = a->as_maxlen = 0;
|
|
a->as_buf = NULL;
|
|
} else {
|
|
a->as_buf = src;
|
|
a->as_len = a->as_maxlen = strlen(src);
|
|
}
|
|
}
|
|
|
|
void
|
|
RtlInitUnicodeString(dst, src)
|
|
unicode_string *dst;
|
|
uint16_t *src;
|
|
{
|
|
unicode_string *u;
|
|
int i;
|
|
|
|
u = dst;
|
|
if (u == NULL)
|
|
return;
|
|
if (src == NULL) {
|
|
u->us_len = u->us_maxlen = 0;
|
|
u->us_buf = NULL;
|
|
} else {
|
|
i = 0;
|
|
while(src[i] != 0)
|
|
i++;
|
|
u->us_buf = src;
|
|
u->us_len = u->us_maxlen = i * 2;
|
|
}
|
|
}
|
|
|
|
ndis_status
|
|
RtlUnicodeStringToInteger(ustr, base, val)
|
|
unicode_string *ustr;
|
|
uint32_t base;
|
|
uint32_t *val;
|
|
{
|
|
uint16_t *uchr;
|
|
int len, neg = 0;
|
|
char abuf[64];
|
|
char *astr;
|
|
|
|
uchr = ustr->us_buf;
|
|
len = ustr->us_len;
|
|
bzero(abuf, sizeof(abuf));
|
|
|
|
if ((char)((*uchr) & 0xFF) == '-') {
|
|
neg = 1;
|
|
uchr++;
|
|
len -= 2;
|
|
} else if ((char)((*uchr) & 0xFF) == '+') {
|
|
neg = 0;
|
|
uchr++;
|
|
len -= 2;
|
|
}
|
|
|
|
if (base == 0) {
|
|
if ((char)((*uchr) & 0xFF) == 'b') {
|
|
base = 2;
|
|
uchr++;
|
|
len -= 2;
|
|
} else if ((char)((*uchr) & 0xFF) == 'o') {
|
|
base = 8;
|
|
uchr++;
|
|
len -= 2;
|
|
} else if ((char)((*uchr) & 0xFF) == 'x') {
|
|
base = 16;
|
|
uchr++;
|
|
len -= 2;
|
|
} else
|
|
base = 10;
|
|
}
|
|
|
|
astr = abuf;
|
|
if (neg) {
|
|
strcpy(astr, "-");
|
|
astr++;
|
|
}
|
|
|
|
ntoskrnl_unicode_to_ascii(uchr, astr, len);
|
|
*val = strtoul(abuf, NULL, base);
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
void
|
|
RtlFreeUnicodeString(ustr)
|
|
unicode_string *ustr;
|
|
{
|
|
if (ustr->us_buf == NULL)
|
|
return;
|
|
ExFreePool(ustr->us_buf);
|
|
ustr->us_buf = NULL;
|
|
}
|
|
|
|
void
|
|
RtlFreeAnsiString(astr)
|
|
ansi_string *astr;
|
|
{
|
|
if (astr->as_buf == NULL)
|
|
return;
|
|
ExFreePool(astr->as_buf);
|
|
astr->as_buf = NULL;
|
|
}
|
|
|
|
static int
|
|
atoi(str)
|
|
const char *str;
|
|
{
|
|
return (int)strtol(str, (char **)NULL, 10);
|
|
}
|
|
|
|
static long
|
|
atol(str)
|
|
const char *str;
|
|
{
|
|
return strtol(str, (char **)NULL, 10);
|
|
}
|
|
|
|
static int
|
|
rand(void)
|
|
{
|
|
struct timeval tv;
|
|
|
|
microtime(&tv);
|
|
srandom(tv.tv_usec);
|
|
return ((int)random());
|
|
}
|
|
|
|
static void
|
|
srand(seed)
|
|
unsigned int seed;
|
|
{
|
|
srandom(seed);
|
|
}
|
|
|
|
static uint8_t
|
|
IoIsWdmVersionAvailable(uint8_t major, uint8_t minor)
|
|
{
|
|
if (major == WDM_MAJOR && minor == WDM_MINOR_WINXP)
|
|
return (TRUE);
|
|
return (FALSE);
|
|
}
|
|
|
|
static ndis_status
|
|
IoGetDeviceObjectPointer(name, reqaccess, fileobj, devobj)
|
|
unicode_string *name;
|
|
uint32_t reqaccess;
|
|
void *fileobj;
|
|
device_object *devobj;
|
|
{
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static ndis_status
|
|
IoGetDeviceProperty(devobj, regprop, buflen, prop, reslen)
|
|
device_object *devobj;
|
|
uint32_t regprop;
|
|
uint32_t buflen;
|
|
void *prop;
|
|
uint32_t *reslen;
|
|
{
|
|
driver_object *drv;
|
|
uint16_t **name;
|
|
|
|
drv = devobj->do_drvobj;
|
|
|
|
switch (regprop) {
|
|
case DEVPROP_DRIVER_KEYNAME:
|
|
name = prop;
|
|
*name = drv->dro_drivername.us_buf;
|
|
*reslen = drv->dro_drivername.us_len;
|
|
break;
|
|
default:
|
|
return (STATUS_INVALID_PARAMETER_2);
|
|
break;
|
|
}
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static void
|
|
KeInitializeMutex(kmutex, level)
|
|
kmutant *kmutex;
|
|
uint32_t level;
|
|
{
|
|
InitializeListHead((&kmutex->km_header.dh_waitlisthead));
|
|
kmutex->km_abandoned = FALSE;
|
|
kmutex->km_apcdisable = 1;
|
|
kmutex->km_header.dh_sigstate = 1;
|
|
kmutex->km_header.dh_type = DISP_TYPE_MUTANT;
|
|
kmutex->km_header.dh_size = sizeof(kmutant) / sizeof(uint32_t);
|
|
kmutex->km_ownerthread = NULL;
|
|
}
|
|
|
|
static uint32_t
|
|
KeReleaseMutex(kmutant *kmutex, uint8_t kwait)
|
|
{
|
|
uint32_t prevstate;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
prevstate = kmutex->km_header.dh_sigstate;
|
|
if (kmutex->km_ownerthread != curthread) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
return (STATUS_MUTANT_NOT_OWNED);
|
|
}
|
|
|
|
kmutex->km_header.dh_sigstate++;
|
|
kmutex->km_abandoned = FALSE;
|
|
|
|
if (kmutex->km_header.dh_sigstate == 1) {
|
|
kmutex->km_ownerthread = NULL;
|
|
ntoskrnl_waittest(&kmutex->km_header, IO_NO_INCREMENT);
|
|
}
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (prevstate);
|
|
}
|
|
|
|
static uint32_t
|
|
KeReadStateMutex(kmutex)
|
|
kmutant *kmutex;
|
|
{
|
|
return (kmutex->km_header.dh_sigstate);
|
|
}
|
|
|
|
void
|
|
KeInitializeEvent(nt_kevent *kevent, uint32_t type, uint8_t state)
|
|
{
|
|
InitializeListHead((&kevent->k_header.dh_waitlisthead));
|
|
kevent->k_header.dh_sigstate = state;
|
|
if (type == EVENT_TYPE_NOTIFY)
|
|
kevent->k_header.dh_type = DISP_TYPE_NOTIFICATION_EVENT;
|
|
else
|
|
kevent->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_EVENT;
|
|
kevent->k_header.dh_size = sizeof(nt_kevent) / sizeof(uint32_t);
|
|
}
|
|
|
|
uint32_t
|
|
KeResetEvent(kevent)
|
|
nt_kevent *kevent;
|
|
{
|
|
uint32_t prevstate;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
prevstate = kevent->k_header.dh_sigstate;
|
|
kevent->k_header.dh_sigstate = FALSE;
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (prevstate);
|
|
}
|
|
|
|
uint32_t
|
|
KeSetEvent(nt_kevent *kevent, uint32_t increment, uint8_t kwait)
|
|
{
|
|
uint32_t prevstate;
|
|
wait_block *w;
|
|
nt_dispatch_header *dh;
|
|
struct thread *td;
|
|
wb_ext *we;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
prevstate = kevent->k_header.dh_sigstate;
|
|
dh = &kevent->k_header;
|
|
|
|
if (IsListEmpty(&dh->dh_waitlisthead))
|
|
/*
|
|
* If there's nobody in the waitlist, just set
|
|
* the state to signalled.
|
|
*/
|
|
dh->dh_sigstate = 1;
|
|
else {
|
|
/*
|
|
* Get the first waiter. If this is a synchronization
|
|
* event, just wake up that one thread (don't bother
|
|
* setting the state to signalled since we're supposed
|
|
* to automatically clear synchronization events anyway).
|
|
*
|
|
* If it's a notification event, or the the first
|
|
* waiter is doing a WAITTYPE_ALL wait, go through
|
|
* the full wait satisfaction process.
|
|
*/
|
|
w = CONTAINING_RECORD(dh->dh_waitlisthead.nle_flink,
|
|
wait_block, wb_waitlist);
|
|
we = w->wb_ext;
|
|
td = we->we_td;
|
|
if (kevent->k_header.dh_type == DISP_TYPE_NOTIFICATION_EVENT ||
|
|
w->wb_waittype == WAITTYPE_ALL) {
|
|
if (prevstate == 0) {
|
|
dh->dh_sigstate = 1;
|
|
ntoskrnl_waittest(dh, increment);
|
|
}
|
|
} else {
|
|
w->wb_awakened |= TRUE;
|
|
cv_broadcastpri(&we->we_cv,
|
|
(w->wb_oldpri - (increment * 4)) > PRI_MIN_KERN ?
|
|
w->wb_oldpri - (increment * 4) : PRI_MIN_KERN);
|
|
}
|
|
}
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (prevstate);
|
|
}
|
|
|
|
void
|
|
KeClearEvent(kevent)
|
|
nt_kevent *kevent;
|
|
{
|
|
kevent->k_header.dh_sigstate = FALSE;
|
|
}
|
|
|
|
uint32_t
|
|
KeReadStateEvent(kevent)
|
|
nt_kevent *kevent;
|
|
{
|
|
return (kevent->k_header.dh_sigstate);
|
|
}
|
|
|
|
/*
|
|
* The object manager in Windows is responsible for managing
|
|
* references and access to various types of objects, including
|
|
* device_objects, events, threads, timers and so on. However,
|
|
* there's a difference in the way objects are handled in user
|
|
* mode versus kernel mode.
|
|
*
|
|
* In user mode (i.e. Win32 applications), all objects are
|
|
* managed by the object manager. For example, when you create
|
|
* a timer or event object, you actually end up with an
|
|
* object_header (for the object manager's bookkeeping
|
|
* purposes) and an object body (which contains the actual object
|
|
* structure, e.g. ktimer, kevent, etc...). This allows Windows
|
|
* to manage resource quotas and to enforce access restrictions
|
|
* on basically every kind of system object handled by the kernel.
|
|
*
|
|
* However, in kernel mode, you only end up using the object
|
|
* manager some of the time. For example, in a driver, you create
|
|
* a timer object by simply allocating the memory for a ktimer
|
|
* structure and initializing it with KeInitializeTimer(). Hence,
|
|
* the timer has no object_header and no reference counting or
|
|
* security/resource checks are done on it. The assumption in
|
|
* this case is that if you're running in kernel mode, you know
|
|
* what you're doing, and you're already at an elevated privilege
|
|
* anyway.
|
|
*
|
|
* There are some exceptions to this. The two most important ones
|
|
* for our purposes are device_objects and threads. We need to use
|
|
* the object manager to do reference counting on device_objects,
|
|
* and for threads, you can only get a pointer to a thread's
|
|
* dispatch header by using ObReferenceObjectByHandle() on the
|
|
* handle returned by PsCreateSystemThread().
|
|
*/
|
|
|
|
static ndis_status
|
|
ObReferenceObjectByHandle(ndis_handle handle, uint32_t reqaccess, void *otype,
|
|
uint8_t accessmode, void **object, void **handleinfo)
|
|
{
|
|
nt_objref *nr;
|
|
|
|
nr = malloc(sizeof(nt_objref), M_DEVBUF, M_NOWAIT|M_ZERO);
|
|
if (nr == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
|
|
InitializeListHead((&nr->no_dh.dh_waitlisthead));
|
|
nr->no_obj = handle;
|
|
nr->no_dh.dh_type = DISP_TYPE_THREAD;
|
|
nr->no_dh.dh_sigstate = 0;
|
|
nr->no_dh.dh_size = (uint8_t)(sizeof(struct thread) /
|
|
sizeof(uint32_t));
|
|
TAILQ_INSERT_TAIL(&ntoskrnl_reflist, nr, link);
|
|
*object = nr;
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static void
|
|
ObfDereferenceObject(object)
|
|
void *object;
|
|
{
|
|
nt_objref *nr;
|
|
|
|
nr = object;
|
|
TAILQ_REMOVE(&ntoskrnl_reflist, nr, link);
|
|
free(nr, M_DEVBUF);
|
|
}
|
|
|
|
static uint32_t
|
|
ZwClose(handle)
|
|
ndis_handle handle;
|
|
{
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static uint32_t
|
|
WmiQueryTraceInformation(traceclass, traceinfo, infolen, reqlen, buf)
|
|
uint32_t traceclass;
|
|
void *traceinfo;
|
|
uint32_t infolen;
|
|
uint32_t reqlen;
|
|
void *buf;
|
|
{
|
|
return (STATUS_NOT_FOUND);
|
|
}
|
|
|
|
static uint32_t
|
|
WmiTraceMessage(uint64_t loghandle, uint32_t messageflags,
|
|
void *guid, uint16_t messagenum, ...)
|
|
{
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static uint32_t
|
|
IoWMIRegistrationControl(dobj, action)
|
|
device_object *dobj;
|
|
uint32_t action;
|
|
{
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
/*
|
|
* This is here just in case the thread returns without calling
|
|
* PsTerminateSystemThread().
|
|
*/
|
|
static void
|
|
ntoskrnl_thrfunc(arg)
|
|
void *arg;
|
|
{
|
|
thread_context *thrctx;
|
|
uint32_t (*tfunc)(void *);
|
|
void *tctx;
|
|
uint32_t rval;
|
|
|
|
thrctx = arg;
|
|
tfunc = thrctx->tc_thrfunc;
|
|
tctx = thrctx->tc_thrctx;
|
|
free(thrctx, M_TEMP);
|
|
|
|
rval = MSCALL1(tfunc, tctx);
|
|
|
|
PsTerminateSystemThread(rval);
|
|
return; /* notreached */
|
|
}
|
|
|
|
static ndis_status
|
|
PsCreateSystemThread(handle, reqaccess, objattrs, phandle,
|
|
clientid, thrfunc, thrctx)
|
|
ndis_handle *handle;
|
|
uint32_t reqaccess;
|
|
void *objattrs;
|
|
ndis_handle phandle;
|
|
void *clientid;
|
|
void *thrfunc;
|
|
void *thrctx;
|
|
{
|
|
int error;
|
|
thread_context *tc;
|
|
struct proc *p;
|
|
|
|
tc = malloc(sizeof(thread_context), M_TEMP, M_NOWAIT);
|
|
if (tc == NULL)
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
|
|
tc->tc_thrctx = thrctx;
|
|
tc->tc_thrfunc = thrfunc;
|
|
|
|
error = kproc_create(ntoskrnl_thrfunc, tc, &p,
|
|
RFHIGHPID, NDIS_KSTACK_PAGES, "Windows Kthread %d", ntoskrnl_kth);
|
|
|
|
if (error) {
|
|
free(tc, M_TEMP);
|
|
return (STATUS_INSUFFICIENT_RESOURCES);
|
|
}
|
|
|
|
*handle = p;
|
|
ntoskrnl_kth++;
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
/*
|
|
* In Windows, the exit of a thread is an event that you're allowed
|
|
* to wait on, assuming you've obtained a reference to the thread using
|
|
* ObReferenceObjectByHandle(). Unfortunately, the only way we can
|
|
* simulate this behavior is to register each thread we create in a
|
|
* reference list, and if someone holds a reference to us, we poke
|
|
* them.
|
|
*/
|
|
static ndis_status
|
|
PsTerminateSystemThread(status)
|
|
ndis_status status;
|
|
{
|
|
struct nt_objref *nr;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
TAILQ_FOREACH(nr, &ntoskrnl_reflist, link) {
|
|
if (nr->no_obj != curthread->td_proc)
|
|
continue;
|
|
nr->no_dh.dh_sigstate = 1;
|
|
ntoskrnl_waittest(&nr->no_dh, IO_NO_INCREMENT);
|
|
break;
|
|
}
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
ntoskrnl_kth--;
|
|
|
|
kproc_exit(0);
|
|
return (0); /* notreached */
|
|
}
|
|
|
|
static uint32_t
|
|
DbgPrint(char *fmt, ...)
|
|
{
|
|
va_list ap;
|
|
|
|
if (bootverbose) {
|
|
va_start(ap, fmt);
|
|
vprintf(fmt, ap);
|
|
}
|
|
|
|
return (STATUS_SUCCESS);
|
|
}
|
|
|
|
static void
|
|
DbgBreakPoint(void)
|
|
{
|
|
|
|
kdb_enter(KDB_WHY_NDIS, "DbgBreakPoint(): breakpoint");
|
|
}
|
|
|
|
static void
|
|
KeBugCheckEx(code, param1, param2, param3, param4)
|
|
uint32_t code;
|
|
u_long param1;
|
|
u_long param2;
|
|
u_long param3;
|
|
u_long param4;
|
|
{
|
|
panic("KeBugCheckEx: STOP 0x%X", code);
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_timercall(arg)
|
|
void *arg;
|
|
{
|
|
ktimer *timer;
|
|
struct timeval tv;
|
|
kdpc *dpc;
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
timer = arg;
|
|
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_timer_fires++;
|
|
#endif
|
|
ntoskrnl_remove_timer(timer);
|
|
|
|
/*
|
|
* This should never happen, but complain
|
|
* if it does.
|
|
*/
|
|
|
|
if (timer->k_header.dh_inserted == FALSE) {
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
printf("NTOS: timer %p fired even though "
|
|
"it was canceled\n", timer);
|
|
return;
|
|
}
|
|
|
|
/* Mark the timer as no longer being on the timer queue. */
|
|
|
|
timer->k_header.dh_inserted = FALSE;
|
|
|
|
/* Now signal the object and satisfy any waits on it. */
|
|
|
|
timer->k_header.dh_sigstate = 1;
|
|
ntoskrnl_waittest(&timer->k_header, IO_NO_INCREMENT);
|
|
|
|
/*
|
|
* If this is a periodic timer, re-arm it
|
|
* so it will fire again. We do this before
|
|
* calling any deferred procedure calls because
|
|
* it's possible the DPC might cancel the timer,
|
|
* in which case it would be wrong for us to
|
|
* re-arm it again afterwards.
|
|
*/
|
|
|
|
if (timer->k_period) {
|
|
tv.tv_sec = 0;
|
|
tv.tv_usec = timer->k_period * 1000;
|
|
timer->k_header.dh_inserted = TRUE;
|
|
ntoskrnl_insert_timer(timer, tvtohz(&tv));
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_timer_reloads++;
|
|
#endif
|
|
}
|
|
|
|
dpc = timer->k_dpc;
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
/* If there's a DPC associated with the timer, queue it up. */
|
|
|
|
if (dpc != NULL)
|
|
KeInsertQueueDpc(dpc, NULL, NULL);
|
|
}
|
|
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
static int
|
|
sysctl_show_timers(SYSCTL_HANDLER_ARGS)
|
|
{
|
|
int ret;
|
|
|
|
ret = 0;
|
|
ntoskrnl_show_timers();
|
|
return (sysctl_handle_int(oidp, &ret, 0, req));
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_show_timers()
|
|
{
|
|
int i = 0;
|
|
list_entry *l;
|
|
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
l = ntoskrnl_calllist.nle_flink;
|
|
while(l != &ntoskrnl_calllist) {
|
|
i++;
|
|
l = l->nle_flink;
|
|
}
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
|
|
printf("\n");
|
|
printf("%d timers available (out of %d)\n", i, NTOSKRNL_TIMEOUTS);
|
|
printf("timer sets: %qu\n", ntoskrnl_timer_sets);
|
|
printf("timer reloads: %qu\n", ntoskrnl_timer_reloads);
|
|
printf("timer cancels: %qu\n", ntoskrnl_timer_cancels);
|
|
printf("timer fires: %qu\n", ntoskrnl_timer_fires);
|
|
printf("\n");
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* Must be called with dispatcher lock held.
|
|
*/
|
|
|
|
static void
|
|
ntoskrnl_insert_timer(timer, ticks)
|
|
ktimer *timer;
|
|
int ticks;
|
|
{
|
|
callout_entry *e;
|
|
list_entry *l;
|
|
struct callout *c;
|
|
|
|
/*
|
|
* Try and allocate a timer.
|
|
*/
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
if (IsListEmpty(&ntoskrnl_calllist)) {
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_show_timers();
|
|
#endif
|
|
panic("out of timers!");
|
|
}
|
|
l = RemoveHeadList(&ntoskrnl_calllist);
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
|
|
e = CONTAINING_RECORD(l, callout_entry, ce_list);
|
|
c = &e->ce_callout;
|
|
|
|
timer->k_callout = c;
|
|
|
|
callout_init(c, CALLOUT_MPSAFE);
|
|
callout_reset(c, ticks, ntoskrnl_timercall, timer);
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_remove_timer(timer)
|
|
ktimer *timer;
|
|
{
|
|
callout_entry *e;
|
|
|
|
e = (callout_entry *)timer->k_callout;
|
|
callout_stop(timer->k_callout);
|
|
|
|
mtx_lock_spin(&ntoskrnl_calllock);
|
|
InsertHeadList((&ntoskrnl_calllist), (&e->ce_list));
|
|
mtx_unlock_spin(&ntoskrnl_calllock);
|
|
}
|
|
|
|
void
|
|
KeInitializeTimer(timer)
|
|
ktimer *timer;
|
|
{
|
|
if (timer == NULL)
|
|
return;
|
|
|
|
KeInitializeTimerEx(timer, EVENT_TYPE_NOTIFY);
|
|
}
|
|
|
|
void
|
|
KeInitializeTimerEx(timer, type)
|
|
ktimer *timer;
|
|
uint32_t type;
|
|
{
|
|
if (timer == NULL)
|
|
return;
|
|
|
|
bzero((char *)timer, sizeof(ktimer));
|
|
InitializeListHead((&timer->k_header.dh_waitlisthead));
|
|
timer->k_header.dh_sigstate = FALSE;
|
|
timer->k_header.dh_inserted = FALSE;
|
|
if (type == EVENT_TYPE_NOTIFY)
|
|
timer->k_header.dh_type = DISP_TYPE_NOTIFICATION_TIMER;
|
|
else
|
|
timer->k_header.dh_type = DISP_TYPE_SYNCHRONIZATION_TIMER;
|
|
timer->k_header.dh_size = sizeof(ktimer) / sizeof(uint32_t);
|
|
}
|
|
|
|
/*
|
|
* DPC subsystem. A Windows Defered Procedure Call has the following
|
|
* properties:
|
|
* - It runs at DISPATCH_LEVEL.
|
|
* - It can have one of 3 importance values that control when it
|
|
* runs relative to other DPCs in the queue.
|
|
* - On SMP systems, it can be set to run on a specific processor.
|
|
* In order to satisfy the last property, we create a DPC thread for
|
|
* each CPU in the system and bind it to that CPU. Each thread
|
|
* maintains three queues with different importance levels, which
|
|
* will be processed in order from lowest to highest.
|
|
*
|
|
* In Windows, interrupt handlers run as DPCs. (Not to be confused
|
|
* with ISRs, which run in interrupt context and can preempt DPCs.)
|
|
* ISRs are given the highest importance so that they'll take
|
|
* precedence over timers and other things.
|
|
*/
|
|
|
|
static void
|
|
ntoskrnl_dpc_thread(arg)
|
|
void *arg;
|
|
{
|
|
kdpc_queue *kq;
|
|
kdpc *d;
|
|
list_entry *l;
|
|
uint8_t irql;
|
|
|
|
kq = arg;
|
|
|
|
InitializeListHead(&kq->kq_disp);
|
|
kq->kq_td = curthread;
|
|
kq->kq_exit = 0;
|
|
kq->kq_running = FALSE;
|
|
KeInitializeSpinLock(&kq->kq_lock);
|
|
KeInitializeEvent(&kq->kq_proc, EVENT_TYPE_SYNC, FALSE);
|
|
KeInitializeEvent(&kq->kq_done, EVENT_TYPE_SYNC, FALSE);
|
|
|
|
/*
|
|
* Elevate our priority. DPCs are used to run interrupt
|
|
* handlers, and they should trigger as soon as possible
|
|
* once scheduled by an ISR.
|
|
*/
|
|
|
|
thread_lock(curthread);
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
sched_bind(curthread, kq->kq_cpu);
|
|
#endif
|
|
sched_prio(curthread, PRI_MIN_KERN);
|
|
thread_unlock(curthread);
|
|
|
|
while (1) {
|
|
KeWaitForSingleObject(&kq->kq_proc, 0, 0, TRUE, NULL);
|
|
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
|
|
if (kq->kq_exit) {
|
|
kq->kq_exit = 0;
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
break;
|
|
}
|
|
|
|
kq->kq_running = TRUE;
|
|
|
|
while (!IsListEmpty(&kq->kq_disp)) {
|
|
l = RemoveHeadList((&kq->kq_disp));
|
|
d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
|
|
InitializeListHead((&d->k_dpclistentry));
|
|
KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
|
|
MSCALL4(d->k_deferedfunc, d, d->k_deferredctx,
|
|
d->k_sysarg1, d->k_sysarg2);
|
|
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
|
|
}
|
|
|
|
kq->kq_running = FALSE;
|
|
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
|
|
KeSetEvent(&kq->kq_done, IO_NO_INCREMENT, FALSE);
|
|
}
|
|
|
|
kproc_exit(0);
|
|
return; /* notreached */
|
|
}
|
|
|
|
static void
|
|
ntoskrnl_destroy_dpc_threads(void)
|
|
{
|
|
kdpc_queue *kq;
|
|
kdpc dpc;
|
|
int i;
|
|
|
|
kq = kq_queues;
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
for (i = 0; i < mp_ncpus; i++) {
|
|
#else
|
|
for (i = 0; i < 1; i++) {
|
|
#endif
|
|
kq += i;
|
|
|
|
kq->kq_exit = 1;
|
|
KeInitializeDpc(&dpc, NULL, NULL);
|
|
KeSetTargetProcessorDpc(&dpc, i);
|
|
KeInsertQueueDpc(&dpc, NULL, NULL);
|
|
while (kq->kq_exit)
|
|
tsleep(kq->kq_td->td_proc, PWAIT, "dpcw", hz/10);
|
|
}
|
|
}
|
|
|
|
static uint8_t
|
|
ntoskrnl_insert_dpc(head, dpc)
|
|
list_entry *head;
|
|
kdpc *dpc;
|
|
{
|
|
list_entry *l;
|
|
kdpc *d;
|
|
|
|
l = head->nle_flink;
|
|
while (l != head) {
|
|
d = CONTAINING_RECORD(l, kdpc, k_dpclistentry);
|
|
if (d == dpc)
|
|
return (FALSE);
|
|
l = l->nle_flink;
|
|
}
|
|
|
|
if (dpc->k_importance == KDPC_IMPORTANCE_LOW)
|
|
InsertTailList((head), (&dpc->k_dpclistentry));
|
|
else
|
|
InsertHeadList((head), (&dpc->k_dpclistentry));
|
|
|
|
return (TRUE);
|
|
}
|
|
|
|
void
|
|
KeInitializeDpc(dpc, dpcfunc, dpcctx)
|
|
kdpc *dpc;
|
|
void *dpcfunc;
|
|
void *dpcctx;
|
|
{
|
|
|
|
if (dpc == NULL)
|
|
return;
|
|
|
|
dpc->k_deferedfunc = dpcfunc;
|
|
dpc->k_deferredctx = dpcctx;
|
|
dpc->k_num = KDPC_CPU_DEFAULT;
|
|
dpc->k_importance = KDPC_IMPORTANCE_MEDIUM;
|
|
InitializeListHead((&dpc->k_dpclistentry));
|
|
}
|
|
|
|
uint8_t
|
|
KeInsertQueueDpc(dpc, sysarg1, sysarg2)
|
|
kdpc *dpc;
|
|
void *sysarg1;
|
|
void *sysarg2;
|
|
{
|
|
kdpc_queue *kq;
|
|
uint8_t r;
|
|
uint8_t irql;
|
|
|
|
if (dpc == NULL)
|
|
return (FALSE);
|
|
|
|
kq = kq_queues;
|
|
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
KeRaiseIrql(DISPATCH_LEVEL, &irql);
|
|
|
|
/*
|
|
* By default, the DPC is queued to run on the same CPU
|
|
* that scheduled it.
|
|
*/
|
|
|
|
if (dpc->k_num == KDPC_CPU_DEFAULT)
|
|
kq += curthread->td_oncpu;
|
|
else
|
|
kq += dpc->k_num;
|
|
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
|
|
#else
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
#endif
|
|
|
|
r = ntoskrnl_insert_dpc(&kq->kq_disp, dpc);
|
|
if (r == TRUE) {
|
|
dpc->k_sysarg1 = sysarg1;
|
|
dpc->k_sysarg2 = sysarg2;
|
|
}
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
|
|
if (r == FALSE)
|
|
return (r);
|
|
|
|
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
|
|
|
|
return (r);
|
|
}
|
|
|
|
uint8_t
|
|
KeRemoveQueueDpc(dpc)
|
|
kdpc *dpc;
|
|
{
|
|
kdpc_queue *kq;
|
|
uint8_t irql;
|
|
|
|
if (dpc == NULL)
|
|
return (FALSE);
|
|
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
KeRaiseIrql(DISPATCH_LEVEL, &irql);
|
|
|
|
kq = kq_queues + dpc->k_num;
|
|
|
|
KeAcquireSpinLockAtDpcLevel(&kq->kq_lock);
|
|
#else
|
|
kq = kq_queues;
|
|
KeAcquireSpinLock(&kq->kq_lock, &irql);
|
|
#endif
|
|
|
|
if (dpc->k_dpclistentry.nle_flink == &dpc->k_dpclistentry) {
|
|
KeReleaseSpinLockFromDpcLevel(&kq->kq_lock);
|
|
KeLowerIrql(irql);
|
|
return (FALSE);
|
|
}
|
|
|
|
RemoveEntryList((&dpc->k_dpclistentry));
|
|
InitializeListHead((&dpc->k_dpclistentry));
|
|
|
|
KeReleaseSpinLock(&kq->kq_lock, irql);
|
|
|
|
return (TRUE);
|
|
}
|
|
|
|
void
|
|
KeSetImportanceDpc(dpc, imp)
|
|
kdpc *dpc;
|
|
uint32_t imp;
|
|
{
|
|
if (imp != KDPC_IMPORTANCE_LOW &&
|
|
imp != KDPC_IMPORTANCE_MEDIUM &&
|
|
imp != KDPC_IMPORTANCE_HIGH)
|
|
return;
|
|
|
|
dpc->k_importance = (uint8_t)imp;
|
|
}
|
|
|
|
void
|
|
KeSetTargetProcessorDpc(kdpc *dpc, uint8_t cpu)
|
|
{
|
|
if (cpu > mp_ncpus)
|
|
return;
|
|
|
|
dpc->k_num = cpu;
|
|
}
|
|
|
|
void
|
|
KeFlushQueuedDpcs(void)
|
|
{
|
|
kdpc_queue *kq;
|
|
int i;
|
|
|
|
/*
|
|
* Poke each DPC queue and wait
|
|
* for them to drain.
|
|
*/
|
|
|
|
#ifdef NTOSKRNL_MULTIPLE_DPCS
|
|
for (i = 0; i < mp_ncpus; i++) {
|
|
#else
|
|
for (i = 0; i < 1; i++) {
|
|
#endif
|
|
kq = kq_queues + i;
|
|
KeSetEvent(&kq->kq_proc, IO_NO_INCREMENT, FALSE);
|
|
KeWaitForSingleObject(&kq->kq_done, 0, 0, TRUE, NULL);
|
|
}
|
|
}
|
|
|
|
uint32_t
|
|
KeGetCurrentProcessorNumber(void)
|
|
{
|
|
return ((uint32_t)curthread->td_oncpu);
|
|
}
|
|
|
|
uint8_t
|
|
KeSetTimerEx(timer, duetime, period, dpc)
|
|
ktimer *timer;
|
|
int64_t duetime;
|
|
uint32_t period;
|
|
kdpc *dpc;
|
|
{
|
|
struct timeval tv;
|
|
uint64_t curtime;
|
|
uint8_t pending;
|
|
|
|
if (timer == NULL)
|
|
return (FALSE);
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
if (timer->k_header.dh_inserted == TRUE) {
|
|
ntoskrnl_remove_timer(timer);
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_timer_cancels++;
|
|
#endif
|
|
timer->k_header.dh_inserted = FALSE;
|
|
pending = TRUE;
|
|
} else
|
|
pending = FALSE;
|
|
|
|
timer->k_duetime = duetime;
|
|
timer->k_period = period;
|
|
timer->k_header.dh_sigstate = FALSE;
|
|
timer->k_dpc = dpc;
|
|
|
|
if (duetime < 0) {
|
|
tv.tv_sec = - (duetime) / 10000000;
|
|
tv.tv_usec = (- (duetime) / 10) -
|
|
(tv.tv_sec * 1000000);
|
|
} else {
|
|
ntoskrnl_time(&curtime);
|
|
if (duetime < curtime)
|
|
tv.tv_sec = tv.tv_usec = 0;
|
|
else {
|
|
tv.tv_sec = ((duetime) - curtime) / 10000000;
|
|
tv.tv_usec = ((duetime) - curtime) / 10 -
|
|
(tv.tv_sec * 1000000);
|
|
}
|
|
}
|
|
|
|
timer->k_header.dh_inserted = TRUE;
|
|
ntoskrnl_insert_timer(timer, tvtohz(&tv));
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_timer_sets++;
|
|
#endif
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (pending);
|
|
}
|
|
|
|
uint8_t
|
|
KeSetTimer(timer, duetime, dpc)
|
|
ktimer *timer;
|
|
int64_t duetime;
|
|
kdpc *dpc;
|
|
{
|
|
return (KeSetTimerEx(timer, duetime, 0, dpc));
|
|
}
|
|
|
|
/*
|
|
* The Windows DDK documentation seems to say that cancelling
|
|
* a timer that has a DPC will result in the DPC also being
|
|
* cancelled, but this isn't really the case.
|
|
*/
|
|
|
|
uint8_t
|
|
KeCancelTimer(timer)
|
|
ktimer *timer;
|
|
{
|
|
uint8_t pending;
|
|
|
|
if (timer == NULL)
|
|
return (FALSE);
|
|
|
|
mtx_lock(&ntoskrnl_dispatchlock);
|
|
|
|
pending = timer->k_header.dh_inserted;
|
|
|
|
if (timer->k_header.dh_inserted == TRUE) {
|
|
timer->k_header.dh_inserted = FALSE;
|
|
ntoskrnl_remove_timer(timer);
|
|
#ifdef NTOSKRNL_DEBUG_TIMERS
|
|
ntoskrnl_timer_cancels++;
|
|
#endif
|
|
}
|
|
|
|
mtx_unlock(&ntoskrnl_dispatchlock);
|
|
|
|
return (pending);
|
|
}
|
|
|
|
uint8_t
|
|
KeReadStateTimer(timer)
|
|
ktimer *timer;
|
|
{
|
|
return (timer->k_header.dh_sigstate);
|
|
}
|
|
|
|
static int32_t
|
|
KeDelayExecutionThread(uint8_t wait_mode, uint8_t alertable, int64_t *interval)
|
|
{
|
|
ktimer timer;
|
|
|
|
if (wait_mode != 0)
|
|
panic("invalid wait_mode %d", wait_mode);
|
|
|
|
KeInitializeTimer(&timer);
|
|
KeSetTimer(&timer, *interval, NULL);
|
|
KeWaitForSingleObject(&timer, 0, 0, alertable, NULL);
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
static uint64_t
|
|
KeQueryInterruptTime(void)
|
|
{
|
|
int ticks;
|
|
struct timeval tv;
|
|
|
|
getmicrouptime(&tv);
|
|
|
|
ticks = tvtohz(&tv);
|
|
|
|
return ticks * ((10000000 + hz - 1) / hz);
|
|
}
|
|
|
|
static struct thread *
|
|
KeGetCurrentThread(void)
|
|
{
|
|
|
|
return curthread;
|
|
}
|
|
|
|
static int32_t
|
|
KeSetPriorityThread(td, pri)
|
|
struct thread *td;
|
|
int32_t pri;
|
|
{
|
|
int32_t old;
|
|
|
|
if (td == NULL)
|
|
return LOW_REALTIME_PRIORITY;
|
|
|
|
if (td->td_priority <= PRI_MIN_KERN)
|
|
old = HIGH_PRIORITY;
|
|
else if (td->td_priority >= PRI_MAX_KERN)
|
|
old = LOW_PRIORITY;
|
|
else
|
|
old = LOW_REALTIME_PRIORITY;
|
|
|
|
thread_lock(td);
|
|
if (pri == HIGH_PRIORITY)
|
|
sched_prio(td, PRI_MIN_KERN);
|
|
if (pri == LOW_REALTIME_PRIORITY)
|
|
sched_prio(td, PRI_MIN_KERN + (PRI_MAX_KERN - PRI_MIN_KERN) / 2);
|
|
if (pri == LOW_PRIORITY)
|
|
sched_prio(td, PRI_MAX_KERN);
|
|
thread_unlock(td);
|
|
|
|
return old;
|
|
}
|
|
|
|
static void
|
|
dummy()
|
|
{
|
|
printf("ntoskrnl dummy called...\n");
|
|
}
|
|
|
|
|
|
image_patch_table ntoskrnl_functbl[] = {
|
|
IMPORT_SFUNC(RtlZeroMemory, 2),
|
|
IMPORT_SFUNC(RtlCopyMemory, 3),
|
|
IMPORT_SFUNC(RtlCompareMemory, 3),
|
|
IMPORT_SFUNC(RtlEqualUnicodeString, 3),
|
|
IMPORT_SFUNC(RtlCopyUnicodeString, 2),
|
|
IMPORT_SFUNC(RtlUnicodeStringToAnsiString, 3),
|
|
IMPORT_SFUNC(RtlAnsiStringToUnicodeString, 3),
|
|
IMPORT_SFUNC(RtlInitAnsiString, 2),
|
|
IMPORT_SFUNC_MAP(RtlInitString, RtlInitAnsiString, 2),
|
|
IMPORT_SFUNC(RtlInitUnicodeString, 2),
|
|
IMPORT_SFUNC(RtlFreeAnsiString, 1),
|
|
IMPORT_SFUNC(RtlFreeUnicodeString, 1),
|
|
IMPORT_SFUNC(RtlUnicodeStringToInteger, 3),
|
|
IMPORT_CFUNC(sprintf, 0),
|
|
IMPORT_CFUNC(vsprintf, 0),
|
|
IMPORT_CFUNC_MAP(_snprintf, snprintf, 0),
|
|
IMPORT_CFUNC_MAP(_vsnprintf, vsnprintf, 0),
|
|
IMPORT_CFUNC(DbgPrint, 0),
|
|
IMPORT_SFUNC(DbgBreakPoint, 0),
|
|
IMPORT_SFUNC(KeBugCheckEx, 5),
|
|
IMPORT_CFUNC(strncmp, 0),
|
|
IMPORT_CFUNC(strcmp, 0),
|
|
IMPORT_CFUNC_MAP(stricmp, strcasecmp, 0),
|
|
IMPORT_CFUNC(strncpy, 0),
|
|
IMPORT_CFUNC(strcpy, 0),
|
|
IMPORT_CFUNC(strlen, 0),
|
|
IMPORT_CFUNC_MAP(toupper, ntoskrnl_toupper, 0),
|
|
IMPORT_CFUNC_MAP(tolower, ntoskrnl_tolower, 0),
|
|
IMPORT_CFUNC_MAP(strstr, ntoskrnl_strstr, 0),
|
|
IMPORT_CFUNC_MAP(strncat, ntoskrnl_strncat, 0),
|
|
IMPORT_CFUNC_MAP(strchr, index, 0),
|
|
IMPORT_CFUNC_MAP(strrchr, rindex, 0),
|
|
IMPORT_CFUNC(memcpy, 0),
|
|
IMPORT_CFUNC_MAP(memmove, ntoskrnl_memmove, 0),
|
|
IMPORT_CFUNC_MAP(memset, ntoskrnl_memset, 0),
|
|
IMPORT_CFUNC_MAP(memchr, ntoskrnl_memchr, 0),
|
|
IMPORT_SFUNC(IoAllocateDriverObjectExtension, 4),
|
|
IMPORT_SFUNC(IoGetDriverObjectExtension, 2),
|
|
IMPORT_FFUNC(IofCallDriver, 2),
|
|
IMPORT_FFUNC(IofCompleteRequest, 2),
|
|
IMPORT_SFUNC(IoAcquireCancelSpinLock, 1),
|
|
IMPORT_SFUNC(IoReleaseCancelSpinLock, 1),
|
|
IMPORT_SFUNC(IoCancelIrp, 1),
|
|
IMPORT_SFUNC(IoConnectInterrupt, 11),
|
|
IMPORT_SFUNC(IoDisconnectInterrupt, 1),
|
|
IMPORT_SFUNC(IoCreateDevice, 7),
|
|
IMPORT_SFUNC(IoDeleteDevice, 1),
|
|
IMPORT_SFUNC(IoGetAttachedDevice, 1),
|
|
IMPORT_SFUNC(IoAttachDeviceToDeviceStack, 2),
|
|
IMPORT_SFUNC(IoDetachDevice, 1),
|
|
IMPORT_SFUNC(IoBuildSynchronousFsdRequest, 7),
|
|
IMPORT_SFUNC(IoBuildAsynchronousFsdRequest, 6),
|
|
IMPORT_SFUNC(IoBuildDeviceIoControlRequest, 9),
|
|
IMPORT_SFUNC(IoAllocateIrp, 2),
|
|
IMPORT_SFUNC(IoReuseIrp, 2),
|
|
IMPORT_SFUNC(IoMakeAssociatedIrp, 2),
|
|
IMPORT_SFUNC(IoFreeIrp, 1),
|
|
IMPORT_SFUNC(IoInitializeIrp, 3),
|
|
IMPORT_SFUNC(KeAcquireInterruptSpinLock, 1),
|
|
IMPORT_SFUNC(KeReleaseInterruptSpinLock, 2),
|
|
IMPORT_SFUNC(KeSynchronizeExecution, 3),
|
|
IMPORT_SFUNC(KeWaitForSingleObject, 5),
|
|
IMPORT_SFUNC(KeWaitForMultipleObjects, 8),
|
|
IMPORT_SFUNC(_allmul, 4),
|
|
IMPORT_SFUNC(_alldiv, 4),
|
|
IMPORT_SFUNC(_allrem, 4),
|
|
IMPORT_RFUNC(_allshr, 0),
|
|
IMPORT_RFUNC(_allshl, 0),
|
|
IMPORT_SFUNC(_aullmul, 4),
|
|
IMPORT_SFUNC(_aulldiv, 4),
|
|
IMPORT_SFUNC(_aullrem, 4),
|
|
IMPORT_RFUNC(_aullshr, 0),
|
|
IMPORT_RFUNC(_aullshl, 0),
|
|
IMPORT_CFUNC(atoi, 0),
|
|
IMPORT_CFUNC(atol, 0),
|
|
IMPORT_CFUNC(rand, 0),
|
|
IMPORT_CFUNC(srand, 0),
|
|
IMPORT_SFUNC(WRITE_REGISTER_USHORT, 2),
|
|
IMPORT_SFUNC(READ_REGISTER_USHORT, 1),
|
|
IMPORT_SFUNC(WRITE_REGISTER_ULONG, 2),
|
|
IMPORT_SFUNC(READ_REGISTER_ULONG, 1),
|
|
IMPORT_SFUNC(READ_REGISTER_UCHAR, 1),
|
|
IMPORT_SFUNC(WRITE_REGISTER_UCHAR, 2),
|
|
IMPORT_SFUNC(ExInitializePagedLookasideList, 7),
|
|
IMPORT_SFUNC(ExDeletePagedLookasideList, 1),
|
|
IMPORT_SFUNC(ExInitializeNPagedLookasideList, 7),
|
|
IMPORT_SFUNC(ExDeleteNPagedLookasideList, 1),
|
|
IMPORT_FFUNC(InterlockedPopEntrySList, 1),
|
|
IMPORT_FFUNC(InterlockedPushEntrySList, 2),
|
|
IMPORT_SFUNC(ExQueryDepthSList, 1),
|
|
IMPORT_FFUNC_MAP(ExpInterlockedPopEntrySList,
|
|
InterlockedPopEntrySList, 1),
|
|
IMPORT_FFUNC_MAP(ExpInterlockedPushEntrySList,
|
|
InterlockedPushEntrySList, 2),
|
|
IMPORT_FFUNC(ExInterlockedPopEntrySList, 2),
|
|
IMPORT_FFUNC(ExInterlockedPushEntrySList, 3),
|
|
IMPORT_SFUNC(ExAllocatePoolWithTag, 3),
|
|
IMPORT_SFUNC(ExFreePool, 1),
|
|
#ifdef __i386__
|
|
IMPORT_FFUNC(KefAcquireSpinLockAtDpcLevel, 1),
|
|
IMPORT_FFUNC(KefReleaseSpinLockFromDpcLevel,1),
|
|
IMPORT_FFUNC(KeAcquireSpinLockRaiseToDpc, 1),
|
|
#else
|
|
/*
|
|
* For AMD64, we can get away with just mapping
|
|
* KeAcquireSpinLockRaiseToDpc() directly to KfAcquireSpinLock()
|
|
* because the calling conventions end up being the same.
|
|
* On i386, we have to be careful because KfAcquireSpinLock()
|
|
* is _fastcall but KeAcquireSpinLockRaiseToDpc() isn't.
|
|
*/
|
|
IMPORT_SFUNC(KeAcquireSpinLockAtDpcLevel, 1),
|
|
IMPORT_SFUNC(KeReleaseSpinLockFromDpcLevel, 1),
|
|
IMPORT_SFUNC_MAP(KeAcquireSpinLockRaiseToDpc, KfAcquireSpinLock, 1),
|
|
#endif
|
|
IMPORT_SFUNC_MAP(KeReleaseSpinLock, KfReleaseSpinLock, 1),
|
|
IMPORT_FFUNC(InterlockedIncrement, 1),
|
|
IMPORT_FFUNC(InterlockedDecrement, 1),
|
|
IMPORT_FFUNC(InterlockedExchange, 2),
|
|
IMPORT_FFUNC(ExInterlockedAddLargeStatistic, 2),
|
|
IMPORT_SFUNC(IoAllocateMdl, 5),
|
|
IMPORT_SFUNC(IoFreeMdl, 1),
|
|
IMPORT_SFUNC(MmAllocateContiguousMemory, 2 + 1),
|
|
IMPORT_SFUNC(MmAllocateContiguousMemorySpecifyCache, 5 + 3),
|
|
IMPORT_SFUNC(MmFreeContiguousMemory, 1),
|
|
IMPORT_SFUNC(MmFreeContiguousMemorySpecifyCache, 3),
|
|
IMPORT_SFUNC(MmSizeOfMdl, 1),
|
|
IMPORT_SFUNC(MmMapLockedPages, 2),
|
|
IMPORT_SFUNC(MmMapLockedPagesSpecifyCache, 6),
|
|
IMPORT_SFUNC(MmUnmapLockedPages, 2),
|
|
IMPORT_SFUNC(MmBuildMdlForNonPagedPool, 1),
|
|
IMPORT_SFUNC(MmGetPhysicalAddress, 1),
|
|
IMPORT_SFUNC(MmIsAddressValid, 1),
|
|
IMPORT_SFUNC(MmMapIoSpace, 3 + 1),
|
|
IMPORT_SFUNC(MmUnmapIoSpace, 2),
|
|
IMPORT_SFUNC(KeInitializeSpinLock, 1),
|
|
IMPORT_SFUNC(IoIsWdmVersionAvailable, 2),
|
|
IMPORT_SFUNC(IoGetDeviceObjectPointer, 4),
|
|
IMPORT_SFUNC(IoGetDeviceProperty, 5),
|
|
IMPORT_SFUNC(IoAllocateWorkItem, 1),
|
|
IMPORT_SFUNC(IoFreeWorkItem, 1),
|
|
IMPORT_SFUNC(IoQueueWorkItem, 4),
|
|
IMPORT_SFUNC(ExQueueWorkItem, 2),
|
|
IMPORT_SFUNC(ntoskrnl_workitem, 2),
|
|
IMPORT_SFUNC(KeInitializeMutex, 2),
|
|
IMPORT_SFUNC(KeReleaseMutex, 2),
|
|
IMPORT_SFUNC(KeReadStateMutex, 1),
|
|
IMPORT_SFUNC(KeInitializeEvent, 3),
|
|
IMPORT_SFUNC(KeSetEvent, 3),
|
|
IMPORT_SFUNC(KeResetEvent, 1),
|
|
IMPORT_SFUNC(KeClearEvent, 1),
|
|
IMPORT_SFUNC(KeReadStateEvent, 1),
|
|
IMPORT_SFUNC(KeInitializeTimer, 1),
|
|
IMPORT_SFUNC(KeInitializeTimerEx, 2),
|
|
IMPORT_SFUNC(KeSetTimer, 3),
|
|
IMPORT_SFUNC(KeSetTimerEx, 4),
|
|
IMPORT_SFUNC(KeCancelTimer, 1),
|
|
IMPORT_SFUNC(KeReadStateTimer, 1),
|
|
IMPORT_SFUNC(KeInitializeDpc, 3),
|
|
IMPORT_SFUNC(KeInsertQueueDpc, 3),
|
|
IMPORT_SFUNC(KeRemoveQueueDpc, 1),
|
|
IMPORT_SFUNC(KeSetImportanceDpc, 2),
|
|
IMPORT_SFUNC(KeSetTargetProcessorDpc, 2),
|
|
IMPORT_SFUNC(KeFlushQueuedDpcs, 0),
|
|
IMPORT_SFUNC(KeGetCurrentProcessorNumber, 1),
|
|
IMPORT_SFUNC(ObReferenceObjectByHandle, 6),
|
|
IMPORT_FFUNC(ObfDereferenceObject, 1),
|
|
IMPORT_SFUNC(ZwClose, 1),
|
|
IMPORT_SFUNC(PsCreateSystemThread, 7),
|
|
IMPORT_SFUNC(PsTerminateSystemThread, 1),
|
|
IMPORT_SFUNC(IoWMIRegistrationControl, 2),
|
|
IMPORT_SFUNC(WmiQueryTraceInformation, 5),
|
|
IMPORT_CFUNC(WmiTraceMessage, 0),
|
|
IMPORT_SFUNC(KeQuerySystemTime, 1),
|
|
IMPORT_CFUNC(KeTickCount, 0),
|
|
IMPORT_SFUNC(KeDelayExecutionThread, 3),
|
|
IMPORT_SFUNC(KeQueryInterruptTime, 0),
|
|
IMPORT_SFUNC(KeGetCurrentThread, 0),
|
|
IMPORT_SFUNC(KeSetPriorityThread, 2),
|
|
|
|
/*
|
|
* This last entry is a catch-all for any function we haven't
|
|
* implemented yet. The PE import list patching routine will
|
|
* use it for any function that doesn't have an explicit match
|
|
* in this table.
|
|
*/
|
|
|
|
{ NULL, (FUNC)dummy, NULL, 0, WINDRV_WRAP_STDCALL },
|
|
|
|
/* End of list. */
|
|
|
|
{ NULL, NULL, NULL }
|
|
};
|