Allow consumer to customize physical pager.
Add support for user-supplied callbacks into phys pager operations, providing custom getpages(), haspage(), and populate() methods implementations. Pager stores user data ptr/val in the object to provide context. Add phys_pager_allocate() helper that takes user ops table as one of the arguments. Current code for these methods is moved to the 'default' ops table, assigned automatically when vm_pager_alloc() is used. Reviewed by: markj Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D24652
This commit is contained in:
parent
6957645145
commit
a720b31c2a
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=365488
@ -51,6 +51,20 @@ static struct pagerlst phys_pager_object_list;
|
|||||||
/* protect access to phys_pager_object_list */
|
/* protect access to phys_pager_object_list */
|
||||||
static struct mtx phys_pager_mtx;
|
static struct mtx phys_pager_mtx;
|
||||||
|
|
||||||
|
static int default_phys_pager_getpages(vm_object_t object, vm_page_t *m,
|
||||||
|
int count, int *rbehind, int *rahead);
|
||||||
|
static int default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
|
||||||
|
int fault_type, vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last);
|
||||||
|
static boolean_t default_phys_pager_haspage(vm_object_t object,
|
||||||
|
vm_pindex_t pindex, int *before, int *after);
|
||||||
|
struct phys_pager_ops default_phys_pg_ops = {
|
||||||
|
.phys_pg_getpages = default_phys_pager_getpages,
|
||||||
|
.phys_pg_populate = default_phys_pager_populate,
|
||||||
|
.phys_pg_haspage = default_phys_pager_haspage,
|
||||||
|
.phys_pg_ctor = NULL,
|
||||||
|
.phys_pg_dtor = NULL,
|
||||||
|
};
|
||||||
|
|
||||||
static void
|
static void
|
||||||
phys_pager_init(void)
|
phys_pager_init(void)
|
||||||
{
|
{
|
||||||
@ -59,12 +73,13 @@ phys_pager_init(void)
|
|||||||
mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF);
|
mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF);
|
||||||
}
|
}
|
||||||
|
|
||||||
static vm_object_t
|
vm_object_t
|
||||||
phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
|
phys_pager_allocate(void *handle, struct phys_pager_ops *ops, void *data,
|
||||||
vm_ooffset_t foff, struct ucred *cred)
|
vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred)
|
||||||
{
|
{
|
||||||
vm_object_t object, object1;
|
vm_object_t object, object1;
|
||||||
vm_pindex_t pindex;
|
vm_pindex_t pindex;
|
||||||
|
bool init;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Offset should be page aligned.
|
* Offset should be page aligned.
|
||||||
@ -73,6 +88,7 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
|
|||||||
return (NULL);
|
return (NULL);
|
||||||
|
|
||||||
pindex = OFF_TO_IDX(foff + PAGE_MASK + size);
|
pindex = OFF_TO_IDX(foff + PAGE_MASK + size);
|
||||||
|
init = true;
|
||||||
|
|
||||||
if (handle != NULL) {
|
if (handle != NULL) {
|
||||||
mtx_lock(&phys_pager_mtx);
|
mtx_lock(&phys_pager_mtx);
|
||||||
@ -97,11 +113,15 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
|
|||||||
*/
|
*/
|
||||||
if (pindex > object->size)
|
if (pindex > object->size)
|
||||||
object->size = pindex;
|
object->size = pindex;
|
||||||
|
init = false;
|
||||||
} else {
|
} else {
|
||||||
object = object1;
|
object = object1;
|
||||||
object1 = NULL;
|
object1 = NULL;
|
||||||
object->handle = handle;
|
object->handle = handle;
|
||||||
vm_object_set_flag(object, OBJ_POPULATE);
|
object->un_pager.phys.ops = ops;
|
||||||
|
object->un_pager.phys.data_ptr = data;
|
||||||
|
if (ops->phys_pg_populate != NULL)
|
||||||
|
vm_object_set_flag(object, OBJ_POPULATE);
|
||||||
TAILQ_INSERT_TAIL(&phys_pager_object_list,
|
TAILQ_INSERT_TAIL(&phys_pager_object_list,
|
||||||
object, pager_object_list);
|
object, pager_object_list);
|
||||||
}
|
}
|
||||||
@ -113,12 +133,25 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
|
|||||||
vm_object_deallocate(object1);
|
vm_object_deallocate(object1);
|
||||||
} else {
|
} else {
|
||||||
object = vm_object_allocate(OBJT_PHYS, pindex);
|
object = vm_object_allocate(OBJT_PHYS, pindex);
|
||||||
vm_object_set_flag(object, OBJ_POPULATE);
|
object->un_pager.phys.ops = ops;
|
||||||
|
object->un_pager.phys.data_ptr = data;
|
||||||
|
if (ops->phys_pg_populate != NULL)
|
||||||
|
vm_object_set_flag(object, OBJ_POPULATE);
|
||||||
}
|
}
|
||||||
|
if (init && ops->phys_pg_ctor != NULL)
|
||||||
|
ops->phys_pg_ctor(object, prot, foff, cred);
|
||||||
|
|
||||||
return (object);
|
return (object);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static vm_object_t
|
||||||
|
phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
|
||||||
|
vm_ooffset_t foff, struct ucred *ucred)
|
||||||
|
{
|
||||||
|
return (phys_pager_allocate(handle, &default_phys_pg_ops, NULL,
|
||||||
|
size, prot, foff, ucred));
|
||||||
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
phys_pager_dealloc(vm_object_t object)
|
phys_pager_dealloc(vm_object_t object)
|
||||||
{
|
{
|
||||||
@ -130,16 +163,18 @@ phys_pager_dealloc(vm_object_t object)
|
|||||||
mtx_unlock(&phys_pager_mtx);
|
mtx_unlock(&phys_pager_mtx);
|
||||||
VM_OBJECT_WLOCK(object);
|
VM_OBJECT_WLOCK(object);
|
||||||
}
|
}
|
||||||
object->handle = NULL;
|
|
||||||
object->type = OBJT_DEAD;
|
object->type = OBJT_DEAD;
|
||||||
|
if (object->un_pager.phys.ops->phys_pg_dtor != NULL)
|
||||||
|
object->un_pager.phys.ops->phys_pg_dtor(object);
|
||||||
|
object->handle = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Fill as many pages as vm_fault has allocated for us.
|
* Fill as many pages as vm_fault has allocated for us.
|
||||||
*/
|
*/
|
||||||
static int
|
static int
|
||||||
phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
|
default_phys_pager_getpages(vm_object_t object, vm_page_t *m, int count,
|
||||||
int *rahead)
|
int *rbehind, int *rahead)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
|
|
||||||
@ -161,6 +196,14 @@ phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
|
|||||||
return (VM_PAGER_OK);
|
return (VM_PAGER_OK);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
|
||||||
|
int *rahead)
|
||||||
|
{
|
||||||
|
return (object->un_pager.phys.ops->phys_pg_getpages(object, m,
|
||||||
|
count, rbehind, rahead));
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Implement a pretty aggressive clustered getpages strategy. Hint that
|
* Implement a pretty aggressive clustered getpages strategy. Hint that
|
||||||
* everything in an entire 4MB window should be prefaulted at once.
|
* everything in an entire 4MB window should be prefaulted at once.
|
||||||
@ -185,7 +228,7 @@ SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN,
|
|||||||
#define PHYSALLOC 16
|
#define PHYSALLOC 16
|
||||||
|
|
||||||
static int
|
static int
|
||||||
phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
|
default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
|
||||||
int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first,
|
int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first,
|
||||||
vm_pindex_t *last)
|
vm_pindex_t *last)
|
||||||
{
|
{
|
||||||
@ -216,6 +259,14 @@ phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
|
|||||||
return (VM_PAGER_OK);
|
return (VM_PAGER_OK);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
phys_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,
|
||||||
|
vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last)
|
||||||
|
{
|
||||||
|
return (object->un_pager.phys.ops->phys_pg_populate(object, pidx,
|
||||||
|
fault_type, max_prot, first, last));
|
||||||
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync,
|
phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync,
|
||||||
int *rtvals)
|
int *rtvals)
|
||||||
@ -225,7 +276,7 @@ phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static boolean_t
|
static boolean_t
|
||||||
phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
|
default_phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
|
||||||
int *after)
|
int *after)
|
||||||
{
|
{
|
||||||
vm_pindex_t base, end;
|
vm_pindex_t base, end;
|
||||||
@ -239,6 +290,14 @@ phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
|
|||||||
return (TRUE);
|
return (TRUE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static boolean_t
|
||||||
|
phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
|
||||||
|
int *after)
|
||||||
|
{
|
||||||
|
return (object->un_pager.phys.ops->phys_pg_haspage(object, pindex,
|
||||||
|
before, after));
|
||||||
|
}
|
||||||
|
|
||||||
struct pagerops physpagerops = {
|
struct pagerops physpagerops = {
|
||||||
.pgo_init = phys_pager_init,
|
.pgo_init = phys_pager_init,
|
||||||
.pgo_alloc = phys_pager_alloc,
|
.pgo_alloc = phys_pager_alloc,
|
||||||
|
@ -286,6 +286,7 @@ vm_object_init(void)
|
|||||||
kernel_object->flags |= OBJ_COLORED;
|
kernel_object->flags |= OBJ_COLORED;
|
||||||
kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
|
kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
|
||||||
#endif
|
#endif
|
||||||
|
kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The lock portion of struct vm_object must be type stable due
|
* The lock portion of struct vm_object must be type stable due
|
||||||
|
@ -173,6 +173,17 @@ struct vm_object {
|
|||||||
struct pctrie swp_blks;
|
struct pctrie swp_blks;
|
||||||
vm_ooffset_t writemappings;
|
vm_ooffset_t writemappings;
|
||||||
} swp;
|
} swp;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Phys pager
|
||||||
|
*/
|
||||||
|
struct {
|
||||||
|
struct phys_pager_ops *ops;
|
||||||
|
union {
|
||||||
|
void *data_ptr;
|
||||||
|
uintptr_t data_val;
|
||||||
|
};
|
||||||
|
} phys;
|
||||||
} un_pager;
|
} un_pager;
|
||||||
struct ucred *cred;
|
struct ucred *cred;
|
||||||
vm_ooffset_t charge;
|
vm_ooffset_t charge;
|
||||||
|
@ -227,5 +227,22 @@ vm_object_t cdev_pager_allocate(void *handle, enum obj_type tp,
|
|||||||
vm_object_t cdev_pager_lookup(void *handle);
|
vm_object_t cdev_pager_lookup(void *handle);
|
||||||
void cdev_pager_free_page(vm_object_t object, vm_page_t m);
|
void cdev_pager_free_page(vm_object_t object, vm_page_t m);
|
||||||
|
|
||||||
|
struct phys_pager_ops {
|
||||||
|
int (*phys_pg_getpages)(vm_object_t vm_obj, vm_page_t *m, int count,
|
||||||
|
int *rbehind, int *rahead);
|
||||||
|
int (*phys_pg_populate)(vm_object_t vm_obj, vm_pindex_t pidx,
|
||||||
|
int fault_type, vm_prot_t max_prot, vm_pindex_t *first,
|
||||||
|
vm_pindex_t *last);
|
||||||
|
boolean_t (*phys_pg_haspage)(vm_object_t obj, vm_pindex_t pindex,
|
||||||
|
int *before, int *after);
|
||||||
|
void (*phys_pg_ctor)(vm_object_t vm_obj, vm_prot_t prot,
|
||||||
|
vm_ooffset_t foff, struct ucred *cred);
|
||||||
|
void (*phys_pg_dtor)(vm_object_t vm_obj);
|
||||||
|
};
|
||||||
|
extern struct phys_pager_ops default_phys_pg_ops;
|
||||||
|
vm_object_t phys_pager_allocate(void *handle, struct phys_pager_ops *ops,
|
||||||
|
void *data, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff,
|
||||||
|
struct ucred *cred);
|
||||||
|
|
||||||
#endif /* _KERNEL */
|
#endif /* _KERNEL */
|
||||||
#endif /* _VM_PAGER_ */
|
#endif /* _VM_PAGER_ */
|
||||||
|
Loading…
Reference in New Issue
Block a user