Fine grain locking inside the cuse(3) kernel module.
Implement one mutex per cuse(3) server instance which also cover the clients belonging to the given server instance. This should significantly reduce the mutex congestion inside the cuse(3) kernel module when multiple servers are in use. MFC after: 1 week Sponsored by: Mellanox Technologies
This commit is contained in:
Hans Petter Selasky
parent
fa9ce9ac29
commit
98029019b6
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=359452
@ -1,6 +1,6 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2010-2017 Hans Petter Selasky. All rights reserved.
|
||||
* Copyright (c) 2010-2020 Hans Petter Selasky. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
@ -114,6 +114,7 @@ struct cuse_server {
|
||||
TAILQ_HEAD(, cuse_server_dev) hdev;
|
||||
TAILQ_HEAD(, cuse_client) hcli;
|
||||
TAILQ_HEAD(, cuse_memory) hmem;
|
||||
struct mtx mtx;
|
||||
struct cv cv;
|
||||
struct selinfo selinfo;
|
||||
pid_t pid;
|
||||
@ -145,7 +146,7 @@ struct cuse_client {
|
||||
static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory");
|
||||
|
||||
static TAILQ_HEAD(, cuse_server) cuse_server_head;
|
||||
static struct mtx cuse_mtx;
|
||||
static struct mtx cuse_global_mtx;
|
||||
static struct cdev *cuse_dev;
|
||||
static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX];
|
||||
static int cuse_alloc_unit_id[CUSE_DEVICES_MAX];
|
||||
@ -216,15 +217,27 @@ static void cuse_client_is_closing(struct cuse_client *);
|
||||
static int cuse_free_unit_by_id_locked(struct cuse_server *, int);
|
||||
|
||||
static void
|
||||
cuse_lock(void)
|
||||
cuse_global_lock(void)
|
||||
{
|
||||
mtx_lock(&cuse_mtx);
|
||||
mtx_lock(&cuse_global_mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
cuse_unlock(void)
|
||||
cuse_global_unlock(void)
|
||||
{
|
||||
mtx_unlock(&cuse_mtx);
|
||||
mtx_unlock(&cuse_global_mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
cuse_server_lock(struct cuse_server *pcs)
|
||||
{
|
||||
mtx_lock(&pcs->mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
cuse_server_unlock(struct cuse_server *pcs)
|
||||
{
|
||||
mtx_unlock(&pcs->mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -244,7 +257,7 @@ cuse_kern_init(void *arg)
|
||||
{
|
||||
TAILQ_INIT(&cuse_server_head);
|
||||
|
||||
mtx_init(&cuse_mtx, "cuse-mtx", NULL, MTX_DEF);
|
||||
mtx_init(&cuse_global_mtx, "cuse-global-mtx", NULL, MTX_DEF);
|
||||
|
||||
cuse_dev = make_dev(&cuse_server_devsw, 0,
|
||||
UID_ROOT, GID_OPERATOR, 0600, "cuse");
|
||||
@ -267,9 +280,9 @@ cuse_kern_uninit(void *arg)
|
||||
|
||||
pause("DRAIN", 2 * hz);
|
||||
|
||||
cuse_lock();
|
||||
cuse_global_lock();
|
||||
ptr = TAILQ_FIRST(&cuse_server_head);
|
||||
cuse_unlock();
|
||||
cuse_global_unlock();
|
||||
|
||||
if (ptr == NULL)
|
||||
break;
|
||||
@ -278,7 +291,7 @@ cuse_kern_uninit(void *arg)
|
||||
if (cuse_dev != NULL)
|
||||
destroy_dev(cuse_dev);
|
||||
|
||||
mtx_destroy(&cuse_mtx);
|
||||
mtx_destroy(&cuse_global_mtx);
|
||||
}
|
||||
SYSUNINIT(cuse_kern_uninit, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_uninit, 0);
|
||||
|
||||
@ -293,14 +306,10 @@ cuse_server_get(struct cuse_server **ppcs)
|
||||
*ppcs = NULL;
|
||||
return (error);
|
||||
}
|
||||
/* check if closing */
|
||||
cuse_lock();
|
||||
if (pcs->is_closing) {
|
||||
cuse_unlock();
|
||||
*ppcs = NULL;
|
||||
return (EINVAL);
|
||||
}
|
||||
cuse_unlock();
|
||||
*ppcs = pcs;
|
||||
return (0);
|
||||
}
|
||||
@ -426,14 +435,14 @@ cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr,
|
||||
goto error_0;
|
||||
}
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
/* check if allocation number already exists */
|
||||
TAILQ_FOREACH(temp, &pcs->hmem, entry) {
|
||||
if (temp->alloc_nr == alloc_nr)
|
||||
break;
|
||||
}
|
||||
if (temp != NULL) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
error = EBUSY;
|
||||
goto error_1;
|
||||
}
|
||||
@ -441,7 +450,7 @@ cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr,
|
||||
mem->page_count = page_count;
|
||||
mem->alloc_nr = alloc_nr;
|
||||
TAILQ_INSERT_TAIL(&pcs->hmem, mem, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
return (0);
|
||||
|
||||
@ -457,17 +466,17 @@ cuse_server_free_memory(struct cuse_server *pcs, uint32_t alloc_nr)
|
||||
{
|
||||
struct cuse_memory *mem;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
TAILQ_FOREACH(mem, &pcs->hmem, entry) {
|
||||
if (mem->alloc_nr == alloc_nr)
|
||||
break;
|
||||
}
|
||||
if (mem == NULL) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (EINVAL);
|
||||
}
|
||||
TAILQ_REMOVE(&pcs->hmem, mem, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
cuse_vm_memory_free(mem);
|
||||
|
||||
@ -486,14 +495,10 @@ cuse_client_get(struct cuse_client **ppcc)
|
||||
*ppcc = NULL;
|
||||
return (error);
|
||||
}
|
||||
/* check if closing */
|
||||
cuse_lock();
|
||||
if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) {
|
||||
cuse_unlock();
|
||||
*ppcc = NULL;
|
||||
return (EINVAL);
|
||||
}
|
||||
cuse_unlock();
|
||||
*ppcc = pcc;
|
||||
return (0);
|
||||
}
|
||||
@ -578,28 +583,28 @@ static int
|
||||
cuse_client_receive_command_locked(struct cuse_client_command *pccmd,
|
||||
uint8_t *arg_ptr, uint32_t arg_len)
|
||||
{
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
|
||||
pcs = pccmd->client->server;
|
||||
error = 0;
|
||||
|
||||
pccmd->proc_curr = curthread->td_proc;
|
||||
|
||||
if (CUSE_CLIENT_CLOSING(pccmd->client) ||
|
||||
pccmd->client->server->is_closing) {
|
||||
if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) {
|
||||
error = CUSE_ERR_OTHER;
|
||||
goto done;
|
||||
}
|
||||
while (pccmd->command == CUSE_CMD_NONE) {
|
||||
if (error != 0) {
|
||||
cv_wait(&pccmd->cv, &cuse_mtx);
|
||||
cv_wait(&pccmd->cv, &pcs->mtx);
|
||||
} else {
|
||||
error = cv_wait_sig(&pccmd->cv, &cuse_mtx);
|
||||
error = cv_wait_sig(&pccmd->cv, &pcs->mtx);
|
||||
|
||||
if (error != 0)
|
||||
cuse_client_got_signal(pccmd);
|
||||
}
|
||||
if (CUSE_CLIENT_CLOSING(pccmd->client) ||
|
||||
pccmd->client->server->is_closing) {
|
||||
if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) {
|
||||
error = CUSE_ERR_OTHER;
|
||||
goto done;
|
||||
}
|
||||
@ -616,7 +621,7 @@ cuse_client_receive_command_locked(struct cuse_client_command *pccmd,
|
||||
pccmd->proc_curr = NULL;
|
||||
|
||||
while (pccmd->proc_refs != 0)
|
||||
cv_wait(&pccmd->cv, &cuse_mtx);
|
||||
cv_wait(&pccmd->cv, &pcs->mtx);
|
||||
|
||||
return (error);
|
||||
}
|
||||
@ -635,7 +640,7 @@ cuse_server_free_dev(struct cuse_server_dev *pcsd)
|
||||
pcs = pcsd->server;
|
||||
|
||||
/* prevent creation of more devices */
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
if (pcsd->kern_dev != NULL)
|
||||
pcsd->kern_dev->si_drv1 = NULL;
|
||||
|
||||
@ -643,7 +648,7 @@ cuse_server_free_dev(struct cuse_server_dev *pcsd)
|
||||
if (pcc->server_dev == pcsd)
|
||||
cuse_client_is_closing(pcc);
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
/* destroy device, if any */
|
||||
if (pcsd->kern_dev != NULL) {
|
||||
@ -659,43 +664,46 @@ cuse_server_unref(struct cuse_server *pcs)
|
||||
struct cuse_server_dev *pcsd;
|
||||
struct cuse_memory *mem;
|
||||
|
||||
cuse_lock();
|
||||
pcs->refs--;
|
||||
if (pcs->refs != 0) {
|
||||
cuse_unlock();
|
||||
cuse_server_lock(pcs);
|
||||
if (--(pcs->refs) != 0) {
|
||||
cuse_server_unlock(pcs);
|
||||
return;
|
||||
}
|
||||
cuse_server_is_closing(pcs);
|
||||
/* final client wakeup, if any */
|
||||
cuse_server_wakeup_all_client_locked(pcs);
|
||||
|
||||
cuse_global_lock();
|
||||
TAILQ_REMOVE(&cuse_server_head, pcs, entry);
|
||||
cuse_global_unlock();
|
||||
|
||||
while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) {
|
||||
TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
cuse_server_free_dev(pcsd);
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
}
|
||||
|
||||
cuse_free_unit_by_id_locked(pcs, -1);
|
||||
|
||||
while ((mem = TAILQ_FIRST(&pcs->hmem)) != NULL) {
|
||||
TAILQ_REMOVE(&pcs->hmem, mem, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
cuse_vm_memory_free(mem);
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
}
|
||||
|
||||
knlist_clear(&pcs->selinfo.si_note, 1);
|
||||
knlist_destroy(&pcs->selinfo.si_note);
|
||||
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
seldrain(&pcs->selinfo);
|
||||
|
||||
cv_destroy(&pcs->cv);
|
||||
|
||||
mtx_destroy(&pcs->mtx);
|
||||
|
||||
free(pcs, M_CUSE);
|
||||
}
|
||||
|
||||
@ -704,7 +712,7 @@ cuse_server_do_close(struct cuse_server *pcs)
|
||||
{
|
||||
int retval;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_server_is_closing(pcs);
|
||||
/* final client wakeup, if any */
|
||||
cuse_server_wakeup_all_client_locked(pcs);
|
||||
@ -712,7 +720,7 @@ cuse_server_do_close(struct cuse_server *pcs)
|
||||
knlist_clear(&pcs->selinfo.si_note, 1);
|
||||
|
||||
retval = pcs->refs;
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
return (retval);
|
||||
}
|
||||
@ -758,12 +766,14 @@ cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
|
||||
cv_init(&pcs->cv, "cuse-server-cv");
|
||||
|
||||
knlist_init_mtx(&pcs->selinfo.si_note, &cuse_mtx);
|
||||
mtx_init(&pcs->mtx, "cuse-server-mtx", NULL, MTX_DEF);
|
||||
|
||||
cuse_lock();
|
||||
knlist_init_mtx(&pcs->selinfo.si_note, &pcs->mtx);
|
||||
|
||||
cuse_global_lock();
|
||||
pcs->refs++;
|
||||
TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry);
|
||||
cuse_unlock();
|
||||
cuse_global_unlock();
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -792,7 +802,8 @@ cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
}
|
||||
|
||||
static int
|
||||
cuse_server_ioctl_copy_locked(struct cuse_client_command *pccmd,
|
||||
cuse_server_ioctl_copy_locked(struct cuse_server *pcs,
|
||||
struct cuse_client_command *pccmd,
|
||||
struct cuse_data_chunk *pchk, int isread)
|
||||
{
|
||||
struct proc *p_proc;
|
||||
@ -819,7 +830,7 @@ cuse_server_ioctl_copy_locked(struct cuse_client_command *pccmd,
|
||||
|
||||
pccmd->proc_refs++;
|
||||
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (isread == 0) {
|
||||
error = copyin(
|
||||
@ -833,7 +844,7 @@ cuse_server_ioctl_copy_locked(struct cuse_client_command *pccmd,
|
||||
pchk->length);
|
||||
}
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
|
||||
pccmd->proc_refs--;
|
||||
|
||||
@ -898,7 +909,8 @@ cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s,
|
||||
}
|
||||
|
||||
static int
|
||||
cuse_server_data_copy_locked(struct cuse_client_command *pccmd,
|
||||
cuse_server_data_copy_locked(struct cuse_server *pcs,
|
||||
struct cuse_client_command *pccmd,
|
||||
struct cuse_data_chunk *pchk, int isread)
|
||||
{
|
||||
struct proc *p_proc;
|
||||
@ -913,7 +925,7 @@ cuse_server_data_copy_locked(struct cuse_client_command *pccmd,
|
||||
|
||||
pccmd->proc_refs++;
|
||||
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (isread == 0) {
|
||||
error = cuse_proc2proc_copy(
|
||||
@ -927,7 +939,7 @@ cuse_server_data_copy_locked(struct cuse_client_command *pccmd,
|
||||
pchk->length);
|
||||
}
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
|
||||
pccmd->proc_refs--;
|
||||
|
||||
@ -1031,16 +1043,16 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
case CUSE_IOCTL_GET_COMMAND:
|
||||
pcmd = (void *)data;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
|
||||
while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) {
|
||||
error = cv_wait_sig(&pcs->cv, &cuse_mtx);
|
||||
error = cv_wait_sig(&pcs->cv, &pcs->mtx);
|
||||
|
||||
if (pcs->is_closing)
|
||||
error = ENXIO;
|
||||
|
||||
if (error) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (error);
|
||||
}
|
||||
}
|
||||
@ -1052,13 +1064,13 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
|
||||
*pcmd = pccmd->sub;
|
||||
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_SYNC_COMMAND:
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) {
|
||||
|
||||
/* send sync command */
|
||||
@ -1069,16 +1081,16 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
/* signal peer, if any */
|
||||
cv_signal(&pccmd->cv);
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_ALLOC_UNIT:
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
n = cuse_alloc_unit_by_id_locked(pcs,
|
||||
CUSE_ID_DEFAULT(0));
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (n < 0)
|
||||
error = ENOMEM;
|
||||
@ -1092,9 +1104,9 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
|
||||
n = (n & CUSE_ID_MASK);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
n = cuse_alloc_unit_by_id_locked(pcs, n);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (n < 0)
|
||||
error = ENOMEM;
|
||||
@ -1108,18 +1120,18 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
|
||||
n = CUSE_ID_DEFAULT(n);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
error = cuse_free_unit_by_id_locked(pcs, n);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_FREE_UNIT_BY_ID:
|
||||
|
||||
n = *(int *)data;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
error = cuse_free_unit_by_id_locked(pcs, n);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_ALLOC_MEMORY:
|
||||
@ -1150,7 +1162,7 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
|
||||
case CUSE_IOCTL_GET_SIG:
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
pccmd = cuse_server_find_command(pcs, curthread);
|
||||
|
||||
if (pccmd != NULL) {
|
||||
@ -1159,7 +1171,7 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
} else {
|
||||
n = 0;
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
*(int *)data = n;
|
||||
|
||||
@ -1167,7 +1179,7 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
|
||||
case CUSE_IOCTL_SET_PFH:
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
pccmd = cuse_server_find_command(pcs, curthread);
|
||||
|
||||
if (pccmd != NULL) {
|
||||
@ -1178,7 +1190,7 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
} else {
|
||||
error = ENXIO;
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_CREATE_DEV:
|
||||
@ -1224,9 +1236,9 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
}
|
||||
pcsd->kern_dev->si_drv1 = pcsd;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
break;
|
||||
|
||||
@ -1236,7 +1248,7 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
if (error)
|
||||
break;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
|
||||
error = EINVAL;
|
||||
|
||||
@ -1244,9 +1256,9 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
while (pcsd != NULL) {
|
||||
if (pcsd->user_dev == *(struct cuse_dev **)data) {
|
||||
TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
cuse_server_free_dev(pcsd);
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
error = 0;
|
||||
pcsd = TAILQ_FIRST(&pcs->hdev);
|
||||
} else {
|
||||
@ -1254,13 +1266,13 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
}
|
||||
}
|
||||
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_WRITE_DATA:
|
||||
case CUSE_IOCTL_READ_DATA:
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
pchk = (struct cuse_data_chunk *)data;
|
||||
|
||||
pccmd = cuse_server_find_command(pcs, curthread);
|
||||
@ -1270,23 +1282,23 @@ cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
} else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) {
|
||||
error = EFAULT; /* NULL pointer */
|
||||
} else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) {
|
||||
error = cuse_server_ioctl_copy_locked(pccmd,
|
||||
error = cuse_server_ioctl_copy_locked(pcs, pccmd,
|
||||
pchk, cmd == CUSE_IOCTL_READ_DATA);
|
||||
} else {
|
||||
error = cuse_server_data_copy_locked(pccmd,
|
||||
error = cuse_server_data_copy_locked(pcs, pccmd,
|
||||
pchk, cmd == CUSE_IOCTL_READ_DATA);
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
case CUSE_IOCTL_SELWAKEUP:
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
/*
|
||||
* We don't know which direction caused the event.
|
||||
* Wakeup both!
|
||||
*/
|
||||
cuse_server_wakeup_all_client_locked(pcs);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
break;
|
||||
|
||||
default:
|
||||
@ -1317,31 +1329,31 @@ cuse_server_mmap_single(struct cdev *dev, vm_ooffset_t *offset,
|
||||
if (error != 0)
|
||||
return (error);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
/* lookup memory structure */
|
||||
TAILQ_FOREACH(mem, &pcs->hmem, entry) {
|
||||
if (mem->alloc_nr == alloc_nr)
|
||||
break;
|
||||
}
|
||||
if (mem == NULL) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (ENOMEM);
|
||||
}
|
||||
/* verify page offset */
|
||||
page_nr %= CUSE_ALLOC_PAGES_MAX;
|
||||
if (page_nr >= mem->page_count) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (ENXIO);
|
||||
}
|
||||
/* verify mmap size */
|
||||
if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE) ||
|
||||
(size > ((mem->page_count - page_nr) * PAGE_SIZE))) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (EINVAL);
|
||||
}
|
||||
vm_object_reference(mem->object);
|
||||
*object = mem->object;
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
/* set new VM object offset to use */
|
||||
*offset = page_nr * PAGE_SIZE;
|
||||
@ -1361,10 +1373,12 @@ cuse_client_free(void *arg)
|
||||
struct cuse_server *pcs;
|
||||
int n;
|
||||
|
||||
cuse_lock();
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_is_closing(pcc);
|
||||
TAILQ_REMOVE(&pcc->server->hcli, pcc, entry);
|
||||
cuse_unlock();
|
||||
TAILQ_REMOVE(&pcs->hcli, pcc, entry);
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
for (n = 0; n != CUSE_CMD_MAX; n++) {
|
||||
|
||||
@ -1374,8 +1388,6 @@ cuse_client_free(void *arg)
|
||||
cv_destroy(&pccmd->cv);
|
||||
}
|
||||
|
||||
pcs = pcc->server;
|
||||
|
||||
free(pcc, M_CUSE);
|
||||
|
||||
/* drop reference on server */
|
||||
@ -1393,11 +1405,12 @@ cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
int error;
|
||||
int n;
|
||||
|
||||
cuse_lock();
|
||||
pcsd = dev->si_drv1;
|
||||
if (pcsd != NULL) {
|
||||
pcs = pcsd->server;
|
||||
pcd = pcsd->user_dev;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
/*
|
||||
* Check that the refcount didn't wrap and that the
|
||||
* same process is not both client and server. This
|
||||
@ -1408,16 +1421,13 @@ cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
if (pcs->refs < 0 || pcs->pid == curproc->p_pid) {
|
||||
/* overflow or wrong PID */
|
||||
pcs->refs--;
|
||||
pcsd = NULL;
|
||||
cuse_server_unlock(pcs);
|
||||
return (EINVAL);
|
||||
}
|
||||
cuse_server_unlock(pcs);
|
||||
} else {
|
||||
pcs = NULL;
|
||||
pcd = NULL;
|
||||
}
|
||||
cuse_unlock();
|
||||
|
||||
if (pcsd == NULL)
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO);
|
||||
if (pcc == NULL) {
|
||||
@ -1448,7 +1458,7 @@ cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
cv_init(&pccmd->cv, "cuse-client-cv");
|
||||
}
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
|
||||
/* cuse_client_free() assumes that the client is listed somewhere! */
|
||||
/* always enqueue */
|
||||
@ -1461,7 +1471,7 @@ cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
} else {
|
||||
error = 0;
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error) {
|
||||
devfs_clear_cdevpriv(); /* XXX bugfix */
|
||||
@ -1471,11 +1481,11 @@ cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
|
||||
|
||||
cuse_cmd_lock(pccmd);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, 0, 0);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error < 0) {
|
||||
error = cuse_convert_error(error);
|
||||
@ -1496,6 +1506,7 @@ cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
|
||||
{
|
||||
struct cuse_client_command *pccmd;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
|
||||
error = cuse_client_get(&pcc);
|
||||
@ -1503,20 +1514,18 @@ cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
|
||||
return (0);
|
||||
|
||||
pccmd = &pcc->cmds[CUSE_CMD_CLOSE];
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_cmd_lock(pccmd);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, 0, 0);
|
||||
cuse_unlock();
|
||||
|
||||
cuse_cmd_unlock(pccmd);
|
||||
|
||||
cuse_lock();
|
||||
cuse_client_is_closing(pcc);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -1524,21 +1533,22 @@ cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
|
||||
static void
|
||||
cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc)
|
||||
{
|
||||
struct cuse_server *pcs = pcc->server;
|
||||
int temp;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ |
|
||||
CUSE_CLI_KNOTE_HAS_WRITE));
|
||||
pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ |
|
||||
CUSE_CLI_KNOTE_NEED_WRITE);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (temp != 0) {
|
||||
/* get the latest polling state from the server */
|
||||
temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL);
|
||||
|
||||
if (temp & (POLLIN | POLLOUT)) {
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
if (temp & POLLIN)
|
||||
pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ;
|
||||
if (temp & POLLOUT)
|
||||
@ -1546,7 +1556,7 @@ cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc)
|
||||
|
||||
/* make sure the "knote" gets woken up */
|
||||
cuse_server_wakeup_locked(pcc->server);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1556,6 +1566,7 @@ cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
{
|
||||
struct cuse_client_command *pccmd;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
int len;
|
||||
|
||||
@ -1564,6 +1575,7 @@ cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
return (error);
|
||||
|
||||
pccmd = &pcc->cmds[CUSE_CMD_READ];
|
||||
pcs = pcc->server;
|
||||
|
||||
if (uio->uio_segflg != UIO_USERSPACE) {
|
||||
return (EINVAL);
|
||||
@ -1580,13 +1592,13 @@ cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
}
|
||||
len = uio->uio_iov->iov_len;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd,
|
||||
(uintptr_t)uio->uio_iov->iov_base,
|
||||
(unsigned long)(unsigned int)len, pcc->fflags, ioflag);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, 0, 0);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error < 0) {
|
||||
error = cuse_convert_error(error);
|
||||
@ -1615,6 +1627,7 @@ cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
{
|
||||
struct cuse_client_command *pccmd;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
int len;
|
||||
|
||||
@ -1623,6 +1636,7 @@ cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
return (error);
|
||||
|
||||
pccmd = &pcc->cmds[CUSE_CMD_WRITE];
|
||||
pcs = pcc->server;
|
||||
|
||||
if (uio->uio_segflg != UIO_USERSPACE) {
|
||||
return (EINVAL);
|
||||
@ -1639,13 +1653,13 @@ cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
|
||||
}
|
||||
len = uio->uio_iov->iov_len;
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd,
|
||||
(uintptr_t)uio->uio_iov->iov_base,
|
||||
(unsigned long)(unsigned int)len, pcc->fflags, ioflag);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, 0, 0);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error < 0) {
|
||||
error = cuse_convert_error(error);
|
||||
@ -1675,6 +1689,7 @@ cuse_client_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
{
|
||||
struct cuse_client_command *pccmd;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
int len;
|
||||
|
||||
@ -1687,6 +1702,7 @@ cuse_client_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
return (ENOMEM);
|
||||
|
||||
pccmd = &pcc->cmds[CUSE_CMD_IOCTL];
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_cmd_lock(pccmd);
|
||||
|
||||
@ -1699,14 +1715,14 @@ cuse_client_ioctl(struct cdev *dev, unsigned long cmd,
|
||||
* is forwarded to the driver.
|
||||
*/
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd,
|
||||
(len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR,
|
||||
(unsigned long)cmd, pcc->fflags,
|
||||
(fflag & O_NONBLOCK) ? IO_NDELAY : 0);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, data, len);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error < 0) {
|
||||
error = cuse_convert_error(error);
|
||||
@ -1730,6 +1746,7 @@ cuse_client_poll(struct cdev *dev, int events, struct thread *td)
|
||||
{
|
||||
struct cuse_client_command *pccmd;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
unsigned long temp;
|
||||
int error;
|
||||
int revents;
|
||||
@ -1739,6 +1756,7 @@ cuse_client_poll(struct cdev *dev, int events, struct thread *td)
|
||||
goto pollnval;
|
||||
|
||||
temp = 0;
|
||||
pcs = pcc->server;
|
||||
|
||||
if (events & (POLLPRI | POLLIN | POLLRDNORM))
|
||||
temp |= CUSE_POLL_READ;
|
||||
@ -1755,14 +1773,14 @@ cuse_client_poll(struct cdev *dev, int events, struct thread *td)
|
||||
|
||||
/* Need to selrecord() first to not loose any events. */
|
||||
if (temp != 0 && td != NULL)
|
||||
selrecord(td, &pcc->server->selinfo);
|
||||
selrecord(td, &pcs->selinfo);
|
||||
|
||||
cuse_lock();
|
||||
cuse_server_lock(pcs);
|
||||
cuse_client_send_command_locked(pccmd,
|
||||
0, temp, pcc->fflags, IO_NDELAY);
|
||||
|
||||
error = cuse_client_receive_command_locked(pccmd, 0, 0);
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
cuse_cmd_unlock(pccmd);
|
||||
|
||||
@ -1793,37 +1811,40 @@ cuse_client_mmap_single(struct cdev *dev, vm_ooffset_t *offset,
|
||||
uint32_t alloc_nr = page_nr / CUSE_ALLOC_PAGES_MAX;
|
||||
struct cuse_memory *mem;
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
int error;
|
||||
|
||||
error = cuse_client_get(&pcc);
|
||||
if (error != 0)
|
||||
return (error);
|
||||
|
||||
cuse_lock();
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
/* lookup memory structure */
|
||||
TAILQ_FOREACH(mem, &pcc->server->hmem, entry) {
|
||||
TAILQ_FOREACH(mem, &pcs->hmem, entry) {
|
||||
if (mem->alloc_nr == alloc_nr)
|
||||
break;
|
||||
}
|
||||
if (mem == NULL) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (ENOMEM);
|
||||
}
|
||||
/* verify page offset */
|
||||
page_nr %= CUSE_ALLOC_PAGES_MAX;
|
||||
if (page_nr >= mem->page_count) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (ENXIO);
|
||||
}
|
||||
/* verify mmap size */
|
||||
if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE) ||
|
||||
(size > ((mem->page_count - page_nr) * PAGE_SIZE))) {
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
return (EINVAL);
|
||||
}
|
||||
vm_object_reference(mem->object);
|
||||
*object = mem->object;
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
/* set new VM object offset to use */
|
||||
*offset = page_nr * PAGE_SIZE;
|
||||
@ -1836,22 +1857,28 @@ static void
|
||||
cuse_client_kqfilter_read_detach(struct knote *kn)
|
||||
{
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
|
||||
cuse_lock();
|
||||
pcc = kn->kn_hook;
|
||||
knlist_remove(&pcc->server->selinfo.si_note, kn, 1);
|
||||
cuse_unlock();
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
knlist_remove(&pcs->selinfo.si_note, kn, 1);
|
||||
cuse_server_unlock(pcs);
|
||||
}
|
||||
|
||||
static void
|
||||
cuse_client_kqfilter_write_detach(struct knote *kn)
|
||||
{
|
||||
struct cuse_client *pcc;
|
||||
struct cuse_server *pcs;
|
||||
|
||||
cuse_lock();
|
||||
pcc = kn->kn_hook;
|
||||
knlist_remove(&pcc->server->selinfo.si_note, kn, 1);
|
||||
cuse_unlock();
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
knlist_remove(&pcs->selinfo.si_note, kn, 1);
|
||||
cuse_server_unlock(pcs);
|
||||
}
|
||||
|
||||
static int
|
||||
@ -1859,9 +1886,10 @@ cuse_client_kqfilter_read_event(struct knote *kn, long hint)
|
||||
{
|
||||
struct cuse_client *pcc;
|
||||
|
||||
mtx_assert(&cuse_mtx, MA_OWNED);
|
||||
|
||||
pcc = kn->kn_hook;
|
||||
|
||||
mtx_assert(&pcc->server->mtx, MA_OWNED);
|
||||
|
||||
return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0);
|
||||
}
|
||||
|
||||
@ -1870,9 +1898,10 @@ cuse_client_kqfilter_write_event(struct knote *kn, long hint)
|
||||
{
|
||||
struct cuse_client *pcc;
|
||||
|
||||
mtx_assert(&cuse_mtx, MA_OWNED);
|
||||
|
||||
pcc = kn->kn_hook;
|
||||
|
||||
mtx_assert(&pcc->server->mtx, MA_OWNED);
|
||||
|
||||
return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0);
|
||||
}
|
||||
|
||||
@ -1887,8 +1916,9 @@ cuse_client_kqfilter(struct cdev *dev, struct knote *kn)
|
||||
if (error != 0)
|
||||
return (error);
|
||||
|
||||
cuse_lock();
|
||||
pcs = pcc->server;
|
||||
|
||||
cuse_server_lock(pcs);
|
||||
switch (kn->kn_filter) {
|
||||
case EVFILT_READ:
|
||||
pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ;
|
||||
@ -1906,7 +1936,7 @@ cuse_client_kqfilter(struct cdev *dev, struct knote *kn)
|
||||
error = EINVAL;
|
||||
break;
|
||||
}
|
||||
cuse_unlock();
|
||||
cuse_server_unlock(pcs);
|
||||
|
||||
if (error == 0)
|
||||
cuse_client_kqfilter_poll(dev, pcc);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user