d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the Tigon 1 and 2 driver to use the sf_buf API for implementing

Browse Source 
zero-copy receive of jumbo frames.  This eliminates the need for the
jumbo frame allocator implemented in kern/uipc_jumbo.c and sys/jumbo.h.
Remove it.

Note: Zero-copy receive of jumbo frames did not work without these changes;
I believe there was insufficient locking on the jumbo vm object.

Tested by: ken@
Discussed with: gallatin@
This commit is contained in:
Alan Cox 2004-12-06 00:43:40 +00:00
parent dc0ff2140b
commit 370abcb3e5
6 changed files with 50 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sys/conf/files
View File

@ -1167,7 +1167,6 @@ kern/tty_tty.c standard
kern/uipc_accf.c optional inet
kern/uipc_cow.c optional zero_copy_sockets
kern/uipc_domain.c standard
kern/uipc_jumbo.c standard
kern/uipc_mbuf.c standard
kern/uipc_mbuf2.c standard
kern/uipc_proto.c standard

38
sys/dev/ti/if_ti.c
View File

@ -120,6 +120,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sockio.h>
#include <sys/uio.h>
#include <sys/lock.h>
#include <sys/sf_buf.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
@ -131,7 +132,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <sys/proc.h>
#include <sys/jumbo.h>
#endif /* !TI_PRIVATE_JUMBOS */
#include <dev/pci/pcireg.h>
@ -1251,8 +1251,9 @@ ti_newbuf_jumbo(sc, idx, m_old)
struct mbuf *m[3] = {NULL, NULL, NULL};
struct ti_rx_desc_ext *r;
vm_page_t frame;
static int color;
/* 1 extra buf to make nobufs easy*/
caddr_t buf[3] = {NULL, NULL, NULL};
struct sf_buf *sf[3] = {NULL, NULL, NULL};
int i;
if (m_old != NULL) {
@ -1291,20 +1292,33 @@ ti_newbuf_jumbo(sc, idx, m_old)
"-- packet dropped!\n", sc->ti_unit);
goto nobufs;
}
if (!(frame = jumbo_pg_alloc())){
frame = vm_page_alloc(NULL, color++,
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED);
if (frame == NULL) {
printf("ti%d: buffer allocation failed "
"-- packet dropped!\n", sc->ti_unit);
printf(" index %d page %d\n", idx, i);
goto nobufs;
}
sf[i] = sf_buf_alloc(frame, SFB_NOWAIT);
if (sf[i] == NULL) {
vm_page_lock_queues();
vm_page_unwire(frame, 0);
vm_page_free(frame);
vm_page_unlock_queues();
printf("ti%d: buffer allocation failed "
"-- packet dropped!\n", sc->ti_unit);
printf(" index %d page %d\n", idx, i);
goto nobufs;
}
buf[i] = jumbo_phys_to_kva(VM_PAGE_TO_PHYS(frame));
}
for (i = 0; i < NPAYLOAD; i++){
/* Attach the buffer to the mbuf. */
m[i]->m_data = (void *)buf[i];
m[i]->m_data = (void *)sf_buf_kva(sf[i]);
m[i]->m_len = PAGE_SIZE;
MEXTADD(m[i], (void *)buf[i], PAGE_SIZE,
jumbo_freem, NULL, 0, EXT_DISPOSABLE);
MEXTADD(m[i], sf_buf_kva(sf[i]), PAGE_SIZE,
sf_buf_mext, sf[i], 0, EXT_DISPOSABLE);
m[i]->m_next = m[i+1];
}
/* link the buffers to the header */
@ -1360,8 +1374,8 @@ nobufs:
for (i = 0; i < 3; i++) {
if (m[i])
m_freem(m[i]);
if (buf[i])
jumbo_pg_free((vm_offset_t)buf[i]);
if (sf[i])
sf_buf_mext((void *)sf_buf_kva(sf[i]), sf[i]);
}
return (ENOBUFS);
}
@ -2116,12 +2130,6 @@ ti_attach(dev)
error = ENXIO;
goto fail;
}
#else
if (!jumbo_vm_init()) {
printf("ti%d: VM initialization failed!\n", sc->ti_unit);
error = ENOMEM;
goto fail;
}
#endif
/*

10
sys/kern/kern_subr.c
View File

@ -83,9 +83,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
* First lookup the kernel page.
*/
kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr));
/*
* XXX The vm object containing kern_pg needs locking.
*/
if ((vm_map_lookup(&map, uaddr,
VM_PROT_WRITE, &entry, &uobject,
&upindex, &prot, &wired)) != KERN_SUCCESS) {
@ -111,7 +109,8 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
else
panic("vm_pgmoveco: renaming busy page");
}
vm_page_rename(kern_pg, uobject, upindex);
vm_page_insert(kern_pg, uobject, upindex);
vm_page_dirty(kern_pg);
kern_pg->valid = VM_PAGE_BITS_ALL;
vm_page_unlock_queues();
VM_OBJECT_UNLOCK(uobject);
@ -221,12 +220,11 @@ userspaceco(void *cp, u_int cnt, struct uio *uio, struct vm_object *obj,
iov = uio->uio_iov;
if (uio->uio_rw == UIO_READ) {
if ((so_zero_copy_receive != 0)
&& (obj != NULL)
&& (obj == NULL)
&& ((cnt & PAGE_MASK) == 0)
&& ((((intptr_t) iov->iov_base) & PAGE_MASK) == 0)
&& ((uio->uio_offset & PAGE_MASK) == 0)
&& ((((intptr_t) cp) & PAGE_MASK) == 0)
&& (obj->type == OBJT_DEFAULT)
&& (disposable != 0)) {
/* SOCKET: use page-trading */
/*

254
sys/kern/uipc_jumbo.c
View File

@ -1,254 +0,0 @@
/*-
* Copyright (c) 1997, Duke University
* All rights reserved.
*
* Author:
* Andrew Gallatin <gallatin@cs.duke.edu>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of Duke University may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* This is a set of routines for allocating large-sized mbuf payload
* areas, and is primarily intended for use in receive side mbuf
* allocation.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/sockio.h>
#include <sys/uio.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_extern.h>
#include <vm/vm_map.h>
#include <vm/vm_param.h>
#include <vm/vm_pageout.h>
#include <sys/vmmeter.h>
#include <vm/vm_page.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <sys/proc.h>
#include <sys/jumbo.h>
/*
* XXX this may be too high or too low.
*/
#define JUMBO_MAX_PAGES 3072
struct jumbo_kmap {
vm_offset_t kva;
SLIST_ENTRY(jumbo_kmap) entries; /* Singly-linked List. */
};
static SLIST_HEAD(jumbo_kmap_head, jumbo_kmap) jumbo_kmap_free,
jumbo_kmap_inuse;
static struct mtx jumbo_mutex;
MTX_SYSINIT(jumbo_lock, &jumbo_mutex, "jumbo mutex", MTX_DEF);
static struct vm_object *jumbo_vm_object;
static unsigned long jumbo_vmuiomove_pgs_freed = 0;
#if 0
static int jumbo_vm_wakeup_wanted = 0;
#endif
vm_offset_t jumbo_basekva;
int
jumbo_vm_init(void)
{
int i;
struct jumbo_kmap *entry;
mtx_lock(&jumbo_mutex);
if (jumbo_vm_object != NULL) {
mtx_unlock(&jumbo_mutex);
return (1);
}
/* allocate our object */
jumbo_vm_object = vm_object_allocate_wait(OBJT_DEFAULT, JUMBO_MAX_PAGES,
M_NOWAIT);
if (jumbo_vm_object == NULL) {
mtx_unlock(&jumbo_mutex);
return (0);
}
SLIST_INIT(&jumbo_kmap_free);
SLIST_INIT(&jumbo_kmap_inuse);
/* grab some kernel virtual address space */
jumbo_basekva = kmem_alloc_nofault(kernel_map,
PAGE_SIZE * JUMBO_MAX_PAGES);
if (jumbo_basekva == 0) {
vm_object_deallocate(jumbo_vm_object);
jumbo_vm_object = NULL;
mtx_unlock(&jumbo_mutex);
return 0;
}
for (i = 0; i < JUMBO_MAX_PAGES; i++) {
entry = malloc(sizeof(struct jumbo_kmap), M_TEMP, M_NOWAIT);
if (!entry && !i) {
mtx_unlock(&jumbo_mutex);
panic("jumbo_vm_init: unable to allocated kvas");
} else if (!entry) {
printf("warning: jumbo_vm_init allocated only %d kva\n",
i);
mtx_unlock(&jumbo_mutex);
return 1;
}
entry->kva = jumbo_basekva + (vm_offset_t)i * PAGE_SIZE;
SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
}
mtx_unlock(&jumbo_mutex);
return 1;
}
void
jumbo_freem(void *addr, void *args)
{
vm_page_t frame;
frame = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)addr));
/*
* Need giant for looking at the hold count below. Convert this
* to the vm mutex once the VM code has been moved out from under
* giant.
*/
GIANT_REQUIRED;
if (frame->hold_count == 0)
jumbo_pg_free((vm_offset_t)addr);
else
printf("jumbo_freem: hold count for %p is %d!!??\n",
frame, frame->hold_count);
}
void
jumbo_pg_steal(vm_page_t pg)
{
vm_offset_t addr;
struct jumbo_kmap *entry;
addr = ptoa(pg->pindex) + jumbo_basekva;
if (pg->object != jumbo_vm_object)
panic("stealing a non jumbo_vm_object page");
vm_page_remove(pg);
mtx_lock(&jumbo_mutex);
pmap_qremove(addr,1);
entry = SLIST_FIRST(&jumbo_kmap_inuse);
entry->kva = addr;
SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries);
SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
mtx_unlock(&jumbo_mutex);
#if 0
if (jumbo_vm_wakeup_wanted)
wakeup(jumbo_vm_object);
#endif
}
vm_page_t
jumbo_pg_alloc(void)
{
vm_page_t pg;
vm_pindex_t pindex;
struct jumbo_kmap *entry;
pg = NULL;
mtx_lock(&jumbo_mutex);
entry = SLIST_FIRST(&jumbo_kmap_free);
if (entry != NULL){
pindex = atop(entry->kva - jumbo_basekva);
VM_OBJECT_LOCK(jumbo_vm_object);
pg = vm_page_alloc(jumbo_vm_object, pindex, VM_ALLOC_INTERRUPT);
VM_OBJECT_UNLOCK(jumbo_vm_object);
if (pg != NULL) {
SLIST_REMOVE_HEAD(&jumbo_kmap_free, entries);
SLIST_INSERT_HEAD(&jumbo_kmap_inuse, entry, entries);
pmap_qenter(entry->kva, &pg, 1);
}
}
mtx_unlock(&jumbo_mutex);
return(pg);
}
void
jumbo_pg_free(vm_offset_t addr)
{
struct jumbo_kmap *entry;
vm_paddr_t paddr;
vm_page_t pg;
paddr = pmap_kextract((vm_offset_t)addr);
pg = PHYS_TO_VM_PAGE(paddr);
VM_OBJECT_LOCK(jumbo_vm_object);
if (pg->object != jumbo_vm_object) {
jumbo_vmuiomove_pgs_freed++;
/* if(vm_page_lookup(jumbo_vm_object, atop(addr - jumbo_basekva)))
panic("vm_page_rename didn't");
printf("freeing uiomoved pg:\t pindex = %d, padd = 0x%lx\n",
atop(addr - jumbo_basekva), paddr);
*/
} else {
vm_page_lock_queues();
vm_page_free(pg);
vm_page_unlock_queues();
}
VM_OBJECT_UNLOCK(jumbo_vm_object);
mtx_lock(&jumbo_mutex);
pmap_qremove(addr,1);
entry = SLIST_FIRST(&jumbo_kmap_inuse);
entry->kva = addr;
SLIST_REMOVE_HEAD(&jumbo_kmap_inuse, entries);
SLIST_INSERT_HEAD(&jumbo_kmap_free, entry, entries);
mtx_unlock(&jumbo_mutex);
#if 0
if (jumbo_vm_wakeup_wanted)
wakeup(jumbo_vm_object);
#endif
}

38
sys/pci/if_ti.c
View File

@ -120,6 +120,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sockio.h>
#include <sys/uio.h>
#include <sys/lock.h>
#include <sys/sf_buf.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
@ -131,7 +132,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <sys/proc.h>
#include <sys/jumbo.h>
#endif /* !TI_PRIVATE_JUMBOS */
#include <dev/pci/pcireg.h>
@ -1251,8 +1251,9 @@ ti_newbuf_jumbo(sc, idx, m_old)
struct mbuf *m[3] = {NULL, NULL, NULL};
struct ti_rx_desc_ext *r;
vm_page_t frame;
static int color;
/* 1 extra buf to make nobufs easy*/
caddr_t buf[3] = {NULL, NULL, NULL};
struct sf_buf *sf[3] = {NULL, NULL, NULL};
int i;
if (m_old != NULL) {
@ -1291,20 +1292,33 @@ ti_newbuf_jumbo(sc, idx, m_old)
"-- packet dropped!\n", sc->ti_unit);
goto nobufs;
}
if (!(frame = jumbo_pg_alloc())){
frame = vm_page_alloc(NULL, color++,
VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED);
if (frame == NULL) {
printf("ti%d: buffer allocation failed "
"-- packet dropped!\n", sc->ti_unit);
printf(" index %d page %d\n", idx, i);
goto nobufs;
}
sf[i] = sf_buf_alloc(frame, SFB_NOWAIT);
if (sf[i] == NULL) {
vm_page_lock_queues();
vm_page_unwire(frame, 0);
vm_page_free(frame);
vm_page_unlock_queues();
printf("ti%d: buffer allocation failed "
"-- packet dropped!\n", sc->ti_unit);
printf(" index %d page %d\n", idx, i);
goto nobufs;
}
buf[i] = jumbo_phys_to_kva(VM_PAGE_TO_PHYS(frame));
}
for (i = 0; i < NPAYLOAD; i++){
/* Attach the buffer to the mbuf. */
m[i]->m_data = (void *)buf[i];
m[i]->m_data = (void *)sf_buf_kva(sf[i]);
m[i]->m_len = PAGE_SIZE;
MEXTADD(m[i], (void *)buf[i], PAGE_SIZE,
jumbo_freem, NULL, 0, EXT_DISPOSABLE);
MEXTADD(m[i], sf_buf_kva(sf[i]), PAGE_SIZE,
sf_buf_mext, sf[i], 0, EXT_DISPOSABLE);
m[i]->m_next = m[i+1];
}
/* link the buffers to the header */
@ -1360,8 +1374,8 @@ nobufs:
for (i = 0; i < 3; i++) {
if (m[i])
m_freem(m[i]);
if (buf[i])
jumbo_pg_free((vm_offset_t)buf[i]);
if (sf[i])
sf_buf_mext((void *)sf_buf_kva(sf[i]), sf[i]);
}
return (ENOBUFS);
}
@ -2116,12 +2130,6 @@ ti_attach(dev)
error = ENXIO;
goto fail;
}
#else
if (!jumbo_vm_init()) {
printf("ti%d: VM initialization failed!\n", sc->ti_unit);
error = ENOMEM;
goto fail;
}
#endif
/*

59
sys/sys/jumbo.h
View File

@ -1,59 +0,0 @@
/*-
* Copyright (c) 1997, Duke University
* All rights reserved.
*
* Author:
* Andrew Gallatin <gallatin@cs.duke.edu>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of Duke University may not be used to endorse or promote
* products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY DUKE UNIVERSITY ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DUKE UNIVERSITY BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITSOR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _SYS_JUMBO_H_
#define _SYS_JUMBO_H_
#ifdef _KERNEL
extern vm_offset_t jumbo_basekva;
static __inline caddr_t jumbo_phys_to_kva(vm_paddr_t pa);
static __inline caddr_t
jumbo_phys_to_kva(vm_paddr_t pa)
{
vm_page_t pg;
pg = PHYS_TO_VM_PAGE(pa);
pg->flags &= ~PG_BUSY;
return (caddr_t)(ptoa((vm_offset_t)pg->pindex) + jumbo_basekva);
}
int jumbo_vm_init(void);
void jumbo_freem(void *addr, void *args);
vm_page_t jumbo_pg_alloc(void);
void jumbo_pg_free(vm_offset_t addr);
void jumbo_pg_steal(vm_page_t pg);
#endif /* _KERNEL */
#endif /* !_SYS_JUMBO_H_ */