Add an AIO & kqueue regression test. It is a good idea to run this

against a disk as the argument.  If you don't it will use a temp file.
The raw disk will use the kernel physio fast path method until the
max number of pending op's is reached then it will queue them.  File
system op's are always queued.  This is more important with LIO since
operation can get split across and accounting of op's is broken with LIO.

Note that this was broken when locking was added to kqueue (ie. 5.3)
My fix needs to be better integrated with FreeBSD.

Next is an LIO test and implementation.

Sponsored by:	IronPort

tools/regression/aio/kqueue/Makefile

@@ -0,0 +1,6 @@
+# $FreeBSD$
+
+PROG=	aio_kqueue
+NO_MAN=	YES
+
+.include <bsd.prog.mk>

tools/regression/aio/kqueue/aio_kqueue.c

@@ -0,0 +1,197 @@
+/*-
+ * Copyright (C) 2005 IronPort Systems, Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/* 
+ * Note: it is a good idea to run this against a physical drive to 
+ * exercise the physio fast path (ie. aio_kqueue /dev/<something safe>)
+ */
+
+#include <aio.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/event.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#define PATH_TEMPLATE   "/tmp/aio.XXXXXXXXXX"
+
+#define MAX 128
+#define MAX_RUNS 300
+/* #define DEBUG */
+
+main(int argc, char *argv[]){
+	int fd;
+	struct aiocb *iocb[MAX], *kq_iocb;
+	int i, result, run, error, j;
+	char buffer[32768];
+	int kq = kqueue();
+	struct kevent ke, kq_returned;
+	struct timespec ts;
+	int cancel, pending, tmp_file = 0, failed = 0;
+	char *file, pathname[sizeof(PATH_TEMPLATE)-1];
+
+	if (kq < 0) {
+		perror("No kqeueue\n");
+		exit(1);
+	}
+
+	if (argc == 1) { 
+		strcpy(pathname, PATH_TEMPLATE);
+		fd = mkstemp(pathname);
+		file = pathname;
+		tmp_file = 1;
+	} else {
+		file = argv[1];
+		fd = open(file, O_RDWR|O_CREAT, 0666);
+	}
+	if (fd < 0){
+		fprintf(stderr, "Can't open %s\n", file);
+		perror("");
+		exit(1);
+	}
+
+	for (run = 0; run < MAX_RUNS; run++){
+#ifdef DEBUG
+		printf("Run %d\n", run);
+#endif
+		for(i = 0; i < MAX; i++) {
+			iocb[i] = (struct aiocb *)malloc(sizeof(struct aiocb));
+			bzero(iocb[i], sizeof(struct aiocb));
+		}
+		
+		pending = 0;	
+		for(i = 0; i < MAX; i++) {
+			pending++;
+			iocb[i]->aio_nbytes = sizeof(buffer);
+			iocb[i]->aio_buf = buffer;
+			iocb[i]->aio_fildes = fd;
+			iocb[i]->aio_offset = iocb[i]->aio_nbytes * i * run;
+			
+			iocb[i]->aio_sigevent.sigev_notify_kqueue = kq;
+			iocb[i]->aio_sigevent.sigev_value.sigval_ptr = iocb[i];
+			iocb[i]->aio_sigevent.sigev_notify = SIGEV_KEVENT;
+			
+			result = aio_write(iocb[i]);
+			if (result != 0) {
+				perror("aio_write");
+				printf("Result %d iteration %d\n",result, i);
+				exit(1);
+			}
+#ifdef DEBUG
+			printf("WRITE %d is at %p\n", i, iocb[i]);
+#endif
+			result = rand();
+			if (result < RAND_MAX/32) {
+				if (result > RAND_MAX/64) {
+					result = aio_cancel(fd, iocb[i]);
+#ifdef DEBUG
+					printf("Cancel %d %p result %d\n", i, iocb[i], result);
+#endif
+					if (result == AIO_CANCELED) {
+						aio_return(iocb[i]);
+						iocb[i]=NULL;
+						pending--;
+					}
+				}
+			}
+		}
+		cancel = MAX - pending;
+		
+		i = 0;
+		while(pending) {
+			for(;;) {
+				bzero(&ke, sizeof(ke));
+				bzero(&kq_returned, sizeof(ke));
+				ts.tv_sec = 0;
+				ts.tv_nsec = 1;
+				result = kevent(kq, NULL, 0, 
+						&kq_returned, 1, &ts);
+				error = errno;
+				if (result < 0) {
+					perror("kevent error: ");
+				}
+				kq_iocb = kq_returned.udata;
+#ifdef DEBUG
+				printf("kevent %d %d errno %d return.ident %p "
+				       "return.data %p return.udata %p %p\n", 
+				       i, result, error, 
+				       kq_returned.ident, kq_returned.data, 
+				       kq_returned.udata, 
+				       kq_iocb);
+#endif
+				
+				if(kq_iocb)
+					break;
+#ifdef DEBUG
+				printf("Try again left %d out of %d %d\n",pending, MAX, cancel);
+#endif
+			}			
+			
+			for(j = 0; j < MAX; j++) {
+				if (iocb[j] == kq_iocb) {
+					break;
+				}
+			}
+#ifdef DEBUG
+			printf("kq_iocb %p\n", kq_iocb);
+			
+			printf("Error Result for %d is %d pending %d\n", j, result, pending);
+#endif
+			result = aio_return(kq_iocb);
+#ifdef DEBUG
+			printf("Return Result for %d is %d\n", j, result);
+			printf("\n");
+#endif
+			if (result != sizeof(buffer)) {
+				printf("FAIL: run %d, operation %d, result %d (errno=%d) should be %d\n", run, pending, result, errno, sizeof(buffer));
+				failed = 1;
+			} else {
+				printf("PASS: run %d, left %d\n", run, pending - 1);
+			}
+
+			free(kq_iocb);
+			iocb[j] = NULL;
+			pending--;
+			i++;
+		}	
+	}
+
+	if (tmp_file) {
+		unlink(pathname);
+	}
+
+	if (failed) {
+		printf("FAIL: Atleast one\n");
+		exit(1);
+	} else {
+		printf("PASS: All\n");
+		exit(0);
+	}
+}