Use UMA instead of plain malloc for getting ATA request storage.

This gives +10% performance on simple tests, so definitly worth it.
A few percent more could be had by not using M_ZERO'd alloc's, but
we then need to clear fields all over the place to be safe, and
that was deemed not worth the trouble (and it makes life dangerous).

sys/dev/ata/ata-all.c

@@ -40,9 +40,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/bio.h>
 #include <sys/malloc.h>
-#include <sys/sema.h>
 #include <sys/sysctl.h>
+#include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
@@ -77,6 +78,7 @@ static void ata_init(void);
 MALLOC_DEFINE(M_ATA, "ATA generic", "ATA driver generic layer");
 struct intr_config_hook *ata_delayed_attach = NULL;
 devclass_t ata_devclass;
+uma_zone_t ata_zone;
 int ata_wc = 1;	 
 
 /* local vars */
@@ -997,10 +999,13 @@ ata_init(void)
 	free(ata_delayed_attach, M_TEMP);
     }
 
-    /* Register a handler to flush write caches on shutdown */
+    /* register handler to flush write caches on shutdown */
     if ((EVENTHANDLER_REGISTER(shutdown_post_sync, ata_shutdown,
 			       NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
 	printf("ata: shutdown event registration failed!\n");
 
+    /* init our UMA zone for ATA requests */
+    ata_zone = uma_zcreate("ata_request", sizeof(struct ata_request),
+			   NULL, NULL, NULL, NULL, 0, 0);
 }
 SYSINIT(atadev, SI_SUB_DRIVERS, SI_ORDER_SECOND, ata_init, NULL)

sys/dev/ata/ata-all.h

@@ -365,7 +365,6 @@ struct atapi_sense {
 
 /* externs */
 extern devclass_t ata_devclass;
-extern struct intr_config_hook *ata_delayed_attach;
 extern int ata_wc;
  
 /* public prototypes */
@@ -391,8 +390,6 @@ int ata_limit_mode(struct ata_device *atadev, int mode, int maxmode);
 /* ata-queue.c: */
 int ata_reinit(struct ata_channel *ch);
 void ata_start(struct ata_channel *ch);
-struct ata_request *ata_alloc_request(void);
-void ata_free_request(struct ata_request *request);
 int ata_controlcmd(struct ata_device *atadev, u_int8_t command, u_int16_t feature, u_int64_t lba, u_int16_t count);
 int ata_atapicmd(struct ata_device *atadev, u_int8_t *ccb, caddr_t data, int count, int flags, int timeout);
 void ata_queue_request(struct ata_request *request);
@@ -411,6 +408,11 @@ void atapi_cam_attach_bus(struct ata_channel *ch);
 void atapi_cam_detach_bus(struct ata_channel *ch);
 void atapi_cam_reinit_bus(struct ata_channel *ch);
 
+/* macros for alloc/free of ata_requests */
+extern uma_zone_t ata_zone;
+#define ata_alloc_request() uma_zalloc(ata_zone, M_NOWAIT | M_ZERO)
+#define ata_free_request(request) uma_zfree(ata_zone, request)
+
 /* macros for locking a channel */
 #define ATA_LOCK_CH(ch, value) \
 	atomic_cmpset_acq_int(&(ch)->state, ATA_IDLE, (value))

sys/dev/ata/ata-card.c

@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>

sys/dev/ata/ata-cbus.c

@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
 #include <sys/rman.h>

sys/dev/ata/ata-chipset.c

@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>

sys/dev/ata/ata-disk.c

@@ -43,8 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <vm/uma.h>
 #include <machine/md_var.h>
 #include <machine/bus.h>
 #include <sys/rman.h>

sys/dev/ata/ata-dma.c

@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <sys/bus.h>
 #include <machine/bus.h>
 #include <sys/rman.h>

sys/dev/ata/ata-isa.c

@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>

sys/dev/ata/ata-lowlevel.c

@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <dev/ata/ata-all.h>

sys/dev/ata/ata-pci.c

@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/stdarg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>

sys/dev/ata/ata-queue.c

@@ -34,11 +34,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/ata.h>
 #include <sys/kernel.h>
-#include <sys/malloc.h>
 #include <sys/bus.h>
 #include <sys/conf.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <dev/ata/ata-all.h>
@@ -48,36 +48,13 @@ static void ata_completed(void *, int);
 static void ata_timeout(struct ata_request *);
 static char *ata_skey2str(u_int8_t);
 
-/* local vars */
-static MALLOC_DEFINE(M_ATA_REQ, "ATA request", "ATA request");
-
-/*
- * ATA request related functions
- */
-struct ata_request *
-ata_alloc_request(void)
-{
-    struct ata_request *request;
-
-    request = malloc(sizeof(struct ata_request), M_ATA_REQ, M_NOWAIT | M_ZERO);
-    if (!request)
-	printf("FAILURE - malloc ATA request failed\n");
-    sema_init(&request->done, 0, "ATA request done");
-    return request;
-}
-
-void
-ata_free_request(struct ata_request *request)
-{
-    sema_destroy(&request->done);
-    free(request, M_ATA_REQ);
-}
-
 void
 ata_queue_request(struct ata_request *request)
 {
-    /* mark request as virgin (it might be a retry) */
+    /* mark request as virgin */
     request->result = request->status = request->error = 0;
+    if (!request->callback)
+	sema_init(&request->done, 0, "ATA request done");
 
     if (request->device->channel->flags & ATA_IMMEDIATE_MODE) {
 
@@ -122,6 +99,8 @@ ata_queue_request(struct ata_request *request)
 	    sema_wait(&request->done);
 	}
     }
+    if (!request->callback)
+	sema_destroy(&request->done);
 }
 
 int

sys/dev/ata/ata-raid.c

@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kthread.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <geom/geom_disk.h>

sys/dev/ata/atapi-cam.c

@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sema.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 
 #include <cam/cam.h>

sys/dev/ata/atapi-cd.c

@@ -45,8 +45,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/fcntl.h>
 #include <sys/conf.h>
 #include <sys/ctype.h>
-#include <sys/taskqueue.h>
 #include <sys/sema.h>
+#include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <geom/geom.h>
 #include <dev/ata/ata-all.h>
@@ -451,7 +452,6 @@ acd_describe(struct acd_softc *cdp)
 	       (cdp->device->unit == ATA_MASTER) ? "master" : "slave",
 	       ata_mode2str(cdp->device->mode) );
     }
-
 }
 
 static __inline void 

sys/dev/ata/atapi-fd.c

@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/cdio.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <geom/geom_disk.h>
 #include <dev/ata/ata-all.h>

sys/dev/ata/atapi-tape.c

@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/devicestat.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
+#include <vm/uma.h>
 #include <machine/bus.h>
 #include <dev/ata/ata-all.h>
 #include <dev/ata/atapi-tape.h>