diff --git a/include/spdk/mmio.h b/include/spdk/mmio.h
new file mode 100644
index 0000000000..6dd9b64953
--- /dev/null
+++ b/include/spdk/mmio.h
@@ -0,0 +1,91 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPDK_MMIO_H
+#define SPDK_MMIO_H
+
+#include <inttypes.h>
+
+#ifdef __x86_64__
+#define SPDK_MMIO_64BIT	1 /* Can do atomic 64-bit memory read/write (over PCIe) */
+#else
+#define SPDK_MMIO_64BIT	0
+#endif
+
+static inline uint32_t
+spdk_mmio_read_4(const volatile uint32_t *addr)
+{
+	return *addr;
+}
+
+static inline void
+spdk_mmio_write_4(volatile uint32_t *addr, uint32_t val)
+{
+	*addr = val;
+}
+
+static inline uint64_t
+spdk_mmio_read_8(volatile uint64_t *addr)
+{
+	uint64_t val;
+	volatile uint32_t *addr32 = (volatile uint32_t *)addr;
+
+	if (SPDK_MMIO_64BIT) {
+		val = *addr;
+	} else {
+		/*
+		 * Read lower 4 bytes before upper 4 bytes.
+		 * This particular order is required by I/OAT.
+		 * If the other order is required, use a pair of spdk_mmio_read_4() calls.
+		 */
+		val = addr32[0];
+		val |= (uint64_t)addr32[1] << 32;
+	}
+
+	return val;
+}
+
+static inline void
+spdk_mmio_write_8(volatile uint64_t *addr, uint64_t val)
+{
+	volatile uint32_t *addr32 = (volatile uint32_t *)addr;
+
+	if (SPDK_MMIO_64BIT) {
+		*addr = val;
+	} else {
+		addr32[0] = (uint32_t)val;
+		addr32[1] = (uint32_t)(val >> 32);
+	}
+}
+
+#endif
diff --git a/lib/ioat/ioat.c b/lib/ioat/ioat.c
index 36b58c9ad7..9166248b6b 100644
--- a/lib/ioat/ioat.c
+++ b/lib/ioat/ioat.c
@@ -116,53 +116,22 @@ ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id)
 	return false;
 }
 
-
-static uint64_t
-ioat_mmio_read_8(volatile uint64_t *addr)
-{
-	uint64_t val;
-	volatile uint32_t *addr32 = (volatile uint32_t *)addr;
-
-	if (IOAT_64BIT_IO) {
-		val = *addr;
-	} else {
-		/* Must read lower 4 bytes before upper 4 bytes. */
-		val = addr32[0];
-		val |= (uint64_t)addr32[1] << 32;
-	}
-
-	return val;
-}
-
-static void
-ioat_mmio_write_8(volatile uint64_t *addr, uint64_t val)
-{
-	volatile uint32_t *addr32 = (volatile uint32_t *)addr;
-
-	if (IOAT_64BIT_IO) {
-		*addr = val;
-	} else {
-		addr32[0] = (uint32_t)val;
-		addr32[1] = (uint32_t)(val >> 32);
-	}
-}
-
 static uint64_t
 ioat_get_chansts(struct ioat_channel *ioat)
 {
-	return ioat_mmio_read_8(&ioat->regs->chansts);
+	return spdk_mmio_read_8(&ioat->regs->chansts);
 }
 
 static void
 ioat_write_chancmp(struct ioat_channel *ioat, uint64_t addr)
 {
-	ioat_mmio_write_8(&ioat->regs->chancmp, addr);
+	spdk_mmio_write_8(&ioat->regs->chancmp, addr);
 }
 
 static void
 ioat_write_chainaddr(struct ioat_channel *ioat, uint64_t addr)
 {
-	ioat_mmio_write_8(&ioat->regs->chainaddr, addr);
+	spdk_mmio_write_8(&ioat->regs->chainaddr, addr);
 }
 
 static inline void
diff --git a/lib/ioat/ioat_internal.h b/lib/ioat/ioat_internal.h
index c8739c56d8..e66605e540 100644
--- a/lib/ioat/ioat_internal.h
+++ b/lib/ioat/ioat_internal.h
@@ -44,16 +44,11 @@
 #include <inttypes.h>
 
 #include "spdk/queue.h"
+#include "spdk/mmio.h"
 
 /* Allocate 2 << 15 (32K) descriptors per channel by default. */
 #define IOAT_DEFAULT_ORDER			15
 
-#ifdef __x86_64__
-#define IOAT_64BIT_IO	1 /* Can do atomic 64-bit memory read/write (over PCIe) */
-#else
-#define IOAT_64BIT_IO	0
-#endif
-
 struct ioat_descriptor {
 	ioat_callback_t		callback_fn;
 	void			*callback_arg;
diff --git a/lib/nvme/nvme_internal.h b/lib/nvme/nvme_internal.h
index 219414eb5a..be90fb7505 100644
--- a/lib/nvme/nvme_internal.h
+++ b/lib/nvme/nvme_internal.h
@@ -51,6 +51,7 @@
 
 #include "spdk/queue.h"
 #include "spdk/barrier.h"
+#include "spdk/mmio.h"
 
 #define NVME_MAX_PRP_LIST_ENTRIES	(32)
 
@@ -304,32 +305,14 @@ extern struct nvme_driver g_nvme_driver;
 
 #define INTEL_DC_P3X00_DEVID	0x09538086
 
-static inline uint32_t
-_nvme_mmio_read_4(const volatile uint32_t *addr)
-{
-	return *addr;
-}
-
-static inline void
-_nvme_mmio_write_4(volatile uint32_t *addr, uint32_t val)
-{
-	*addr = val;
-}
-
-static inline void
-_nvme_mmio_write_8(volatile uint64_t *addr, uint64_t val)
-{
-	*addr = val;
-}
-
 #define nvme_mmio_read_4(sc, reg) \
-	_nvme_mmio_read_4(&(sc)->regs->reg)
+	spdk_mmio_read_4(&(sc)->regs->reg)
 
 #define nvme_mmio_write_4(sc, reg, val) \
-	_nvme_mmio_write_4(&(sc)->regs->reg, val)
+	spdk_mmio_write_4(&(sc)->regs->reg, val)
 
 #define nvme_mmio_write_8(sc, reg, val) \
-	_nvme_mmio_write_8(&(sc)->regs->reg, val)
+	spdk_mmio_write_8(&(sc)->regs->reg, val)
 
 #define nvme_delay		usleep
 
diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c
index 61fa8dfde6..ae315871f7 100644
--- a/lib/nvme/nvme_qpair.c
+++ b/lib/nvme/nvme_qpair.c
@@ -480,7 +480,7 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair, uint32_t max_completion
 			qpair->phase = !qpair->phase;
 		}
 
-		_nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
+		spdk_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);
 
 		if (max_completions > 0 && --max_completions == 0) {
 			break;
@@ -635,7 +635,7 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr)
 	}
 
 	wmb();
-	_nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
+	spdk_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);
 }
 
 static void