From e2d3cc650288590fc223438a77e79dcb9a931fe4 Mon Sep 17 00:00:00 2001
From: Daniel Verkamp <daniel.verkamp@intel.com>
Date: Mon, 27 Jun 2016 09:49:30 -0700
Subject: [PATCH] nvme: replace rte_memcpy with specialized function

Use the knowledge that both the source and destination of
nvme_copy_command() are aligned to emit the aligned variants of the
SSE2/AVX mov instructions.

Change-Id: I0a7e32a3bb10b9a1920cd85691b79fa7172eecb3
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
---
 examples/nvme/fio_plugin/fio_plugin.c |  1 -
 lib/nvme/nvme_impl.h                  |  6 ------
 lib/nvme/nvme_qpair.c                 | 23 +++++++++++++++++++++++
 test/lib/nvme/unit/nvme_impl.h        |  5 -----
 4 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c
index b92c83ba39..36463b38e3 100644
--- a/examples/nvme/fio_plugin/fio_plugin.c
+++ b/examples/nvme/fio_plugin/fio_plugin.c
@@ -42,7 +42,6 @@
 #include "rte_mempool.h"
 #include "rte_malloc.h"
 #include "rte_eal.h"
-#include "rte_memcpy.h"
 
 #include "spdk/nvme.h"
 #include "spdk/pci.h"
diff --git a/lib/nvme/nvme_impl.h b/lib/nvme/nvme_impl.h
index bb78a21e99..3bb64feee7 100644
--- a/lib/nvme/nvme_impl.h
+++ b/lib/nvme/nvme_impl.h
@@ -54,7 +54,6 @@
 #include <rte_cycles.h>
 #include <rte_malloc.h>
 #include <rte_mempool.h>
-#include <rte_memcpy.h>
 
 #ifdef USE_PCIACCESS
 #include <pciaccess.h>
@@ -327,9 +326,4 @@ nvme_mutex_init_recursive(nvme_mutex_t *mtx)
 	return rc;
 }
 
-/**
- * Copy a struct nvme_command from one memory location to another.
- */
-#define nvme_copy_command(dst, src)	rte_memcpy((dst), (src), sizeof(struct spdk_nvme_cmd))
-
 #endif /* __NVME_IMPL_H__ */
diff --git a/lib/nvme/nvme_qpair.c b/lib/nvme/nvme_qpair.c
index 480d919491..6bd8bf80aa 100644
--- a/lib/nvme/nvme_qpair.c
+++ b/lib/nvme/nvme_qpair.c
@@ -290,6 +290,29 @@ nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phy
 	tr->active = false;
 }
 
+static inline void
+nvme_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)
+{
+	/* dst and src are known to be non-overlapping and 64-byte aligned. */
+#if defined(__AVX__)
+	__m256i *d256 = (__m256i *)dst;
+	const __m256i *s256 = (const __m256i *)src;
+
+	_mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0]));
+	_mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1]));
+#elif defined(__SSE2__)
+	__m128i *d128 = (__m128i *)dst;
+	const __m128i *s128 = (const __m128i *)src;
+
+	_mm_store_si128(&d128[0], _mm_load_si128(&s128[0]));
+	_mm_store_si128(&d128[1], _mm_load_si128(&s128[1]));
+	_mm_store_si128(&d128[2], _mm_load_si128(&s128[2]));
+	_mm_store_si128(&d128[3], _mm_load_si128(&s128[3]));
+#else
+	*dst = *src;
+#endif
+}
+
 static void
 nvme_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
 {
diff --git a/test/lib/nvme/unit/nvme_impl.h b/test/lib/nvme/unit/nvme_impl.h
index 0ba92b1226..cd17ef4ae9 100644
--- a/test/lib/nvme/unit/nvme_impl.h
+++ b/test/lib/nvme/unit/nvme_impl.h
@@ -153,9 +153,4 @@ nvme_mutex_init_recursive(nvme_mutex_t *mtx)
 	return rc;
 }
 
-/**
- * Copy a struct nvme_command from one memory location to another.
- */
-#define nvme_copy_command(dst, src)	memcpy((dst), (src), sizeof(struct spdk_nvme_cmd))
-
 #endif /* __NVME_IMPL_H__ */