docker: Add virtio traffic generator

Expand the traffic generator container into two:
- traffic-generator-nvme, which uses NVMe-oF to connect
to proxy-container
- traffic-generator-virtio, which uses Virtio to connect
to proxy-container

Added second device Malloc device in storage-target,
and second subsystem shared between storage-target and
proxy-container.
The proxy-container and traffic-generator-virtio share
named volume in order to pass the vhost socket file.

Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: I889dc19f523255f10b22e15f5e5f437b33ae796d

Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9667
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Xiaodong Liu <xiaodong.liu@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>

docker/README.md

@@ -30,12 +30,13 @@ docker-compose build --build-arg PROXY=$http_proxy
 ## How-To
 
 `docker-compose.yaml` shows an example deployment of the storage containers based on SPDK.
-Running `docker-compose build` creates 4 docker images:
+Running `docker-compose build` creates 5 docker images:
 
 - build_base
 - storage-target
 - proxy-container
-- traffic-generator
+- traffic-generator-nvme
+- traffic-generator-virtio
 
 The `build_base` image provides the core components required to containerize SPDK
 applications. The fedora:33 image from the Fedora Container Registry is used and then SPDK is installed. SPDK is installed out of `build_base/spdk.tar.gz` provided.
@@ -46,8 +47,11 @@ Running `docker-compose up` creates 3 docker containers:
 -- storage-target: Contains SPDK NVMe-oF target exposing single subsystem to
 `proxy-container` based on malloc bdev.
 -- proxy-container: Contains SPDK NVMe-oF target connecting to `storage-target`
-and then exposing the same subsystem to `traffic-generator`.
--- traffic-generator: Contains FIO using SPDK plugin to connect to `proxy-container`
+and then exposing the same devices to `traffic-generator-nvme` using NVMe-oF and
+to `traffic-generator-virtio` using Virtio.
+-- traffic-generator-nvme: Contains FIO using SPDK plugin to connect to `proxy-container`
+and runs a sample workload.
+-- traffic-generator-virtio: Contains FIO using SPDK plugin to connect to `proxy-container`
 and runs a sample workload.
 
 Each container is connected to a separate "spdk" network which is created before
@@ -63,11 +67,12 @@ docker-compose up
 ~~~
 
 The `storage-target` and `proxy-container` can be started as services.
-Allowing for multiple `traffic-generator` containers to connect.
+Allowing for multiple traffic generator containers to connect.
 
 ~~~{.sh}
 docker-compose up -d proxy-container
-docker-compose run traffic-generator
+docker-compose run traffic-generator-nvme
+docker-compose run traffic-generator-virtio
 ~~~
 
 Enviroment variables to containers can be passed as shown in
@@ -75,7 +80,7 @@ Enviroment variables to containers can be passed as shown in
 For example extra arguments to fio can be passed as so:
 
 ~~~{.sh}
-docker-compose run -e FIO_ARGS="--minimal" traffic-generator
+docker-compose run -e FIO_ARGS="--minimal" traffic-generator-nvme
 ~~~
 
 As each container includes SPDK installation it is possible to use rpc.py to

docker/build_base/pre-install

@@ -36,6 +36,7 @@ dnf install -y rpm-build
 # Deploy SPDK inside the container
 DEPS="no" "$spdk_repo/rpmbuild/rpm.sh" \
 	--with-shared \
+	--with-virtio \
 	--with-fio
 
 mv "$HOME/rpmbuild/rpm/x86_64/"*.rpm /tmp

docker/docker-compose.yaml

@@ -36,22 +36,38 @@ services:
     volumes:
       - /dev/hugepages:/dev/hugepages
       - ./spdk-app/proxy-container.conf:/config
+      - vhost-user:/vhost-user
     environment:
-      - SPDK_ARGS=-m 0x4
+      - SPDK_ARGS=-m 0x4 -S /vhost-user
     privileged: true
-  traffic-generator:
+  traffic-generator-virtio:
     image: traffic-generator
     build:
       context: traffic-generator
-    container_name: traffic-generator
+    container_name: traffic-generator-virtio
     depends_on:
       - proxy-container
     networks:
       spdk:
     volumes:
       - /dev/hugepages:/dev/hugepages
-      - ./traffic-generator/conf:/config
-      - ./traffic-generator/fio.conf:/fio.conf
+      - ./traffic-generator/conf-virtio:/config
+      - vhost-user:/vhost-user
+      - ./traffic-generator/fio-virtio.conf:/fio.conf
+    privileged: true
+  traffic-generator-nvme:
+    image: traffic-generator
+    build:
+      context: traffic-generator
+    container_name: traffic-generator-nvme
+    depends_on:
+      - proxy-container
+    networks:
+      spdk:
+    volumes:
+      - /dev/hugepages:/dev/hugepages
+      - ./traffic-generator/conf-nvme:/config
+      - ./traffic-generator/fio-nvme.conf:/fio.conf
     privileged: true
 networks:
   spdk:
@@ -60,3 +76,5 @@ networks:
       config:
         - subnet: 192.168.42.0/29
           gateway: 192.168.42.1
+volumes:
+  vhost-user:

docker/spdk-app/proxy-container.conf

@@ -63,6 +63,18 @@
           }
         }
       ]
+    },
+    {
+      "subsystem": "vhost",
+      "config": [
+        {
+          "method": "vhost_create_blk_controller",
+          "params": {
+            "ctrlr": "VirtioBlk0",
+            "dev_name": "Nvme0n2"
+          }
+        }
+      ]
     }
   ]
 }

docker/spdk-app/storage-target.conf

@@ -10,6 +10,14 @@
             "num_blocks": 131072,
             "block_size": 512
           }
+        },
+        {
+          "method": "bdev_malloc_create",
+          "params": {
+            "name": "Malloc1",
+            "num_blocks": 131072,
+            "block_size": 512
+          }
         }
       ]
     },
@@ -45,6 +53,16 @@
             }
           }
         },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:cnode1",
+            "namespace": {
+              "nsid": 2,
+              "bdev_name": "Malloc1"
+            }
+          }
+        },
         {
           "method": "nvmf_subsystem_add_listener",
           "params": {

docker/traffic-generator/conf-virtio

@@ -0,0 +1,18 @@
+{
+  "subsystems": [
+    {
+      "subsystem": "bdev",
+      "config": [
+        {
+          "method": "bdev_virtio_attach_controller",
+          "params": {
+            "name": "VirtioBlk0",
+            "trtype": "user",
+            "traddr": "/vhost-user/VirtioBlk0",
+            "dev_type": "blk"
+          }
+        }
+      ]
+    }
+  ]
+}

docker/traffic-generator/fio-nvme.conf

@@ -1,6 +1,7 @@
 [global]
 ioengine=spdk_bdev
 spdk_json_conf=/config
+spdk_mem=1024
 thread=1
 direct=1
 rw=randread

docker/traffic-generator/fio-virtio.conf

@@ -0,0 +1,18 @@
+[global]
+ioengine=spdk_bdev
+spdk_json_conf=/config
+spdk_single_seg=1
+spdk_mem=1024
+thread=1
+direct=1
+rw=randread
+ramp_time=0
+norandommap=1
+time_based=1
+bs=4k
+numjobs=1
+runtime=10
+
+[filename0]
+filename=VirtioBlk0
+iodepth=128