lib/vmd: LED management support

This patch adds support for LED management of the devices behind VMD.
It's possible to set four different states: off, on (fault), 1Hz
blinking (rebuild), and 4Hz blinking (identification).

Change-Id: Icce25d0b46c1c4bed803f18201606203980ccc38
Signed-off-by: orden smith <orden.e.smith@intel.com>
Signed-off-by: Konrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/470651
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Wojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

include/spdk/vmd.h

@@ -67,6 +67,35 @@ int spdk_vmd_init(void);
  */
 int spdk_vmd_pci_device_list(struct spdk_pci_addr vmd_addr, struct spdk_pci_device *nvme_list);
 
+/** State of the LEDs */
+enum spdk_vmd_led_state {
+	SPDK_VMD_LED_STATE_OFF,
+	SPDK_VMD_LED_STATE_IDENTIFY,
+	SPDK_VMD_LED_STATE_FAULT,
+	SPDK_VMD_LED_STATE_REBUILD,
+	SPDK_VMD_LED_STATE_UNKNOWN,
+};
+
+/**
+ * Sets the state of the LED on specified PCI device.  The device needs to be behind VMD.
+ *
+ * \param pci_device PCI device
+ * \param state LED state to set
+ *
+ * \return 0 on success, negative errno otherwise
+ */
+int spdk_vmd_set_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state state);
+
+/**
+ * Retrieves the state of the LED on specified PCI device.  The device needs to be behind VMD.
+ *
+ * \param pci_device PCI device
+ * \param state current LED state
+ *
+ * \return 0 on success, negative errno otherwise
+ */
+int spdk_vmd_get_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state *state);
+
 #ifdef __cplusplus
 }
 #endif

lib/vmd/Makefile

@@ -34,7 +34,7 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-C_SRCS = vmd.c
+C_SRCS = vmd.c led.c
 LIBNAME = vmd
 
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

lib/vmd/led.c

@@ -0,0 +1,166 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+#include "spdk/likely.h"
+#include "spdk/log.h"
+#include "vmd.h"
+
+struct vmd_led_indicator_config {
+	uint8_t attention_indicator	: 2;
+	uint8_t power_indicator		: 2;
+	uint8_t reserved		: 4;
+};
+
+/*
+ * VMD LED     Attn       Power       LED Amber
+ * State       Indicator  Indicator
+ *             Control    Control
+ * ------------------------------------------------
+ * Off         11b        11b         Off
+ * Ident       11b        01b         Blink 4Hz
+ * Fault       01b        11b         On
+ * Rebuild     01b        01b         Blink 1Hz
+ */
+static const struct vmd_led_indicator_config g_led_config[] = {
+	[SPDK_VMD_LED_STATE_OFF]	= { .attention_indicator = 3, .power_indicator = 3 },
+	[SPDK_VMD_LED_STATE_IDENTIFY]	= { .attention_indicator = 3, .power_indicator = 1 },
+	[SPDK_VMD_LED_STATE_FAULT]	= { .attention_indicator = 1, .power_indicator = 3 },
+	[SPDK_VMD_LED_STATE_REBUILD]	= { .attention_indicator = 1, .power_indicator = 1 },
+};
+
+static void
+vmd_led_set_indicator_control(struct vmd_pci_device *vmd_device, enum spdk_vmd_led_state state)
+{
+	const struct vmd_led_indicator_config *config;
+	union express_slot_control_register slot_control;
+
+	assert(state >= SPDK_VMD_LED_STATE_OFF && state <= SPDK_VMD_LED_STATE_REBUILD);
+	config = &g_led_config[state];
+
+	slot_control = vmd_device->pcie_cap->slot_control;
+	slot_control.bit_field.attention_indicator_control = config->attention_indicator;
+	slot_control.bit_field.power_indicator_control = config->power_indicator;
+
+	/*
+	 * Due to the fact that writes to the PCI config space are posted writes, we need to issue
+	 * a read to the register we've just written to ensure it reached its destination.
+	 * TODO: wrap all register writes with a function taking care of that.
+	 */
+	vmd_device->pcie_cap->slot_control = slot_control;
+	vmd_device->cached_slot_control = vmd_device->pcie_cap->slot_control;
+}
+
+static unsigned int
+vmd_led_get_state(struct vmd_pci_device *vmd_device)
+{
+	const struct vmd_led_indicator_config *config;
+	union express_slot_control_register slot_control;
+	unsigned int state;
+
+	slot_control = vmd_device->cached_slot_control;
+	for (state = SPDK_VMD_LED_STATE_OFF; state <= SPDK_VMD_LED_STATE_REBUILD; ++state) {
+		config = &g_led_config[state];
+
+		if (slot_control.bit_field.attention_indicator_control == config->attention_indicator &&
+		    slot_control.bit_field.power_indicator_control == config->power_indicator) {
+			return state;
+		}
+	}
+
+	return SPDK_VMD_LED_STATE_UNKNOWN;
+}
+
+/*
+ * The identifying device under VMD is located in the global list of VMD controllers.  If the BDF
+ * identifies an endpoint, then the LED is attached to the endpoint's parent.  If the BDF identifies
+ * a type 1 header, then this device has the corresponding LED. This may arise when a user wants to
+ * identify a given empty slot under VMD.
+ */
+static struct vmd_pci_device *
+vmd_get_led_device(const struct spdk_pci_device *pci_device)
+{
+	struct vmd_pci_device *vmd_device;
+
+	assert(strcmp(spdk_pci_device_get_type(pci_device), "vmd") == 0);
+
+	vmd_device = vmd_find_device(&pci_device->addr);
+	if (spdk_unlikely(vmd_device == NULL)) {
+		return NULL;
+	}
+
+	if (vmd_device->header_type == PCI_HEADER_TYPE_NORMAL) {
+		if (spdk_unlikely(vmd_device->parent == NULL)) {
+			return NULL;
+		}
+
+		return vmd_device->parent->self;
+	}
+
+	return vmd_device;
+}
+
+int
+spdk_vmd_set_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state state)
+{
+	struct vmd_pci_device *vmd_device;
+
+	if (state < SPDK_VMD_LED_STATE_OFF || state > SPDK_VMD_LED_STATE_REBUILD) {
+		SPDK_ERRLOG("Invalid LED state\n");
+		return -EINVAL;
+	}
+
+	vmd_device = vmd_get_led_device(pci_device);
+	if (spdk_unlikely(vmd_device == NULL)) {
+		SPDK_ERRLOG("The PCI device is not behind the VMD\n");
+		return -ENODEV;
+	}
+
+	vmd_led_set_indicator_control(vmd_device, state);
+	return 0;
+}
+
+int
+spdk_vmd_get_led_state(struct spdk_pci_device *pci_device, enum spdk_vmd_led_state *state)
+{
+	struct vmd_pci_device *vmd_device;
+
+	vmd_device = vmd_get_led_device(pci_device);
+	if (spdk_unlikely(vmd_device == NULL)) {
+		SPDK_ERRLOG("The PCI device is not behind the VMD\n");
+		return -ENODEV;
+	}
+
+	*state = (enum spdk_vmd_led_state)vmd_led_get_state(vmd_device);
+	return 0;
+}

lib/vmd/vmd.c

@@ -687,6 +687,7 @@ vmd_dev_init(struct vmd_pci_device *dev)
 	dev->pci.cfg_read = vmd_dev_cfg_read;
 	dev->pci.cfg_write = vmd_dev_cfg_write;
 	dev->pci.detach = vmd_dev_detach;
+	dev->cached_slot_control = dev->pcie_cap->slot_control;
 
 	if (vmd_is_supported_device(dev)) {
 		spdk_pci_addr_fmt(bdf, sizeof(bdf), &dev->pci.addr);
@@ -923,6 +924,32 @@ vmd_enumerate_devices(struct vmd_adapter *vmd)
 	return vmd_scan_pcibus(&vmd->vmd_bus);
 }
 
+struct vmd_pci_device *
+vmd_find_device(const struct spdk_pci_addr *addr)
+{
+	struct vmd_pci_bus *bus;
+	struct vmd_pci_device *dev;
+	int i;
+
+	for (i = 0; i < MAX_VMD_TARGET; ++i) {
+		for (bus = g_vmd_container.vmd[i].bus_list; bus != NULL; bus = bus->next) {
+			if (bus->self) {
+				if (spdk_pci_addr_compare(&bus->self->pci.addr, addr) == 0) {
+					return bus->self;
+				}
+			}
+
+			for (dev = bus->dev_list; dev != NULL; dev = dev->next) {
+				if (spdk_pci_addr_compare(&dev->pci.addr, addr) == 0) {
+					return dev;
+				}
+			}
+		}
+	}
+
+	return NULL;
+}
+
 static int
 vmd_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)
 {

lib/vmd/vmd.h

@@ -101,6 +101,8 @@ struct vmd_pci_device {
 	uint32_t  target         : 16;
 
 	struct vmd_hot_plug *hp;
+	/* Cached version of the slot_control register */
+	union express_slot_control_register cached_slot_control;
 };
 
 
@@ -198,4 +200,6 @@ vmd_hp_get_next_bus_number(struct vmd_hot_plug *hp)
 	return 0;
 }
 
+struct vmd_pci_device *vmd_find_device(const struct spdk_pci_addr *addr);
+
 #endif /* VMD_H */

lib/vmd/vmd_spec.h

@@ -378,7 +378,7 @@ union express_root_control_register {
 		uint16_t Rsvd : 11;
 	} bit_field;
 	uint16_t as_uint16_t;
-} express_root_control_register;
+};
 
 struct pci_express_cap {
 	uint8_t capid;