eal: handle compressed firmware

Introduce an internal firmware loading helper to remove code duplication
in our drivers and handle xz compressed firmware by calling libarchive.

This helper tries to look for .xz suffixes so that drivers are not aware
the firmware has been compressed.

libarchive is set as an optional dependency: without libarchive, a
runtime warning is emitted so that users know there is a compressed
firmware.

Windows implementation is left as an empty stub.

Signed-off-by: David Marchand <david.marchand@redhat.com>
Reviewed-by: Igor Russkikh <irusskikh@marvell.com>
Acked-by: Aaron Conole <aconole@redhat.com>
Tested-by: Haiyue Wang <haiyue.wang@intel.com>

.github/workflows/build.yml

@@ -93,6 +93,7 @@ jobs:
       run: sudo apt install -y ccache libnuma-dev python3-setuptools
         python3-wheel python3-pip python3-pyelftools ninja-build libbsd-dev
         libpcap-dev libibverbs-dev libcrypto++-dev libfdt-dev libjansson-dev
+        libarchive-dev
     - name: Install libabigail build dependencies if no cache is available
       if: env.ABI_CHECKS == 'true' && steps.libabigail-cache.outputs.cache-hit != 'true'
       run: sudo apt install -y autoconf automake libtool pkg-config libxml2-dev

.travis.yml

@@ -16,6 +16,7 @@ addons:
     packages: &required_packages
       - [libnuma-dev, python3-setuptools, python3-wheel, python3-pip, python3-pyelftools, ninja-build]
       - [libbsd-dev, libpcap-dev, libibverbs-dev, libcrypto++-dev, libfdt-dev, libjansson-dev]
+      - [libarchive-dev]
 
 _aarch64_packages: &aarch64_packages
   - *required_packages

config/meson.build

@@ -172,6 +172,16 @@ if libexecinfo.found() and cc.has_header('execinfo.h')
     dpdk_extra_ldflags += '-lexecinfo'
 endif
 
+libarchive = dependency('libarchive', required: false, method: 'pkg-config')
+if libarchive.found()
+    dpdk_conf.set('RTE_HAS_LIBARCHIVE', 1)
+    # Push libarchive link dependency at the project level to support
+    # statically linking dpdk apps. Details at:
+    # https://inbox.dpdk.org/dev/20210605004024.660267a1@sovereign/
+    add_project_link_arguments('-larchive', language: 'c')
+    dpdk_extra_ldflags += '-larchive'
+endif
+
 # check for libbsd
 libbsd = dependency('libbsd', required: false, method: 'pkg-config')
 if libbsd.found()

doc/guides/rel_notes/release_21_08.rst

@@ -61,6 +61,12 @@ New Features
   representing sub-domains of functionality. Each auxiliary device
   represents a part of its parent functionality.
 
+* **Added XZ compressed firmware support.**
+
+  Using ``rte_firmware_read``, a driver can now handle XZ compressed firmware
+  in a transparent way, with EAL uncompressing using libarchive if this library
+  is available when building DPDK.
+
 * **Updated Intel iavf driver.**
 
   * Added Tx QoS VF queue TC mapping.

drivers/net/bnx2x/bnx2x.c

@@ -26,9 +26,12 @@
 #include <arpa/inet.h>
 #include <fcntl.h>
 #include <zlib.h>
+
 #include <rte_bitops.h>
 #include <rte_string_fns.h>
 
+#include "eal_firmware.h"
+
 #define BNX2X_PMD_VER_PREFIX "BNX2X PMD"
 #define BNX2X_PMD_VERSION_MAJOR 1
 #define BNX2X_PMD_VERSION_MINOR 1
@@ -9655,44 +9658,33 @@ static void bnx2x_init_rte(struct bnx2x_softc *sc)
 void bnx2x_load_firmware(struct bnx2x_softc *sc)
 {
 	const char *fwname;
-	int f;
+	void *buf;
-	struct stat st;
+	size_t bufsz;
 
 	fwname = sc->devinfo.device_id == CHIP_NUM_57711
 		? FW_NAME_57711 : FW_NAME_57810;
-	f = open(fwname, O_RDONLY);
+	if (rte_firmware_read(fwname, &buf, &bufsz) != 0) {
-	if (f < 0) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't open firmware file");
 		return;
 	}
 
-	if (fstat(f, &st) < 0) {
+	sc->firmware = rte_zmalloc("bnx2x_fw", bufsz, RTE_CACHE_LINE_SIZE);
-		PMD_DRV_LOG(NOTICE, sc, "Can't stat firmware file");
-		close(f);
-		return;
-	}
-
-	sc->firmware = rte_zmalloc("bnx2x_fw", st.st_size, RTE_CACHE_LINE_SIZE);
 	if (!sc->firmware) {
 		PMD_DRV_LOG(NOTICE, sc, "Can't allocate memory for firmware");
-		close(f);
+		goto out;
-		return;
 	}
 
-	if (read(f, sc->firmware, st.st_size) != st.st_size) {
+	sc->fw_len = bufsz;
-		PMD_DRV_LOG(NOTICE, sc, "Can't read firmware data");
-		close(f);
-		return;
-	}
-	close(f);
-
-	sc->fw_len = st.st_size;
 	if (sc->fw_len < FW_HEADER_LEN) {
 		PMD_DRV_LOG(NOTICE, sc,
 			    "Invalid fw size: %" PRIu64, sc->fw_len);
-		return;
+		goto out;
 	}
+
+	memcpy(sc->firmware, buf, sc->fw_len);
 	PMD_DRV_LOG(DEBUG, sc, "fw_len = %" PRIu64, sc->fw_len);
+out:
+	free(buf);
 }
 
 static void

drivers/net/ice/ice_ethdev.c

@@ -12,6 +12,8 @@
 
 #include <rte_tailq.h>
 
+#include "eal_firmware.h"
+
 #include "base/ice_sched.h"
 #include "base/ice_flow.h"
 #include "base/ice_dcb.h"
@@ -1675,22 +1677,14 @@ ice_load_pkg_type(struct ice_hw *hw)
 	return package_type;
 }
 
-#ifdef RTE_EXEC_ENV_WINDOWS
-#define ice_access _access
-#else
-#define ice_access access
-#endif
-
 int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 {
 	struct ice_hw *hw = &adapter->hw;
 	char pkg_file[ICE_MAX_PKG_FILENAME_SIZE];
 	char opt_ddp_filename[ICE_MAX_PKG_FILENAME_SIZE];
+	void *buf;
+	size_t bufsz;
 	int err;
-	uint8_t *buf = NULL;
-	int buf_len;
-	FILE *file;
-	struct stat fstat;
 
 	if (!use_dsn)
 		goto no_dsn;
@@ -1700,57 +1694,31 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 		"ice-%016" PRIx64 ".pkg", dsn);
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_UPDATES,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 	strncpy(pkg_file, ICE_PKG_FILE_SEARCH_PATH_DEFAULT,
 		ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(strcat(pkg_file, opt_ddp_filename), 0))
+	strcat(pkg_file, opt_ddp_filename);
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
 
 no_dsn:
 	strncpy(pkg_file, ICE_PKG_FILE_UPDATES, ICE_MAX_PKG_FILENAME_SIZE);
-	if (!ice_access(pkg_file, 0))
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) == 0)
 		goto load_fw;
+
 	strncpy(pkg_file, ICE_PKG_FILE_DEFAULT, ICE_MAX_PKG_FILENAME_SIZE);
-	if (ice_access(pkg_file, 0)) {
+	if (rte_firmware_read(pkg_file, &buf, &bufsz) < 0) {
 		PMD_INIT_LOG(ERR, "failed to search file path\n");
 		return -1;
 	}
 
 load_fw:
-	file = fopen(pkg_file, "rb");
-	if (!file)  {
-		PMD_INIT_LOG(ERR, "failed to open file: %s\n", pkg_file);
-		return -1;
-	}
-
 	PMD_INIT_LOG(DEBUG, "DDP package name: %s", pkg_file);
 
-	err = stat(pkg_file, &fstat);
+	err = ice_copy_and_init_pkg(hw, buf, bufsz);
-	if (err) {
-		PMD_INIT_LOG(ERR, "failed to get file stats\n");
-		goto out;
-	}
-
-	buf_len = fstat.st_size;
-	buf = rte_malloc(NULL, buf_len, 0);
-
-	if (!buf) {
-		PMD_INIT_LOG(ERR, "failed to allocate buf of size %d for package\n",
-				buf_len);
-		err = -1;
-		goto out;
-	}
-
-	err = fread(buf, buf_len, 1, file);
-	if (err != 1) {
-		PMD_INIT_LOG(ERR, "failed to read package data\n");
-		err = -1;
-		goto out;
-	}
-
-	err = ice_copy_and_init_pkg(hw, buf, buf_len);
 	if (err) {
 		PMD_INIT_LOG(ERR, "ice_copy_and_init_hw failed: %d\n", err);
 		goto out;
@@ -1760,13 +1728,10 @@ int ice_load_pkg(struct ice_adapter *adapter, bool use_dsn, uint64_t dsn)
 	adapter->active_pkg_type = ice_load_pkg_type(hw);
 
 out:
-	fclose(file);
+	free(buf);
-	rte_free(buf);
 	return err;
 }
 
-#undef ice_access
-
 static void
 ice_base_queue_get(struct ice_pf *pf)
 {

drivers/net/nfp/nfp_net.c

@@ -30,6 +30,8 @@
 #include <rte_spinlock.h>
 #include <rte_service_component.h>
 
+#include "eal_firmware.h"
+
 #include "nfpcore/nfp_cpp.h"
 #include "nfpcore/nfp_nffw.h"
 #include "nfpcore/nfp_hwinfo.h"
@@ -3366,12 +3368,10 @@ static int
 nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
-	int fw_f;
+	void *fw_buf;
-	char *fw_buf;
 	char fw_name[125];
 	char serial[40];
-	struct stat file_stat;
+	size_t fsize;
-	off_t fsize, bytes;
 
 	/* Looking for firmware file in order of priority */
 
@@ -3384,66 +3384,34 @@ nfp_fw_upload(struct rte_pci_device *dev, struct nfp_nsp *nsp, char *card)
 
 	snprintf(fw_name, sizeof(fw_name), "%s/%s.nffw", DEFAULT_FW_PATH,
 			serial);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
-	if (fw_f >= 0)
+		goto load_fw;
-		goto read_fw;
 
 	/* Then try the PCI name */
 	snprintf(fw_name, sizeof(fw_name), "%s/pci-%s.nffw", DEFAULT_FW_PATH,
 			dev->device.name);
-
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) == 0)
-	if (fw_f >= 0)
+		goto load_fw;
-		goto read_fw;
 
 	/* Finally try the card type and media */
 	snprintf(fw_name, sizeof(fw_name), "%s/%s", DEFAULT_FW_PATH, card);
 	PMD_DRV_LOG(DEBUG, "Trying with fw file: %s", fw_name);
-	fw_f = open(fw_name, O_RDONLY);
+	if (rte_firmware_read(fw_name, &fw_buf, &fsize) < 0) {
-	if (fw_f < 0) {
 		PMD_DRV_LOG(INFO, "Firmware file %s not found.", fw_name);
 		return -ENOENT;
 	}
 
-read_fw:
+load_fw:
-	if (fstat(fw_f, &file_stat) < 0) {
+	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %zu",
-		PMD_DRV_LOG(INFO, "Firmware file %s size is unknown", fw_name);
+		fw_name, fsize);
-		close(fw_f);
-		return -ENOENT;
-	}
-
-	fsize = file_stat.st_size;
-	PMD_DRV_LOG(INFO, "Firmware file found at %s with size: %" PRIu64 "",
-			    fw_name, (uint64_t)fsize);
-
-	fw_buf = malloc((size_t)fsize);
-	if (!fw_buf) {
-		PMD_DRV_LOG(INFO, "malloc failed for fw buffer");
-		close(fw_f);
-		return -ENOMEM;
-	}
-	memset(fw_buf, 0, fsize);
-
-	bytes = read(fw_f, fw_buf, fsize);
-	if (bytes != fsize) {
-		PMD_DRV_LOG(INFO, "Reading fw to buffer failed."
-				   "Just %" PRIu64 " of %" PRIu64 " bytes read",
-				   (uint64_t)bytes, (uint64_t)fsize);
-		free(fw_buf);
-		close(fw_f);
-		return -EIO;
-	}
 
 	PMD_DRV_LOG(INFO, "Uploading the firmware ...");
-	nfp_nsp_load_fw(nsp, fw_buf, bytes);
+	nfp_nsp_load_fw(nsp, fw_buf, fsize);
 	PMD_DRV_LOG(INFO, "Done");
 
 	free(fw_buf);
-	close(fw_f);
-
 	return 0;
 }
 

drivers/net/qede/qede_main.c

@@ -5,9 +5,12 @@
  */
 
 #include <limits.h>
+
 #include <rte_alarm.h>
 #include <rte_string_fns.h>
 
+#include "eal_firmware.h"
+
 #include "qede_ethdev.h"
 /* ######### DEBUG ###########*/
 #include "qede_debug.h"
@@ -127,51 +130,40 @@ static void qed_free_stream_mem(struct ecore_dev *edev)
 #ifdef CONFIG_ECORE_BINARY_FW
 static int qed_load_firmware_data(struct ecore_dev *edev)
 {
-	int fd;
-	struct stat st;
 	const char *fw = RTE_LIBRTE_QEDE_FW;
+	void *buf;
+	size_t bufsz;
+	int ret;
 
 	if (strcmp(fw, "") == 0)
 		strcpy(qede_fw_file, QEDE_DEFAULT_FIRMWARE);
 	else
 		strcpy(qede_fw_file, fw);
 
-	fd = open(qede_fw_file, O_RDONLY);
+	if (rte_firmware_read(qede_fw_file, &buf, &bufsz) < 0) {
-	if (fd < 0) {
+		DP_ERR(edev, "Can't read firmware data: %s\n", qede_fw_file);
-		DP_ERR(edev, "Can't open firmware file\n");
-		return -ENOENT;
-	}
-
-	if (fstat(fd, &st) < 0) {
-		DP_ERR(edev, "Can't stat firmware file\n");
-		close(fd);
 		return -1;
 	}
 
-	edev->firmware = rte_zmalloc("qede_fw", st.st_size,
+	edev->firmware = rte_zmalloc("qede_fw", bufsz, RTE_CACHE_LINE_SIZE);
-				    RTE_CACHE_LINE_SIZE);
 	if (!edev->firmware) {
 		DP_ERR(edev, "Can't allocate memory for firmware\n");
-		close(fd);
+		ret = -ENOMEM;
-		return -ENOMEM;
+		goto out;
 	}
 
-	if (read(fd, edev->firmware, st.st_size) != st.st_size) {
+	memcpy(edev->firmware, buf, bufsz);
-		DP_ERR(edev, "Can't read firmware data\n");
+	edev->fw_len = bufsz;
-		close(fd);
-		return -1;
-	}
-
-	edev->fw_len = st.st_size;
 	if (edev->fw_len < 104) {
 		DP_ERR(edev, "Invalid fw size: %" PRIu64 "\n",
 			  edev->fw_len);
-		close(fd);
+		ret = -EINVAL;
-		return -EINVAL;
+		goto out;
 	}
-
+	ret = 0;
-	close(fd);
+out:
-	return 0;
+	free(buf);
+	return ret;
 }
 #endif
 

lib/eal/common/eal_firmware.h

@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifndef _EAL_FIRMWARE_H_
+#define _EAL_FIRMWARE_H_
+
+#include <sys/types.h>
+
+#include <rte_compat.h>
+
+/**
+ * Load a firmware in a dynamically allocated buffer, dealing with compressed
+ * files if libarchive is available.
+ *
+ * @param[in] name
+ *      Firmware filename to load.
+ * @param[out] buf
+ *      Buffer allocated by this function. If this function succeeds, the
+ *      caller is responsible for calling free() on this buffer.
+ * @param[out] bufsz
+ *      Size of the data in the buffer.
+ *
+ * @return
+ *      0 if successful.
+ *      Negative otherwise, buf and bufsize contents are invalid.
+ */
+__rte_internal
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz);
+
+#endif /* _EAL_FIRMWARE_H_ */

lib/eal/unix/eal_firmware.c

@@ -0,0 +1,150 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Red Hat, Inc.
+ */
+
+#ifdef RTE_HAS_LIBARCHIVE
+#include <archive.h>
+#endif
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+
+#include "eal_firmware.h"
+
+#ifdef RTE_HAS_LIBARCHIVE
+
+struct firmware_read_ctx {
+	struct archive *a;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
+{
+	struct archive_entry *e;
+
+	ctx->a = archive_read_new();
+	if (ctx->a == NULL)
+		return -1;
+	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK ||
+			archive_read_support_filter_xz(ctx->a) != ARCHIVE_OK ||
+			archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK ||
+			archive_read_next_header(ctx->a, &e) != ARCHIVE_OK) {
+		archive_read_free(ctx->a);
+		ctx->a = NULL;
+		return -1;
+	}
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return archive_read_data(ctx->a, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	archive_read_free(ctx->a);
+	ctx->a = NULL;
+}
+
+#else /* !RTE_HAS_LIBARCHIVE */
+
+struct firmware_read_ctx {
+	int fd;
+};
+
+static int
+firmware_open(struct firmware_read_ctx *ctx, const char *name,
+	__rte_unused size_t blocksize)
+{
+	ctx->fd = open(name, O_RDONLY);
+	if (ctx->fd < 0)
+		return -1;
+	return 0;
+}
+
+static ssize_t
+firmware_read_block(struct firmware_read_ctx *ctx, void *buf, size_t count)
+{
+	return read(ctx->fd, buf, count);
+}
+
+static void
+firmware_close(struct firmware_read_ctx *ctx)
+{
+	close(ctx->fd);
+	ctx->fd = -1;
+}
+
+#endif /* !RTE_HAS_LIBARCHIVE */
+
+static int
+firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	const size_t blocksize = 4096;
+	struct firmware_read_ctx ctx;
+	int ret = -1;
+	int err;
+
+	*buf = NULL;
+	*bufsz = 0;
+
+	if (firmware_open(&ctx, name, blocksize) < 0)
+		return -1;
+
+	do {
+		void *tmp;
+
+		tmp = realloc(*buf, *bufsz + blocksize);
+		if (tmp == NULL) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*buf = tmp;
+
+		err = firmware_read_block(&ctx, RTE_PTR_ADD(*buf, *bufsz), blocksize);
+		if (err < 0) {
+			free(*buf);
+			*buf = NULL;
+			*bufsz = 0;
+			goto out;
+		}
+		*bufsz += err;
+
+	} while (err != 0);
+
+	ret = 0;
+out:
+	firmware_close(&ctx);
+	return ret;
+}
+
+int
+rte_firmware_read(const char *name, void **buf, size_t *bufsz)
+{
+	char path[PATH_MAX];
+	int ret;
+
+	ret = firmware_read(name, buf, bufsz);
+	if (ret < 0) {
+		snprintf(path, sizeof(path), "%s.xz", name);
+		path[PATH_MAX - 1] = '\0';
+#ifndef RTE_HAS_LIBARCHIVE
+		if (access(path, F_OK) == 0) {
+			RTE_LOG(WARNING, EAL, "libarchive not linked, %s cannot be decompressed\n",
+				path);
+		}
+#else
+		ret = firmware_read(path, buf, bufsz);
+#endif
+	}
+	return ret;
+}

lib/eal/unix/meson.build

@@ -5,5 +5,6 @@ sources += files(
         'eal_file.c',
         'eal_unix_memory.c',
         'eal_unix_timer.c',
+        'eal_firmware.c',
         'rte_thread.c',
 )

lib/eal/version.map

@@ -428,6 +428,7 @@ EXPERIMENTAL {
 INTERNAL {
 	global:
 
+	rte_firmware_read;
 	rte_mem_lock;
 	rte_mem_map;
 	rte_mem_page_size;

lib/eal/windows/eal.c

@@ -22,6 +22,7 @@
 #include <rte_service_component.h>
 #include <rte_vfio.h>
 
+#include "eal_firmware.h"
 #include "eal_hugepages.h"
 #include "eal_trace.h"
 #include "eal_log.h"
@@ -465,3 +466,11 @@ rte_vfio_container_dma_unmap(__rte_unused int container_fd,
 {
 	return -1;
 }
+
+int
+rte_firmware_read(__rte_unused const char *name,
+			__rte_unused void **buf,
+			__rte_unused size_t *bufsz)
+{
+	return -1;
+}