d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add kernel and userspace code to dump the firmware state of supported

Browse Source 
ConnectX-4/5 devices in mlx5core.

The dump is obtained by reading a predefined register map from the
non-destructive crspace, accessible by the vendor-specific PCIe
capability (VSC). The dump is stored in preallocated kernel memory and
managed by the mlx5tool(8), which communicates with the driver using a
character device node.

The utility allows to store the dump in format
    <address> <value>
into a file, to reset the dump content, and to manually initiate the
dump.

A call to mlx5_fwdump() should be added at the places where a dump
must be fetched automatically. The most likely place is right before a
firmware reset request.

Submitted by:	kib@
MFC after:	1 week
Sponsored by:	Mellanox Technologies
This commit is contained in:
Hans Petter Selasky 2018-03-08 15:21:56 +00:00
parent 479c9e50e1
commit e808190a59
21 changed files with 12087 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/mtree/BSD.include.dist
View File

@ -132,6 +132,8 @@
..
mfi
..
mlx5
..
mmc
..
mpt

9
include/Makefile
View File

@ -255,6 +255,11 @@ copies: .PHONY .META
${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 nvpair.h \
${SDESTDIR}${INCLUDEDIR}/sys
.endif
.if ${MK_MLX5TOOL} != "no"
cd ${SRCTOP}/sys/dev/mlx5; \
${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 mlx5io.h \
${SDESTDIR}${INCLUDEDIR}/dev/mlx5
.endif
symlinks: .PHONY .META
@${ECHO} "Setting up symlinks to kernel source tree..."
@ -384,6 +389,10 @@ symlinks: .PHONY .META
${INSTALL_SYMLINK} ${TAG_ARGS} ../../../sys/rpc/$$h \
${SDESTDIR}${INCLUDEDIR}/rpc; \
done
.if ${MK_MLX5TOOL} != "no"
${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/mlx5/mlx5io.h \
${SDESTDIR}${INCLUDEDIR}/dev/mlx5
.endif
.include <bsd.prog.mk>

4
share/man/man4/Makefile
View File

@ -928,6 +928,10 @@ MAN+= mlx4ib.4
MAN+= mlx5ib.4
.endif
.if ${MK_MLX5TOOL} != "no"
MAN+= mlx5io.4
.endif
.if ${MK_TESTS} != "no"
ATF= ${SRCTOP}/contrib/atf
.PATH: ${ATF}/doc

142
share/man/man4/mlx5io.4 Normal file
View File

@ -0,0 +1,142 @@
.\"
.\" Copyright (c) 2018 Mellanox Technologies
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\"
.Dd February 20, 2018
.Dt mlx5io 4
.Os
.Sh NAME
.Nm mlx5io
.Nd IOCTL interface to manage Connect-X 4/5 Mellanox network adapters
.Sh SYNOPSIS
.In dev/mlx5/mlx5io.h
.Sh DESCRIPTION
The
.Nm
interface is provided for management of the Connect-X 4 and 5 network adapters
in the aspects not covered by the generic network configuration,
mostly related to the PCIe attachment and internal card working.
Interface consists of the commands, which are passed by means of
.Xr ioctl 2
on the file descriptor, opened from the
.Pa /dev/mlx5ctl
device node.
.Pp
The following commands are implemented:
.Bl -tag -width indent
.It Dv MLX5_FWDUMP_FORCE
Take the snapshot of the firmware registers state and store it in the
kernel buffer.
The buffer must be empty, in other words, no dumps should be written so
far, or existing dump cleared with the
.Dv MLX5_FWDUMP_RESET
command for the specified device.
The argument for the command should point to the
.Vt struct mlx5_fwdump_addr
structure, containing the PCIe bus address of the device.
.Bd -literal
struct mlx5_fwdump_addr {
uint32_t domain;
uint8_t bus;
uint8_t slot;
uint8_t func;
};
.Ed
.It Dv MLX5_FWDUMP_RESET
Clear the stored firmware dump, preparing the kernel buffer for
the next dump.
The argument for the command should point to the
.Vt struct mlx5_fwdump_addr
structure, containing the PCIe bus address of the device.
.It Dv MLX5_FWDUMP_GET
Fetch the stored firmware dump into the user memory.
The argument to the command should point to the input/output
.Vt struct mlx5_fwdump_get
structure.
Its
.Dv devaddr
field specifies the address of the device, the
.Dv buf
fields points to the array of
.Vt struct mlx5_fwdump_reg
of records of the registers values, the size of the array is specified
in the
.Dv reg_cnt
field.
.Bd -literal
struct mlx5_fwdump_get {
struct mlx5_fwdump_addr devaddr;
struct mlx5_fwdump_reg *buf;
size_t reg_cnt;
size_t reg_filled; /* out */
};
.Ed
.Pp
On successfull return, the
.Dv reg_filled
field reports the number of the
.Dv buf
array elements actually filled with the registers values.
If
.Dv buf
contains the
.Dv NULL
pointer, no registers are filled, but
.Dv reg_filled
still contains the number of registers that should be passed for
the complete dump.
.Pp
The
.Vt struct mlx5_fwdump_reg
element contains the address of the register in the field
.Dv addr ,
and its value in the field
.Dv val .
.Bd -literal
struct mlx5_fwdump_reg {
uint32_t addr;
uint32_t val;
};
.Ed
.El
.Sh FILES
The
.Pa /dev/mlx5ctl
.Xr devfs 5
node is used to pass commands to the driver.
.Sh RETURN VALUES
If successful, the IOCTL returns zero.
Otherwise, -1 is returned and the global variable
.Va errno
is set to indicate the error.
.Sh SEE ALSO
.Xr errno 2 ,
.Xr ioctl 2 ,
.Xr mlx5en 4 ,
.Xr mlx5ib 4 ,
.Xr mlx5tool 8
and
.Xr pci 9 .

6
share/man/man5/src.conf.5
View File

@ -436,12 +436,18 @@ Set to not build CUSE-related programs and libraries.
.It Va WITHOUT_CXGBETOOL
Set to not build
.Xr cxgbetool 8
.It Va WITHOUT_MLX5TOOL
Set to not build
.Xr mlx5tool 8
.Pp
This is a default setting on
arm/arm, arm/armeb, arm/armv6, arm/armv7, mips/mipsel, mips/mips, mips/mips64el, mips/mips64, mips/mipsn32, mips/mipselhf, mips/mipshf, mips/mips64elhf, mips/mips64hf, powerpc/powerpc, powerpc/powerpcspe, riscv/riscv64 and riscv/riscv64sf.
.It Va WITH_CXGBETOOL
Set to build
.Xr cxgbetool 8
.It Va WITH_MLX5TOOL
Set to build
.Xr mlx5tool 8
.Pp
This is a default setting on
amd64/amd64, arm64/aarch64, i386/i386, powerpc/powerpc64 and sparc64/sparc64.

2
share/mk/src.opts.mk
View File

@ -320,8 +320,10 @@ BROKEN_OPTIONS+=PROFILE
.if ${__T} == "aarch64" || ${__T} == "amd64" || ${__T} == "i386" || \
${__T} == "powerpc64" || ${__T} == "sparc64"
__DEFAULT_YES_OPTIONS+=CXGBETOOL
__DEFAULT_YES_OPTIONS+=MLX5TOOL
.else
__DEFAULT_NO_OPTIONS+=CXGBETOOL
__DEFAULT_NO_OPTIONS+=MLX5TOOL
.endif
.include <bsd.mkopt.mk>

4
sys/conf/files
View File

@ -4731,6 +4731,10 @@ dev/mlx5/mlx5_core/mlx5_fs_tree.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_fw.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_fwdump.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_fwdump_regmaps.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_health.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_mad.c optional mlx5 pci \

2
sys/dev/mlx5/driver.h
View File

@ -615,6 +615,7 @@ struct mlx5_special_contexts {
};
struct mlx5_flow_root_namespace;
struct mlx5_dump_data;
struct mlx5_core_dev {
struct pci_dev *pdev;
/* sync pci state */
@ -648,6 +649,7 @@ struct mlx5_core_dev {
struct mlx5_flow_root_namespace *sniffer_rx_root_ns;
struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER];
struct mlx5_dump_data *dump_data;
};
enum {

13
sys/dev/mlx5/mlx5_core/mlx5_core.h
View File

@ -80,4 +80,17 @@ void mlx5e_cleanup(void);
int mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name);
int mlx5_fwdump_init(void);
void mlx5_fwdump_fini(void);
int mlx5_fwdump_prep(struct mlx5_core_dev *mdev);
void mlx5_fwdump(struct mlx5_core_dev *mdev);
void mlx5_fwdump_clean(struct mlx5_core_dev *mdev);
struct mlx5_crspace_regmap {
uint32_t addr;
unsigned cnt;
};
extern struct pci_driver mlx5_core_driver;
#endif /* __MLX5_CORE_H__ */

308
sys/dev/mlx5/mlx5_core/mlx5_fwdump.c Normal file
View File

@ -0,0 +1,308 @@
/*-
* Copyright (c) 2018, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <dev/mlx5/driver.h>
#include <dev/mlx5/device.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
#include <dev/mlx5/mlx5io.h>
extern const struct mlx5_crspace_regmap mlx5_crspace_regmap_mt4117[];
extern const struct mlx5_crspace_regmap mlx5_crspace_regmap_mt4115[];
extern const struct mlx5_crspace_regmap mlx5_crspace_regmap_connectx5[];
struct mlx5_dump_data {
const struct mlx5_crspace_regmap *rege;
uint32_t *dump;
unsigned dump_size;
int dump_valid;
struct mtx dump_lock;
};
static MALLOC_DEFINE(M_MLX5_DUMP, "MLX5DUMP", "MLX5 Firmware dump");
static unsigned
mlx5_fwdump_getsize(const struct mlx5_crspace_regmap *rege)
{
const struct mlx5_crspace_regmap *r;
unsigned sz;
for (sz = 0, r = rege; r->cnt != 0; r++)
sz += r->cnt;
return (sz);
}
static void
mlx5_fwdump_destroy_dd(struct mlx5_dump_data *dd)
{
mtx_destroy(&dd->dump_lock);
free(dd->dump, M_MLX5_DUMP);
free(dd, M_MLX5_DUMP);
}
int
mlx5_fwdump_prep(struct mlx5_core_dev *mdev)
{
struct mlx5_dump_data *dd;
int error;
error = mlx5_vsc_find_cap(mdev);
if (error != 0)
return (error);
dd = malloc(sizeof(struct mlx5_dump_data), M_MLX5_DUMP, M_WAITOK);
switch (pci_get_device(mdev->pdev->dev.bsddev)) {
case 0x1013:
dd->rege = mlx5_crspace_regmap_mt4115;
break;
case 0x1015:
dd->rege = mlx5_crspace_regmap_mt4117;
break;
case 0x1017:
case 0x1019:
dd->rege = mlx5_crspace_regmap_connectx5;
break;
default:
free(dd, M_MLX5_DUMP);
return (0); /* silently fail to not prevent driver attach */
}
dd->dump_size = mlx5_fwdump_getsize(dd->rege);
dd->dump = malloc(dd->dump_size * sizeof(uint32_t), M_MLX5_DUMP,
M_WAITOK | M_ZERO);
dd->dump_valid = 0;
mtx_init(&dd->dump_lock, "mlx5dmp", NULL, MTX_DEF | MTX_NEW);
if (atomic_cmpset_rel_ptr((uintptr_t *)&mdev->dump_data, 0,
(uintptr_t)dd) == 0)
mlx5_fwdump_destroy_dd(dd);
return (0);
}
void
mlx5_fwdump(struct mlx5_core_dev *mdev)
{
struct mlx5_dump_data *dd;
const struct mlx5_crspace_regmap *r;
uint32_t i, ri;
int error;
dd = (struct mlx5_dump_data *)atomic_load_acq_ptr((uintptr_t *)
&mdev->dump_data);
if (dd == NULL)
return;
mtx_lock(&dd->dump_lock);
if (dd->dump_valid)
/* only one dump */
goto failed;
/* mlx5_vsc already warns, be silent. */
error = mlx5_vsc_lock(mdev);
if (error != 0)
goto failed;
error = mlx5_vsc_set_space(mdev, MLX5_VSC_DOMAIN_PROTECTED_CRSPACE);
if (error != 0)
goto unlock_vsc;
for (i = 0, r = dd->rege; r->cnt != 0; r++) {
for (ri = 0; ri < r->cnt; ri++) {
error = mlx5_vsc_read(mdev, r->addr + ri * 4,
&dd->dump[i]);
if (error != 0)
goto unlock_vsc;
i++;
}
}
atomic_store_rel_int(&dd->dump_valid, 1);
unlock_vsc:
mlx5_vsc_unlock(mdev);
failed:
mtx_unlock(&dd->dump_lock);
}
void
mlx5_fwdump_clean(struct mlx5_core_dev *mdev)
{
struct mlx5_dump_data *dd;
for (;;) {
dd = mdev->dump_data;
if (dd == NULL)
return;
if (atomic_cmpset_ptr((uintptr_t *)&mdev->dump_data,
(uintptr_t)dd, 0) == 1) {
mlx5_fwdump_destroy_dd(dd);
return;
}
}
}
static int
mlx5_dbsf_to_core(const struct mlx5_fwdump_addr *devaddr,
struct mlx5_core_dev **mdev)
{
device_t dev;
struct pci_dev *pdev;
dev = pci_find_dbsf(devaddr->domain, devaddr->bus, devaddr->slot,
devaddr->func);
if (dev == NULL)
return (ENOENT);
if (device_get_devclass(dev) != mlx5_core_driver.bsdclass)
return (EINVAL);
pdev = device_get_softc(dev);
*mdev = pci_get_drvdata(pdev);
if (*mdev == NULL)
return (ENOENT);
return (0);
}
static int
mlx5_fwdump_copyout(struct mlx5_dump_data *dd, struct mlx5_fwdump_get *fwg)
{
const struct mlx5_crspace_regmap *r;
struct mlx5_fwdump_reg rv, *urv;
uint32_t i, ri;
int error;
if (dd == NULL)
return (ENOENT);
if (fwg->buf == NULL) {
fwg->reg_filled = dd->dump_size;
return (0);
}
if (atomic_load_acq_int(&dd->dump_valid) == 0)
return (ENOENT);
urv = fwg->buf;
for (i = 0, r = dd->rege; r->cnt != 0; r++) {
for (ri = 0; ri < r->cnt; ri++) {
if (i >= fwg->reg_cnt)
goto out;
rv.addr = r->addr + ri * 4;
rv.val = dd->dump[i];
error = copyout(&rv, urv, sizeof(rv));
if (error != 0)
return (error);
urv++;
i++;
}
}
out:
fwg->reg_filled = i;
return (0);
}
static int
mlx5_fwdump_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
struct thread *td)
{
struct mlx5_core_dev *mdev;
struct mlx5_fwdump_get *fwg;
struct mlx5_fwdump_addr *devaddr;
struct mlx5_dump_data *dd;
int error;
error = 0;
switch (cmd) {
case MLX5_FWDUMP_GET:
if ((fflag & FREAD) == 0) {
error = EBADF;
break;
}
fwg = (struct mlx5_fwdump_get *)data;
devaddr = &fwg->devaddr;
error = mlx5_dbsf_to_core(devaddr, &mdev);
if (error != 0)
break;
error = mlx5_fwdump_copyout(mdev->dump_data, fwg);
break;
case MLX5_FWDUMP_RESET:
if ((fflag & FWRITE) == 0) {
error = EBADF;
break;
}
devaddr = (struct mlx5_fwdump_addr *)data;
error = mlx5_dbsf_to_core(devaddr, &mdev);
if (error != 0)
break;
dd = mdev->dump_data;
if (dd != NULL)
atomic_store_rel_int(&dd->dump_valid, 0);
else
error = ENOENT;
break;
case MLX5_FWDUMP_FORCE:
if ((fflag & FWRITE) == 0) {
error = EBADF;
break;
}
devaddr = (struct mlx5_fwdump_addr *)data;
error = mlx5_dbsf_to_core(devaddr, &mdev);
if (error != 0)
break;
mlx5_fwdump(mdev);
break;
default:
error = ENOTTY;
break;
}
return (error);
}
static struct cdevsw mlx5_fwdump_devsw = {
.d_version = D_VERSION,
.d_ioctl = mlx5_fwdump_ioctl,
};
static struct cdev *mlx5_fwdump_dev;
int
mlx5_fwdump_init(void)
{
struct make_dev_args mda;
int error;
make_dev_args_init(&mda);
mda.mda_flags = MAKEDEV_WAITOK | MAKEDEV_CHECKNAME;
mda.mda_devsw = &mlx5_fwdump_devsw;
mda.mda_uid = UID_ROOT;
mda.mda_gid = GID_OPERATOR;
mda.mda_mode = 0640;
error = make_dev_s(&mda, &mlx5_fwdump_dev, "mlx5ctl");
return (-error);
}
void
mlx5_fwdump_fini(void)
{
if (mlx5_fwdump_dev != NULL)
destroy_dev(mlx5_fwdump_dev);
}

11187
sys/dev/mlx5/mlx5_core/mlx5_fwdump_regmaps.c Normal file
View File

File diff suppressed because it is too large Load Diff

10
sys/dev/mlx5/mlx5_core/mlx5_main.c
View File

@ -1089,6 +1089,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
goto out;
}
mlx5_fwdump_clean(dev);
mlx5_unregister_device(dev);
mlx5_cleanup_fs(dev);
@ -1374,7 +1375,7 @@ static const struct pci_device_id mlx5_core_pci_table[] = {
MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
static struct pci_driver mlx5_core_driver = {
struct pci_driver mlx5_core_driver = {
.name = DRIVER_NAME,
.id_table = mlx5_core_pci_table,
.shutdown = shutdown_one,
@ -1391,15 +1392,22 @@ static int __init init(void)
if (err)
goto err_debug;
err = mlx5_fwdump_init();
if (err)
goto err_fwdump;
return 0;
err_fwdump:
pci_unregister_driver(&mlx5_core_driver);
err_debug:
return err;
}
static void __exit cleanup(void)
{
mlx5_fwdump_fini();
pci_unregister_driver(&mlx5_core_driver);
}

60
sys/dev/mlx5/mlx5io.h Normal file
View File

@ -0,0 +1,60 @@
/*-
* Copyright (c) 2018, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _DEV_MLX5_MLX5IO_H_
#define _DEV_MLX5_MLX5IO_H_
#include <sys/ioccom.h>
struct mlx5_fwdump_reg {
uint32_t addr;
uint32_t val;
};
struct mlx5_fwdump_addr {
uint32_t domain;
uint8_t bus;
uint8_t slot;
uint8_t func;
};
struct mlx5_fwdump_get {
struct mlx5_fwdump_addr devaddr;
struct mlx5_fwdump_reg *buf;
size_t reg_cnt;
size_t reg_filled; /* out */
};
#define MLX5_FWDUMP_GET _IOWR('m', 1, struct mlx5_fwdump_get)
#define MLX5_FWDUMP_RESET _IOW('m', 2, struct mlx5_fwdump_addr)
#define MLX5_FWDUMP_FORCE _IOW('m', 3, struct mlx5_fwdump_addr)
#ifndef _KERNEL
#define MLX5_DEV_PATH _PATH_DEV"mlx5ctl"
#endif
#endif

2
sys/modules/mlx5/Makefile
View File

@ -11,6 +11,8 @@ mlx5_eq.c \
mlx5_fs_cmd.c \
mlx5_fs_tree.c \
mlx5_fw.c \
mlx5_fwdump.c \
mlx5_fwdump_regmaps.c \
mlx5_health.c \
mlx5_mad.c \
mlx5_main.c \

4
targets/pseudo/userland/Makefile.depend
View File

@ -889,6 +889,10 @@ DIRDEPS+= \
DIRDEPS+= usr.sbin/cxgbetool
.endif
.if ${MK_MLX5TOOL} != "no"
DIRDEPS+= usr.sbin/mlx5tool
.endif
.if ${MK_GPL_DTC} != "yes"
DIRDEPS+= usr.bin/dtc
.endif

3
tools/build/options/WITHOUT_MLX5TOOL Normal file
View File

@ -0,0 +1,3 @@
.\" $FreeBSD$
Set to not build
.Xr mlx5tool 8

3
tools/build/options/WITH_MLX5TOOL Normal file
View File

@ -0,0 +1,3 @@
.\" $FreeBSD$
Set to build
.Xr mlx5tool 8

1
usr.sbin/Makefile
View File

@ -121,6 +121,7 @@ SUBDIR.${MK_BSDINSTALL}+= bsdinstall
SUBDIR.${MK_BSNMP}+= bsnmpd
SUBDIR.${MK_CTM}+= ctm
SUBDIR.${MK_CXGBETOOL}+= cxgbetool
SUBDIR.${MK_MLX5TOOL}+= mlx5tool
SUBDIR.${MK_DIALOG}+= bsdconfig
SUBDIR.${MK_EFI}+= efivar efidp efibootmgr
SUBDIR.${MK_FLOPPY}+= fdcontrol

7
usr.sbin/mlx5tool/Makefile Normal file
View File

@ -0,0 +1,7 @@
# $FreeBSD$
PROG= mlx5tool
MAN= mlx5tool.8
WARNS?= 6
.include <bsd.prog.mk>

97
usr.sbin/mlx5tool/mlx5tool.8 Normal file
View File

@ -0,0 +1,97 @@
.\"
.\" Copyright (c) 2018 Mellanox Technologies
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\" notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\" notice, this list of conditions and the following disclaimer in the
.\" documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
.\" $FreeBSD$
.\"
.Dd March 8, 2018
.Dt mlx5tool 8
.Os
.Sh NAME
.Nm mlx5tool
.Nd Utility for managing Connect-X 4/5 Mellanox network adapters
.Sh SYNOPSIS
.Nm
.Fl d Ar domain:bus:slot:func
.Fl e
.Nm
.Fl d Ar domain:bus:slot:func
.Fl r
.Nm
.Fl d Ar domain:bus:slot:func
.Fl o Ar file
.Fl w
.Sh DESCRIPTION
The
.Nm
utility is provided for management of the Connect-X 4 and 5 network adapters
in the aspects not covered by the generic
.Xr ifconfig 8
command, mostly related to the PCIe attachment and internal card working.
The utility executes commands on specific adapter, which is addressed using
.Em device:bus:slot:function
conventions of the PCIe buses.
You can match adapters ethernet name and addresses using the
.X pciconf 8
utility.
The address is passed as an argument of the
.Fl d
option, which must be specified for each invocation.
.Pp
When the driver detects the malfunctioning of the hardware, or by user
request, it is possible to create
.Em firmware dump ,
which contains debugging information about internal device state, for
analysis of the failure by the Mellanox support team.
.Pp
The following commands are currently implemented:
.Bl -tag -width indent
.It Fl e
Take the snapshot of the firmware registers state and store it in the
kernel buffer.
The buffer must be empty, in other words, no dumps should be written so
far, or existing dump cleared with the
.Fl r
command for the specified device.
.It Fl r
Clear the stored firmware dump, preparing the kernel buffer for
the next dump.
.It Fl w
Fetche the stored firmware dump and writes it into the file specified
by the
.Fl o
option argument.
.El
.Sh FILES
The
.Pa /dev/mlx5ctl
.Xr devfs 5
node is used to pass commands to the driver.
.Sh SEE ALSO
.Xr mlx5en 4 ,
.Xr mlx5ib 4 ,
.Xr mlx5io 4 ,
.Xr ifconfig 8
and
.Xr pciconf 8 .

220
usr.sbin/mlx5tool/mlx5tool.c Normal file
View File

@ -0,0 +1,220 @@
/*-
* Copyright (c) 2018, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/ioctl.h>
#include <dev/mlx5/mlx5io.h>
#include <ctype.h>
#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <paths.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* stolen from pciconf.c: parsesel() */
static int
parse_pci_addr(const char *addrstr, struct mlx5_fwdump_addr *addr)
{
char *eppos;
unsigned long selarr[4];
int i;
if (strncmp(addrstr, "pci", 3) == 0) {
addrstr += 3;
i = 0;
while (isdigit(*addrstr) && i < 4) {
selarr[i++] = strtoul(addrstr, &eppos, 10);
addrstr = eppos;
if (*addrstr == ':')
addrstr++;
}
if (i > 0 && *addrstr == '\0') {
addr->func = (i > 2) ? selarr[--i] : 0;
addr->slot = (i > 0) ? selarr[--i] : 0;
addr->bus = (i > 0) ? selarr[--i] : 0;
addr->domain = (i > 0) ? selarr[--i] : 0;
return (0);
}
}
warnx("invalid pci address %s", addrstr);
return (1);
}
static int
mlx5tool_save_dump(int ctldev, const struct mlx5_fwdump_addr *addr,
const char *dumpname)
{
struct mlx5_fwdump_get fdg;
struct mlx5_fwdump_reg *rege;
FILE *dump;
size_t cnt;
int error, res;
if (dumpname == NULL)
dump = stdout;
else
dump = fopen(dumpname, "w");
if (dump == NULL) {
warn("open %s", dumpname);
return (1);
}
res = 1;
memset(&fdg, 0, sizeof(fdg));
fdg.devaddr = *addr;
error = ioctl(ctldev, MLX5_FWDUMP_GET, &fdg);
if (error != 0) {
warn("MLX5_FWDUMP_GET dumpsize");
goto out;
}
rege = calloc(fdg.reg_filled, sizeof(*rege));
if (rege == NULL) {
warn("alloc rege");
goto out;
}
fdg.buf = rege;
fdg.reg_cnt = fdg.reg_filled;
error = ioctl(ctldev, MLX5_FWDUMP_GET, &fdg);
if (error != 0) {
if (errno == ENOENT)
warnx("no dump recorded");
else
warn("MLX5_FWDUMP_GET dump fetch");
goto out;
}
for (cnt = 0; cnt < fdg.reg_cnt; cnt++, rege++)
fprintf(dump, "0x%08x\t0x%08x\n", rege->addr, rege->val);
res = 0;
out:
if (dump != stdout)
fclose(dump);
return (res);
}
static int
mlx5tool_dump_reset(int ctldev, const struct mlx5_fwdump_addr *addr)
{
if (ioctl(ctldev, MLX5_FWDUMP_RESET, addr) == -1) {
warn("MLX5_FWDUMP_RESET");
return (1);
}
return (0);
}
static int
mlx5tool_dump_force(int ctldev, const struct mlx5_fwdump_addr *addr)
{
if (ioctl(ctldev, MLX5_FWDUMP_FORCE, addr) == -1) {
warn("MLX5_FWDUMP_FORCE");
return (1);
}
return (0);
}
static void
usage(void)
{
fprintf(stderr,
"Usage: mlx5tool -d pci<d:b:s:f> [-w -o dump.file | -r | -e]\n");
fprintf(stderr, "\t-w - write firmware dump to the specified file\n");
fprintf(stderr, "\t-r - reset dump\n");
fprintf(stderr, "\t-e - force dump\n");
exit(1);
}
enum mlx5_action {
ACTION_DUMP_GET,
ACTION_DUMP_RESET,
ACTION_DUMP_FORCE,
ACTION_NONE,
};
int
main(int argc, char *argv[])
{
struct mlx5_fwdump_addr addr;
char *dumpname;
char *addrstr;
int c, ctldev, res;
enum mlx5_action act;
act = ACTION_NONE;
addrstr = NULL;
dumpname = NULL;
while ((c = getopt(argc, argv, "d:eho:rw")) != -1) {
switch (c) {
case 'd':
addrstr = optarg;
break;
case 'w':
act = ACTION_DUMP_GET;
break;
case 'e':
act= ACTION_DUMP_FORCE;
break;
case 'o':
dumpname = optarg;
break;
case 'r':
act = ACTION_DUMP_RESET;
break;
case 'h':
default:
usage();
}
}
if (act == ACTION_NONE || (dumpname != NULL && act != ACTION_DUMP_GET))
usage();
if (parse_pci_addr(addrstr, &addr) != 0)
exit(1);
ctldev = open(MLX5_DEV_PATH, O_RDWR);
if (ctldev == -1)
err(1, "open "MLX5_DEV_PATH);
switch (act) {
case ACTION_DUMP_GET:
res = mlx5tool_save_dump(ctldev, &addr, dumpname);
break;
case ACTION_DUMP_RESET:
res = mlx5tool_dump_reset(ctldev, &addr);
break;
case ACTION_DUMP_FORCE:
res = mlx5tool_dump_force(ctldev, &addr);
break;
default:
res = 0;
break;
}
close(ctldev);
exit(res);
}