From 986e7b9227668caf9620f207e3c1d708c87b634d Mon Sep 17 00:00:00 2001 From: Artur Rojek Date: Thu, 12 Aug 2021 10:34:27 +0200 Subject: [PATCH] ena: Move RSS logic into its own source files Delegate RSS related functionality into separate .c/.h files in preparation for the full RSS support. While at it, reorder functions and remove prototypes for ones with internal linkage. Obtained from: Semihalf MFC after: 2 weeks Sponsored by: Amazon, Inc. --- sys/dev/ena/ena.c | 149 +------------------------------- sys/dev/ena/ena.h | 1 - sys/dev/ena/ena_rss.c | 182 +++++++++++++++++++++++++++++++++++++++ sys/dev/ena/ena_rss.h | 49 +++++++++++ sys/modules/ena/Makefile | 2 +- 5 files changed, 233 insertions(+), 150 deletions(-) create mode 100644 sys/dev/ena/ena_rss.c create mode 100644 sys/dev/ena/ena_rss.h diff --git a/sys/dev/ena/ena.c b/sys/dev/ena/ena.c index faa1278c6d81..7dff59043bb2 100644 --- a/sys/dev/ena/ena.c +++ b/sys/dev/ena/ena.c @@ -63,9 +63,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#ifdef RSS -#include -#endif #include #include @@ -84,6 +81,7 @@ __FBSDID("$FreeBSD$"); #include "ena_datapath.h" #include "ena.h" #include "ena_sysctl.h" +#include "ena_rss.h" #ifdef DEV_NETMAP #include "ena_netmap.h" @@ -143,7 +141,6 @@ static void ena_free_io_irq(struct ena_adapter *); static void ena_free_irqs(struct ena_adapter*); static void ena_disable_msix(struct ena_adapter *); static void ena_unmask_all_io_irqs(struct ena_adapter *); -static int ena_rss_configure(struct ena_adapter *); static int ena_up_complete(struct ena_adapter *); static uint64_t ena_get_counter(if_t, ift_counter); static int ena_media_change(if_t); @@ -161,8 +158,6 @@ static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *, struct ena_com_dev_get_features_ctx *); static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *); -static int ena_rss_init_default(struct ena_adapter *); -static void ena_rss_init_default_deferred(void *); static void ena_config_host_info(struct ena_com_dev *, device_t); static int ena_attach(device_t); static int ena_detach(device_t); @@ -265,27 +260,6 @@ ena_dma_alloc(device_t dmadev, bus_size_t size, return (error); } -/* - * This function should generate unique key for the whole driver. - * If the key was already genereated in the previous call (for example - * for another adapter), then it should be returned instead. - */ -void -ena_rss_key_fill(void *key, size_t size) -{ - static bool key_generated; - static uint8_t default_key[ENA_HASH_KEY_SIZE]; - - KASSERT(size <= ENA_HASH_KEY_SIZE, ("Requested more bytes than ENA RSS key can hold")); - - if (!key_generated) { - arc4random_buf(default_key, ENA_HASH_KEY_SIZE); - key_generated = true; - } - - memcpy(key, default_key, size); -} - static void ena_free_pci_resources(struct ena_adapter *adapter) { @@ -1922,43 +1896,6 @@ ena_unmask_all_io_irqs(struct ena_adapter *adapter) } } -/* Configure the Rx forwarding */ -static int -ena_rss_configure(struct ena_adapter *adapter) -{ - struct ena_com_dev *ena_dev = adapter->ena_dev; - int rc; - - /* In case the RSS table was destroyed */ - if (!ena_dev->rss.tbl_log_size) { - rc = ena_rss_init_default(adapter); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { - ena_log(adapter->pdev, ERR, - "WARNING: RSS was not properly re-initialized," - " it will affect bandwidth\n"); - ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); - return (rc); - } - } - - /* Set indirect table */ - rc = ena_com_indirect_table_set(ena_dev); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) - return (rc); - - /* Configure hash function (if supported) */ - rc = ena_com_set_hash_function(ena_dev); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) - return (rc); - - /* Configure hash inputs (if supported) */ - rc = ena_com_set_hash_ctrl(ena_dev); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) - return (rc); - - return (0); -} - static int ena_up_complete(struct ena_adapter *adapter) { @@ -2730,90 +2667,6 @@ ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx) return (0); } -static int -ena_rss_init_default(struct ena_adapter *adapter) -{ - struct ena_com_dev *ena_dev = adapter->ena_dev; - device_t dev = adapter->pdev; - int qid, rc, i; - - rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); - if (unlikely(rc != 0)) { - ena_log(dev, ERR, "Cannot init indirect table\n"); - return (rc); - } - - for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { - qid = i % adapter->num_io_queues; - rc = ena_com_indirect_table_fill_entry(ena_dev, i, - ENA_IO_RXQ_IDX(qid)); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { - ena_log(dev, ERR, "Cannot fill indirect table\n"); - goto err_rss_destroy; - } - } - -#ifdef RSS - uint8_t rss_algo = rss_gethashalgo(); - if (rss_algo == RSS_HASH_TOEPLITZ) { - uint8_t hash_key[RSS_KEYSIZE]; - - rss_getkey(hash_key); - rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, - hash_key, RSS_KEYSIZE, 0xFFFFFFFF); - } else -#endif - rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, - ENA_HASH_KEY_SIZE, 0xFFFFFFFF); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { - ena_log(dev, ERR, "Cannot fill hash function\n"); - goto err_rss_destroy; - } - - rc = ena_com_set_default_hash_ctrl(ena_dev); - if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { - ena_log(dev, ERR, "Cannot fill hash control\n"); - goto err_rss_destroy; - } - - return (0); - -err_rss_destroy: - ena_com_rss_destroy(ena_dev); - return (rc); -} - -static void -ena_rss_init_default_deferred(void *arg) -{ - struct ena_adapter *adapter; - devclass_t dc; - int max; - int rc; - - dc = devclass_find("ena"); - if (unlikely(dc == NULL)) { - ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); - return; - } - - max = devclass_get_maxunit(dc); - while (max-- >= 0) { - adapter = devclass_get_softc(dc, max); - if (adapter != NULL) { - rc = ena_rss_init_default(adapter); - ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); - if (unlikely(rc != 0)) { - ena_log(adapter->pdev, WARN, - "WARNING: RSS was not properly initialized," - " it will affect bandwidth\n"); - ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); - } - } - } -} -SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL); - static void ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev) { diff --git a/sys/dev/ena/ena.h b/sys/dev/ena/ena.h index f05c2d69fbe4..c7e952f5b571 100644 --- a/sys/dev/ena/ena.h +++ b/sys/dev/ena/ena.h @@ -34,7 +34,6 @@ #ifndef ENA_H #define ENA_H -#include #include "ena-com/ena_com.h" #include "ena-com/ena_eth_com.h" diff --git a/sys/dev/ena/ena_rss.c b/sys/dev/ena/ena_rss.c new file mode 100644 index 000000000000..f314905559d2 --- /dev/null +++ b/sys/dev/ena/ena_rss.c @@ -0,0 +1,182 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_rss.h" + +#include "ena_rss.h" + +/* + * This function should generate unique key for the whole driver. + * If the key was already genereated in the previous call (for example + * for another adapter), then it should be returned instead. + */ +void +ena_rss_key_fill(void *key, size_t size) +{ + static bool key_generated; + static uint8_t default_key[ENA_HASH_KEY_SIZE]; + + KASSERT(size <= ENA_HASH_KEY_SIZE, ("Requested more bytes than ENA RSS key can hold")); + + if (!key_generated) { + arc4random_buf(default_key, ENA_HASH_KEY_SIZE); + key_generated = true; + } + + memcpy(key, default_key, size); +} + +static int +ena_rss_init_default(struct ena_adapter *adapter) +{ + struct ena_com_dev *ena_dev = adapter->ena_dev; + device_t dev = adapter->pdev; + int qid, rc, i; + + rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); + if (unlikely(rc != 0)) { + ena_log(dev, ERR, "Cannot init indirect table\n"); + return (rc); + } + + for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { + qid = i % adapter->num_io_queues; + rc = ena_com_indirect_table_fill_entry(ena_dev, i, + ENA_IO_RXQ_IDX(qid)); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { + ena_log(dev, ERR, "Cannot fill indirect table\n"); + goto err_rss_destroy; + } + } + + +#ifdef RSS + uint8_t rss_algo = rss_gethashalgo(); + if (rss_algo == RSS_HASH_TOEPLITZ) { + uint8_t hash_key[RSS_KEYSIZE]; + + rss_getkey(hash_key); + rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, + hash_key, RSS_KEYSIZE, 0xFFFFFFFF); + } else +#endif + rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, + ENA_HASH_KEY_SIZE, 0xFFFFFFFF); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { + ena_log(dev, ERR, "Cannot fill hash function\n"); + goto err_rss_destroy; + } + + rc = ena_com_set_default_hash_ctrl(ena_dev); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { + ena_log(dev, ERR, "Cannot fill hash control\n"); + goto err_rss_destroy; + } + + return (0); + +err_rss_destroy: + ena_com_rss_destroy(ena_dev); + return (rc); +} + +/* Configure the Rx forwarding */ +int +ena_rss_configure(struct ena_adapter *adapter) +{ + struct ena_com_dev *ena_dev = adapter->ena_dev; + int rc; + + /* In case the RSS table was destroyed */ + if (!ena_dev->rss.tbl_log_size) { + rc = ena_rss_init_default(adapter); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { + ena_log(adapter->pdev, ERR, + "WARNING: RSS was not properly re-initialized," + " it will affect bandwidth\n"); + ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); + return (rc); + } + } + + /* Set indirect table */ + rc = ena_com_indirect_table_set(ena_dev); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) + return (rc); + + /* Configure hash function (if supported) */ + rc = ena_com_set_hash_function(ena_dev); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) + return (rc); + + /* Configure hash inputs (if supported) */ + rc = ena_com_set_hash_ctrl(ena_dev); + if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) + return (rc); + + return (0); +} + +static void +ena_rss_init_default_deferred(void *arg) +{ + struct ena_adapter *adapter; + devclass_t dc; + int max; + int rc; + + dc = devclass_find("ena"); + if (unlikely(dc == NULL)) { + ena_log_raw(ERR, "SYSINIT: %s: No devclass ena\n", __func__); + return; + } + + max = devclass_get_maxunit(dc); + while (max-- >= 0) { + adapter = devclass_get_softc(dc, max); + if (adapter != NULL) { + rc = ena_rss_init_default(adapter); + ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); + if (unlikely(rc != 0)) { + ena_log(adapter->pdev, WARN, + "WARNING: RSS was not properly initialized," + " it will affect bandwidth\n"); + ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); + } + } + } +} +SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL); diff --git a/sys/dev/ena/ena_rss.h b/sys/dev/ena/ena_rss.h new file mode 100644 index 000000000000..14e686a5c045 --- /dev/null +++ b/sys/dev/ena/ena_rss.h @@ -0,0 +1,49 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef ENA_RSS_H +#define ENA_RSS_H + +#include "opt_rss.h" + +#include + +#ifdef RSS +#include +#endif + +#include "ena.h" + +int ena_rss_configure(struct ena_adapter *); + +#endif /* !(ENA_RSS_H) */ diff --git a/sys/modules/ena/Makefile b/sys/modules/ena/Makefile index c6a0c56e7ffb..d619b5d7fa56 100644 --- a/sys/modules/ena/Makefile +++ b/sys/modules/ena/Makefile @@ -35,7 +35,7 @@ KMOD = if_ena SRCS = ena_com.c ena_eth_com.c -SRCS += ena.c ena_sysctl.c ena_datapath.c ena_netmap.c +SRCS += ena.c ena_sysctl.c ena_datapath.c ena_netmap.c ena_rss.c SRCS += device_if.h bus_if.h pci_if.h SRCS += opt_rss.h CFLAGS += -I${SRCTOP}/sys/contrib