diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index 55caf8283da3..616bfb4aa04a 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -151,6 +151,11 @@ OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_ OLD_DIRS+=usr/lib/clang/3.9.1/lib/freebsd OLD_DIRS+=usr/lib/clang/3.9.1/lib OLD_DIRS+=usr/lib/clang/3.9.1 +# 20170110: Four files from ggate tests consolidated into one +OLD_FILES+=usr/tests/sys/geom/class/gate/1_test +OLD_FILES+=usr/tests/sys/geom/class/gate/2_test +OLD_FILES+=usr/tests/sys/geom/class/gate/3_test +OLD_FILES+=usr/tests/sys/geom/class/gate/conf.sh # 20170103: libbsnmptools.so made into an INTERNALLIB OLD_FILES+=usr/lib/libbsnmptools.a OLD_FILES+=usr/lib/libbsnmptools_p.a @@ -161,8 +166,8 @@ OLD_FILES+=usr/share/man/man3/sysdecode_getfsstat_flags.3.gz # 20161230: libarchive ACL pax test renamed to test_acl_pax_posix1e.tar.uu OLD_FILES+=usr/tests/lib/libarchive/test_acl_pax.tar.uu # 20161229: Three files from gnop tests consolidated into one -OLD_FILES+=usr/tests/sys/geom/class/nop/1_test.sh -OLD_FILES+=usr/tests/sys/geom/class/nop/2_test.sh +OLD_FILES+=usr/tests/sys/geom/class/nop/1_test +OLD_FILES+=usr/tests/sys/geom/class/nop/2_test OLD_FILES+=usr/tests/sys/geom/class/nop/conf.sh # 20161217: new clang import which bumps version from 3.9.0 to 3.9.1. OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/allocator_interface.h diff --git a/UPDATING b/UPDATING index d3795efde49f..e2723881f234 100644 --- a/UPDATING +++ b/UPDATING @@ -56,6 +56,11 @@ NOTE TO PEOPLE WHO THINK THAT FreeBSD 12.x IS SLOW: Please see the 20141231 entry below for information about prerequisites and upgrading, if you are not already using clang 3.5.0 or higher. +20170109: + The igb(4), em(4) and lem(4) ethernet drivers are now implemented via + IFLIB. If you have a custom kernel configuration that excludes em(4) + but you use igb(4), you need to re-add em(4) to your custom configuration. + 20161217: Clang, llvm, lldb, compiler-rt and libc++ have been upgraded to 3.9.1. Please see the 20141231 entry below for information about prerequisites diff --git a/contrib/libarchive/libarchive/archive_acl.c b/contrib/libarchive/libarchive/archive_acl.c index b8e8ba40d2ec..7c682095962a 100644 --- a/contrib/libarchive/libarchive/archive_acl.c +++ b/contrib/libarchive/libarchive/archive_acl.c @@ -786,7 +786,8 @@ append_entry_w(wchar_t **wp, const wchar_t *prefix, int type, } else if (tag == ARCHIVE_ENTRY_ACL_USER || tag == ARCHIVE_ENTRY_ACL_GROUP) { append_id_w(wp, id); - id = -1; + if ((type & ARCHIVE_ENTRY_ACL_TYPE_NFS4) == 0) + id = -1; } /* Solaris style has no second colon after other and mask */ if (((flags & ARCHIVE_ENTRY_ACL_STYLE_SOLARIS) == 0) @@ -1042,7 +1043,8 @@ append_entry(char **p, const char *prefix, int type, } else if (tag == ARCHIVE_ENTRY_ACL_USER || tag == ARCHIVE_ENTRY_ACL_GROUP) { append_id(p, id); - id = -1; + if ((type & ARCHIVE_ENTRY_ACL_TYPE_NFS4) == 0) + id = -1; } /* Solaris style has no second colon after other and mask */ if (((flags & ARCHIVE_ENTRY_ACL_STYLE_SOLARIS) == 0) @@ -1328,6 +1330,7 @@ archive_acl_from_text_w(struct archive_acl *acl, const wchar_t *text, tag == ARCHIVE_ENTRY_ACL_GROUP) { n = 1; name = field[1]; + isint_w(name.start, name.end, &id); } else n = 0; @@ -1799,6 +1802,7 @@ archive_acl_from_text_l(struct archive_acl *acl, const char *text, tag == ARCHIVE_ENTRY_ACL_GROUP) { n = 1; name = field[1]; + isint(name.start, name.end, &id); } else n = 0; diff --git a/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c b/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c index 1c5a36f11434..87b121ac3076 100644 --- a/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c +++ b/contrib/libarchive/libarchive/archive_read_disk_entry_from_file.c @@ -526,6 +526,11 @@ setup_acls(struct archive_read_disk *a, /* Only directories can have default ACLs. */ if (S_ISDIR(archive_entry_mode(entry))) { +#if HAVE_ACL_GET_FD_NP + if (*fd >= 0) + acl = acl_get_fd_np(*fd, ACL_TYPE_DEFAULT); + else +#endif acl = acl_get_file(accpath, ACL_TYPE_DEFAULT); if (acl != NULL) { r = translate_acl(a, entry, acl, @@ -581,7 +586,10 @@ static struct { {ARCHIVE_ENTRY_ACL_ENTRY_FILE_INHERIT, ACL_ENTRY_FILE_INHERIT}, {ARCHIVE_ENTRY_ACL_ENTRY_DIRECTORY_INHERIT, ACL_ENTRY_DIRECTORY_INHERIT}, {ARCHIVE_ENTRY_ACL_ENTRY_NO_PROPAGATE_INHERIT, ACL_ENTRY_NO_PROPAGATE_INHERIT}, - {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY} + {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY}, + {ARCHIVE_ENTRY_ACL_ENTRY_SUCCESSFUL_ACCESS, ACL_ENTRY_SUCCESSFUL_ACCESS}, + {ARCHIVE_ENTRY_ACL_ENTRY_FAILED_ACCESS, ACL_ENTRY_FAILED_ACCESS}, + {ARCHIVE_ENTRY_ACL_ENTRY_INHERITED, ACL_ENTRY_INHERITED} }; #endif static int diff --git a/contrib/libarchive/libarchive/archive_read_disk_posix.c b/contrib/libarchive/libarchive/archive_read_disk_posix.c index bc7fad1d9d2e..b89370421c07 100644 --- a/contrib/libarchive/libarchive/archive_read_disk_posix.c +++ b/contrib/libarchive/libarchive/archive_read_disk_posix.c @@ -675,7 +675,7 @@ setup_suitable_read_buffer(struct archive_read_disk *a) asize = cf->min_xfer_size; /* Increase a buffer size up to 64K bytes in - * a proper incremant size. */ + * a proper increment size. */ while (asize < 1024*64) asize += incr; /* Take a margin to adjust to the filesystem @@ -1656,7 +1656,7 @@ setup_current_filesystem(struct archive_read_disk *a) archive_set_error(&a->archive, errno, "statvfs failed"); return (ARCHIVE_FAILED); } else if (xr == 1) { - /* Usuall come here unless NetBSD supports _PC_REC_XFER_ALIGN + /* Usually come here unless NetBSD supports _PC_REC_XFER_ALIGN * for pathconf() function. */ t->current_filesystem->xfer_align = sfs.f_frsize; t->current_filesystem->max_xfer_size = -1; @@ -1944,7 +1944,7 @@ setup_current_filesystem(struct archive_read_disk *a) if (nm == -1) # endif /* _PC_NAME_MAX */ /* - * Some sysmtes (HP-UX or others?) incorrectly defined + * Some systems (HP-UX or others?) incorrectly defined * NAME_MAX macro to be a smaller value. */ # if defined(NAME_MAX) && NAME_MAX >= 255 diff --git a/contrib/libarchive/libarchive/archive_read_support_filter_lz4.c b/contrib/libarchive/libarchive/archive_read_support_filter_lz4.c index 37b2f59004b1..4c66ed04dc95 100644 --- a/contrib/libarchive/libarchive/archive_read_support_filter_lz4.c +++ b/contrib/libarchive/libarchive/archive_read_support_filter_lz4.c @@ -180,7 +180,7 @@ lz4_reader_bid(struct archive_read_filter_bidder *self, return (0); bits_checked += 8; BD = buffer[5]; - /* A block maximum size shuld be more than 3. */ + /* A block maximum size should be more than 3. */ if (((BD & 0x70) >> 4) < 4) return (0); /* Reserved bits must be "0". */ @@ -417,7 +417,7 @@ lz4_filter_read_descriptor(struct archive_read_filter *self) /* Reserved bits must be zero. */ if (bd & 0x8f) goto malformed_error; - /* Get a maxinum block size. */ + /* Get a maximum block size. */ switch (read_buf[1] >> 4) { case 4: /* 64 KB */ state->flags.block_maximum_size = 64 * 1024; @@ -627,7 +627,7 @@ lz4_filter_read_default_stream(struct archive_read_filter *self, const void **p) if (state->stage == SELECT_STREAM) { state->stage = READ_DEFAULT_STREAM; - /* First, read a desciprtor. */ + /* First, read a descriptor. */ if((ret = lz4_filter_read_descriptor(self)) != ARCHIVE_OK) return (ret); state->stage = READ_DEFAULT_BLOCK; diff --git a/contrib/libarchive/libarchive/archive_read_support_filter_lzop.c b/contrib/libarchive/libarchive/archive_read_support_filter_lzop.c index 44ac9964ae11..a1c392f4f364 100644 --- a/contrib/libarchive/libarchive/archive_read_support_filter_lzop.c +++ b/contrib/libarchive/libarchive/archive_read_support_filter_lzop.c @@ -436,7 +436,7 @@ lzop_filter_read(struct archive_read_filter *self, const void **p) } /* - * Drive lzo uncompresison. + * Drive lzo uncompression. */ out_size = (lzo_uint)state->uncompressed_size; r = lzo1x_decompress_safe(b, (lzo_uint)state->compressed_size, diff --git a/contrib/libarchive/libarchive/archive_read_support_format_7zip.c b/contrib/libarchive/libarchive/archive_read_support_format_7zip.c index f8d52fb71797..3387eaf7e7f5 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_7zip.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_7zip.c @@ -552,7 +552,7 @@ skip_sfx(struct archive_read *a, ssize_t bytes_avail) /* * If bytes_avail > SFX_MIN_ADDR we do not have to call * __archive_read_seek() at this time since we have - * alredy had enough data. + * already had enough data. */ if (bytes_avail > SFX_MIN_ADDR) __archive_read_consume(a, SFX_MIN_ADDR); @@ -760,7 +760,7 @@ archive_read_format_7zip_read_header(struct archive_read *a, symsize += size; } if (symsize == 0) { - /* If there is no synname, handle it as a regular + /* If there is no symname, handle it as a regular * file. */ zip_entry->mode &= ~AE_IFMT; zip_entry->mode |= AE_IFREG; @@ -3288,7 +3288,7 @@ read_stream(struct archive_read *a, const void **buff, size_t size, return (r); /* - * Skip the bytes we alrady has skipped in skip_stream(). + * Skip the bytes we already has skipped in skip_stream(). */ while (skip_bytes) { ssize_t skipped; @@ -3506,7 +3506,7 @@ setup_decode_folder(struct archive_read *a, struct _7z_folder *folder, return (ARCHIVE_FATAL); } - /* Allocate memory for the decorded data of a sub + /* Allocate memory for the decoded data of a sub * stream. */ b[i] = malloc((size_t)zip->folder_outbytes_remaining); if (b[i] == NULL) { @@ -3591,7 +3591,7 @@ skip_stream(struct archive_read *a, size_t skip_bytes) if (zip->folder_index == 0) { /* * Optimization for a list mode. - * Avoid unncecessary decoding operations. + * Avoid unnecessary decoding operations. */ zip->si.ci.folders[zip->entry->folderIndex].skipped_bytes += skip_bytes; diff --git a/contrib/libarchive/libarchive/archive_read_support_format_iso9660.c b/contrib/libarchive/libarchive/archive_read_support_format_iso9660.c index 67628577c3eb..8c7e9eb41219 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_iso9660.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_iso9660.c @@ -322,7 +322,7 @@ struct iso9660 { struct archive_string pathname; char seenRockridge; /* Set true if RR extensions are used. */ - char seenSUSP; /* Set true if SUSP is beging used. */ + char seenSUSP; /* Set true if SUSP is being used. */ char seenJoliet; unsigned char suspOffset; diff --git a/contrib/libarchive/libarchive/archive_read_support_format_lha.c b/contrib/libarchive/libarchive/archive_read_support_format_lha.c index a7f1d8d949f5..52a5531b0ff9 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_lha.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_lha.c @@ -1711,7 +1711,7 @@ lha_crc16(uint16_t crc, const void *pp, size_t len) */ for (;len >= 8; len -= 8) { /* This if statement expects compiler optimization will - * remove the stament which will not be executed. */ + * remove the statement which will not be executed. */ #undef bswap16 #if defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio */ # define bswap16(x) _byteswap_ushort(x) diff --git a/contrib/libarchive/libarchive/archive_read_support_format_rar.c b/contrib/libarchive/libarchive/archive_read_support_format_rar.c index 9c9f6f12ec01..1e9849fdd629 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_rar.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_rar.c @@ -906,7 +906,7 @@ archive_read_format_rar_read_header(struct archive_read *a, sizeof(rar->reserved2)); } - /* Main header is password encrytped, so we cannot read any + /* Main header is password encrypted, so we cannot read any file names or any other info about files from the header. */ if (rar->main_flags & MHD_PASSWORD) { diff --git a/contrib/libarchive/libarchive/archive_read_support_format_tar.c b/contrib/libarchive/libarchive/archive_read_support_format_tar.c index 540d6789b543..54c2ea7c35ca 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_tar.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_tar.c @@ -204,13 +204,14 @@ static int archive_read_format_tar_read_header(struct archive_read *, struct archive_entry *); static int checksum(struct archive_read *, const void *); static int pax_attribute(struct archive_read *, struct tar *, - struct archive_entry *, const char *key, const char *value); + struct archive_entry *, const char *key, const char *value, + size_t value_length); static int pax_attribute_acl(struct archive_read *, struct tar *, struct archive_entry *, const char *, int); static int pax_attribute_xattr(struct archive_entry *, const char *, const char *); static int pax_header(struct archive_read *, struct tar *, - struct archive_entry *, char *attr); + struct archive_entry *, struct archive_string *); static void pax_time(const char *, int64_t *sec, long *nanos); static ssize_t readline(struct archive_read *, struct tar *, const char **, ssize_t limit, size_t *); @@ -1483,7 +1484,7 @@ header_pax_extensions(struct archive_read *a, struct tar *tar, * and then skip any fields in the standard header that were * defined in the pax header. */ - err2 = pax_header(a, tar, entry, tar->pax_header.s); + err2 = pax_header(a, tar, entry, &tar->pax_header); err = err_combine(err, err2); tar->entry_padding = 0x1ff & (-tar->entry_bytes_remaining); return (err); @@ -1564,16 +1565,17 @@ header_ustar(struct archive_read *a, struct tar *tar, */ static int pax_header(struct archive_read *a, struct tar *tar, - struct archive_entry *entry, char *attr) + struct archive_entry *entry, struct archive_string *in_as) { - size_t attr_length, l, line_length; + size_t attr_length, l, line_length, value_length; char *p; char *key, *value; struct archive_string *as; struct archive_string_conv *sconv; int err, err2; + char *attr = in_as->s; - attr_length = strlen(attr); + attr_length = in_as->length; tar->pax_hdrcharset_binary = 0; archive_string_empty(&(tar->entry_gname)); archive_string_empty(&(tar->entry_linkpath)); @@ -1638,11 +1640,13 @@ pax_header(struct archive_read *a, struct tar *tar, } *p = '\0'; - /* Identify null-terminated 'value' portion. */ value = p + 1; + /* Some values may be binary data */ + value_length = attr + line_length - 1 - value; + /* Identify this attribute and set it in the entry. */ - err2 = pax_attribute(a, tar, entry, key, value); + err2 = pax_attribute(a, tar, entry, key, value, value_length); if (err2 == ARCHIVE_FATAL) return (err2); err = err_combine(err, err2); @@ -1763,6 +1767,20 @@ pax_attribute_xattr(struct archive_entry *entry, return 0; } +static int +pax_attribute_schily_xattr(struct archive_entry *entry, + const char *name, const char *value, size_t value_length) +{ + if (strlen(name) < 14 || (memcmp(name, "SCHILY.xattr.", 13)) != 0) + return 1; + + name += 13; + + archive_entry_xattr_add_entry(entry, name, value, value_length); + + return 0; +} + static int pax_attribute_acl(struct archive_read *a, struct tar *tar, struct archive_entry *entry, const char *value, int type) @@ -1824,7 +1842,7 @@ pax_attribute_acl(struct archive_read *a, struct tar *tar, */ static int pax_attribute(struct archive_read *a, struct tar *tar, - struct archive_entry *entry, const char *key, const char *value) + struct archive_entry *entry, const char *key, const char *value, size_t value_length) { int64_t s; long n; @@ -1961,6 +1979,9 @@ pax_attribute(struct archive_read *a, struct tar *tar, } else if (strcmp(key, "SCHILY.realsize") == 0) { tar->realsize = tar_atol10(value, strlen(value)); archive_entry_set_size(entry, tar->realsize); + } else if (strncmp(key, "SCHILY.xattr.", 13) == 0) { + pax_attribute_schily_xattr(entry, key, value, + value_length); } else if (strcmp(key, "SUN.holesdata") == 0) { /* A Solaris extension for sparse. */ r = solaris_sparse_parse(a, tar, entry, value); diff --git a/contrib/libarchive/libarchive/archive_read_support_format_warc.c b/contrib/libarchive/libarchive/archive_read_support_format_warc.c index a287fc2a06b5..8eda519eb149 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_warc.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_warc.c @@ -88,7 +88,7 @@ typedef enum { WT_RVIS, /* conversion, unsupported */ WT_CONV, - /* continutation, unsupported at the moment */ + /* continuation, unsupported at the moment */ WT_CONT, /* invalid type */ LAST_WT @@ -562,7 +562,7 @@ xstrpisotime(const char *s, char **endptr) goto out; } - /* massage TM to fulfill some of POSIX' contraints */ + /* massage TM to fulfill some of POSIX' constraints */ tm.tm_year -= 1900; tm.tm_mon--; diff --git a/contrib/libarchive/libarchive/archive_read_support_format_zip.c b/contrib/libarchive/libarchive/archive_read_support_format_zip.c index 1aa74458e4ac..3388395a3f35 100644 --- a/contrib/libarchive/libarchive/archive_read_support_format_zip.c +++ b/contrib/libarchive/libarchive/archive_read_support_format_zip.c @@ -199,7 +199,7 @@ struct zip { struct trad_enc_ctx tctx; char tctx_valid; - /* WinZip AES decyption. */ + /* WinZip AES decryption. */ /* Contexts used for AES decryption. */ archive_crypto_ctx cctx; char cctx_valid; @@ -242,7 +242,7 @@ trad_enc_update_keys(struct trad_enc_ctx *ctx, uint8_t c) } static uint8_t -trad_enc_decypt_byte(struct trad_enc_ctx *ctx) +trad_enc_decrypt_byte(struct trad_enc_ctx *ctx) { unsigned temp = ctx->keys[2] | 2; return (uint8_t)((temp * (temp ^ 1)) >> 8) & 0xff; @@ -257,7 +257,7 @@ trad_enc_decrypt_update(struct trad_enc_ctx *ctx, const uint8_t *in, max = (unsigned)((in_len < out_len)? in_len: out_len); for (i = 0; i < max; i++) { - uint8_t t = in[i] ^ trad_enc_decypt_byte(ctx); + uint8_t t = in[i] ^ trad_enc_decrypt_byte(ctx); out[i] = t; trad_enc_update_keys(ctx, t); } @@ -710,7 +710,7 @@ process_extra(struct archive_read *a, const char *p, size_t extra_length, struct break; } case 0x9901: - /* WinZIp AES extra data field. */ + /* WinZip AES extra data field. */ if (p[offset + 2] == 'A' && p[offset + 3] == 'E') { /* Vendor version. */ zip_entry->aes_extra.vendor = @@ -1518,7 +1518,7 @@ read_decryption_header(struct archive_read *a) case 0x6720:/* Blowfish */ case 0x6721:/* Twofish */ case 0x6801:/* RC4 */ - /* Suuported encryption algorithm. */ + /* Supported encryption algorithm. */ break; default: archive_set_error(&a->archive, @@ -1627,7 +1627,7 @@ read_decryption_header(struct archive_read *a) __archive_read_consume(a, 4); /*return (ARCHIVE_OK); - * This is not fully implemnted yet.*/ + * This is not fully implemented yet.*/ archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, "Encrypted file is unsupported"); return (ARCHIVE_FAILED); @@ -1709,7 +1709,7 @@ init_traditional_PKWARE_decryption(struct archive_read *a) } /* - * Initialize ctx for Traditional PKWARE Decyption. + * Initialize ctx for Traditional PKWARE Decryption. */ r = trad_enc_init(&zip->tctx, passphrase, strlen(passphrase), p, ENC_HEADER_SIZE, &crcchk); diff --git a/contrib/libarchive/libarchive/archive_string.c b/contrib/libarchive/libarchive/archive_string.c index cf10f093bfc2..f738c549ffba 100644 --- a/contrib/libarchive/libarchive/archive_string.c +++ b/contrib/libarchive/libarchive/archive_string.c @@ -219,6 +219,12 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) return (as); } +struct archive_string * +archive_array_append(struct archive_string *as, const char *p, size_t s) +{ + return archive_string_append(as, p, s); +} + void archive_string_concat(struct archive_string *dest, struct archive_string *src) { @@ -597,7 +603,7 @@ archive_wstring_append_from_mbs(struct archive_wstring *dest, wcs = dest->s + dest->length; /* * We cannot use mbsrtowcs/mbstowcs here because those may convert - * extra MBS when strlen(p) > len and one wide character consis of + * extra MBS when strlen(p) > len and one wide character consists of * multi bytes. */ while (*mbs && mbs_length > 0) { @@ -1248,7 +1254,7 @@ create_sconv_object(const char *fc, const char *tc, sc->cd = iconv_open(tc, fc); if (sc->cd == (iconv_t)-1 && (sc->flag & SCONV_BEST_EFFORT)) { /* - * Unfortunaly, all of iconv implements do support + * Unfortunately, all of iconv implements do support * "CP932" character-set, so we should use "SJIS" * instead if iconv_open failed. */ @@ -1261,7 +1267,7 @@ create_sconv_object(const char *fc, const char *tc, /* * archive_mstring on Windows directly convert multi-bytes * into archive_wstring in order not to depend on locale - * so that you can do a I18N programing. This will be + * so that you can do a I18N programming. This will be * used only in archive_mstring_copy_mbs_len_l so far. */ if (flag & SCONV_FROM_CHARSET) { @@ -1726,7 +1732,7 @@ archive_string_conversion_from_charset(struct archive *a, const char *charset, * in tar or zip files. But mbstowcs/wcstombs(CRT) usually use CP_ACP * unless you use setlocale(LC_ALL, ".OCP")(specify CP_OEMCP). * So we should make a string conversion between CP_ACP and CP_OEMCP - * for compatibillty. + * for compatibility. */ #if defined(_WIN32) && !defined(__CYGWIN__) struct archive_string_conv * @@ -2220,7 +2226,7 @@ best_effort_strncat_in_locale(struct archive_string *as, const void *_p, /* * If a character is ASCII, this just copies it. If not, this - * assigns '?' charater instead but in UTF-8 locale this assigns + * assigns '?' character instead but in UTF-8 locale this assigns * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, * a Replacement Character in Unicode. */ @@ -2554,7 +2560,7 @@ utf16_to_unicode(uint32_t *pwc, const char *s, size_t n, int be) /* * Surrogate pair values(0xd800 through 0xdfff) are only - * used by UTF-16, so, after above culculation, the code + * used by UTF-16, so, after above calculation, the code * must not be surrogate values, and Unicode has no codes * larger than 0x10ffff. Thus, those are not legal Unicode * values. @@ -2903,7 +2909,7 @@ get_nfc(uint32_t uc, uint32_t uc2) /* * Normalize UTF-8/UTF-16BE characters to Form C and copy the result. * - * TODO: Convert composition exclusions,which are never converted + * TODO: Convert composition exclusions, which are never converted * from NFC,NFD,NFKC and NFKD, to Form C. */ static int @@ -3437,7 +3443,7 @@ strncat_from_utf8_libarchive2(struct archive_string *as, } /* - * As libarchie 2.x, translates the UTF-8 characters into + * As libarchive 2.x, translates the UTF-8 characters into * wide-characters in the assumption that WCS is Unicode. */ if (n < 0) { @@ -3947,7 +3953,7 @@ archive_mstring_get_mbs_l(struct archive_mstring *aes, #if defined(_WIN32) && !defined(__CYGWIN__) /* - * Internationalization programing on Windows must use Wide + * Internationalization programming on Windows must use Wide * characters because Windows platform cannot make locale UTF-8. */ if (sc != NULL && (aes->aes_set & AES_SET_WCS) != 0) { @@ -4079,7 +4085,7 @@ archive_mstring_copy_mbs_len_l(struct archive_mstring *aes, archive_string_empty(&(aes->aes_utf8)); #if defined(_WIN32) && !defined(__CYGWIN__) /* - * Internationalization programing on Windows must use Wide + * Internationalization programming on Windows must use Wide * characters because Windows platform cannot make locale UTF-8. */ if (sc == NULL) { diff --git a/contrib/libarchive/libarchive/archive_string.h b/contrib/libarchive/libarchive/archive_string.h index 60ebf0e5afa6..f5953d008913 100644 --- a/contrib/libarchive/libarchive/archive_string.h +++ b/contrib/libarchive/libarchive/archive_string.h @@ -81,6 +81,10 @@ archive_strappend_char(struct archive_string *, char); struct archive_wstring * archive_wstrappend_wchar(struct archive_wstring *, wchar_t); +/* Append a raw array to an archive_string, resizing as necessary */ +struct archive_string * +archive_array_append(struct archive_string *, const char *, size_t); + /* Convert a Unicode string to current locale and append the result. */ /* Returns -1 if conversion fails. */ int diff --git a/contrib/libarchive/libarchive/archive_string_composition.h b/contrib/libarchive/libarchive/archive_string_composition.h index be41e3365124..8902ac1f7f30 100644 --- a/contrib/libarchive/libarchive/archive_string_composition.h +++ b/contrib/libarchive/libarchive/archive_string_composition.h @@ -1009,7 +1009,7 @@ static const char u_decomposable_blocks[0x1D2+1] = { (((uc) > 0x1D244)?0:\ ccc_val[ccc_val_index[ccc_index[(uc)>>8]][((uc)>>4)&0x0F]][(uc)&0x0F]) -/* The table of the value of Canonical Cimbining Class */ +/* The table of the value of Canonical Combining Class */ static const unsigned char ccc_val[][16] = { /* idx=0: XXXX0 - XXXXF */ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, diff --git a/contrib/libarchive/libarchive/archive_write.c b/contrib/libarchive/libarchive/archive_write.c index f5a9f61bee5c..e441dbe1f1c1 100644 --- a/contrib/libarchive/libarchive/archive_write.c +++ b/contrib/libarchive/libarchive/archive_write.c @@ -231,7 +231,7 @@ __archive_write_filter(struct archive_write_filter *f, if (length == 0) return(ARCHIVE_OK); if (f->write == NULL) - /* If unset, a fatal error has already ocuured, so this filter + /* If unset, a fatal error has already occurred, so this filter * didn't open. We cannot write anything. */ return(ARCHIVE_FATAL); r = (f->write)(f, buff, length); diff --git a/contrib/libarchive/libarchive/archive_write_add_filter_xz.c b/contrib/libarchive/libarchive/archive_write_add_filter_xz.c index 46a6c38aa6e1..b0f25a6ef0ed 100644 --- a/contrib/libarchive/libarchive/archive_write_add_filter_xz.c +++ b/contrib/libarchive/libarchive/archive_write_add_filter_xz.c @@ -233,7 +233,7 @@ archive_compressor_xz_init_stream(struct archive_write_filter *f, if (f->code == ARCHIVE_FILTER_XZ) { #ifdef HAVE_LZMA_STREAM_ENCODER_MT if (data->threads != 1) { - bzero(&mt_options, sizeof(mt_options)); + memset(&mt_options, 0, sizeof(mt_options)); mt_options.threads = data->threads; mt_options.timeout = 300; mt_options.filters = data->lzmafilters; diff --git a/contrib/libarchive/libarchive/archive_write_disk_acl.c b/contrib/libarchive/libarchive/archive_write_disk_acl.c index e47384a6efaf..706ab62a07c8 100644 --- a/contrib/libarchive/libarchive/archive_write_disk_acl.c +++ b/contrib/libarchive/libarchive/archive_write_disk_acl.c @@ -124,7 +124,10 @@ static struct { {ARCHIVE_ENTRY_ACL_ENTRY_FILE_INHERIT, ACL_ENTRY_FILE_INHERIT}, {ARCHIVE_ENTRY_ACL_ENTRY_DIRECTORY_INHERIT, ACL_ENTRY_DIRECTORY_INHERIT}, {ARCHIVE_ENTRY_ACL_ENTRY_NO_PROPAGATE_INHERIT, ACL_ENTRY_NO_PROPAGATE_INHERIT}, - {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY} + {ARCHIVE_ENTRY_ACL_ENTRY_INHERIT_ONLY, ACL_ENTRY_INHERIT_ONLY}, + {ARCHIVE_ENTRY_ACL_ENTRY_SUCCESSFUL_ACCESS, ACL_ENTRY_SUCCESSFUL_ACCESS}, + {ARCHIVE_ENTRY_ACL_ENTRY_FAILED_ACCESS, ACL_ENTRY_FAILED_ACCESS}, + {ARCHIVE_ENTRY_ACL_ENTRY_INHERITED, ACL_ENTRY_INHERITED} }; #endif @@ -292,29 +295,41 @@ set_acl(struct archive *a, int fd, const char *name, } /* Try restoring the ACL through 'fd' if we can. */ -#if HAVE_ACL_SET_FD - if (fd >= 0 && acl_type == ACL_TYPE_ACCESS && acl_set_fd(fd, acl) == 0) - ret = ARCHIVE_OK; - else -#else +#if HAVE_ACL_SET_FD_NP || HAVE_ACL_SET_FD #if HAVE_ACL_SET_FD_NP - if (fd >= 0 && acl_set_fd_np(fd, acl, acl_type) == 0) - ret = ARCHIVE_OK; - else -#endif + if (fd >= 0) { + if (acl_set_fd_np(fd, acl, acl_type) == 0) +#else /* HAVE_ACL_SET_FD */ + if (fd >= 0 && acl_type == ACL_TYPE_ACCESS) { + if (acl_set_fd(fd, acl) == 0) #endif + ret = ARCHIVE_OK; + else { + if (errno == EOPNOTSUPP) { + /* Filesystem doesn't support ACLs */ + ret = ARCHIVE_OK; + } else { + archive_set_error(a, errno, + "Failed to set %s acl on fd", tname); + } + } + } else +#endif /* HAVE_ACL_SET_FD_NP || HAVE_ACL_SET_FD */ #if HAVE_ACL_SET_LINK_NP - if (acl_set_link_np(name, acl_type, acl) != 0) { - archive_set_error(a, errno, "Failed to set %s acl", tname); - ret = ARCHIVE_WARN; - } + if (acl_set_link_np(name, acl_type, acl) != 0) { #else /* TODO: Skip this if 'name' is a symlink. */ if (acl_set_file(name, acl_type, acl) != 0) { - archive_set_error(a, errno, "Failed to set %s acl", tname); - ret = ARCHIVE_WARN; - } #endif + if (errno == EOPNOTSUPP) { + /* Filesystem doesn't support ACLs */ + ret = ARCHIVE_OK; + } else { + archive_set_error(a, errno, "Failed to set %s acl", + tname); + ret = ARCHIVE_WARN; + } + } exit_free: acl_free(acl); return (ret); diff --git a/contrib/libarchive/libarchive/archive_write_set_format_7zip.c b/contrib/libarchive/libarchive/archive_write_set_format_7zip.c index abd521a4641c..41ed74daf1b9 100644 --- a/contrib/libarchive/libarchive/archive_write_set_format_7zip.c +++ b/contrib/libarchive/libarchive/archive_write_set_format_7zip.c @@ -1358,7 +1358,7 @@ make_header(struct archive_write *a, uint64_t offset, uint64_t pack_size, if (r < 0) return (r); - /* Write Nume size. */ + /* Write Name size. */ r = enc_uint64(a, zip->total_bytes_entry_name+1); if (r < 0) return (r); diff --git a/contrib/libarchive/libarchive/archive_write_set_format_pax.c b/contrib/libarchive/libarchive/archive_write_set_format_pax.c index 682c1d17ff68..1a245d742faa 100644 --- a/contrib/libarchive/libarchive/archive_write_set_format_pax.c +++ b/contrib/libarchive/libarchive/archive_write_set_format_pax.c @@ -62,10 +62,17 @@ struct pax { struct sparse_block *sparse_tail; struct archive_string_conv *sconv_utf8; int opt_binary; + + unsigned flags; +#define WRITE_SCHILY_XATTR (1 << 0) +#define WRITE_LIBARCHIVE_XATTR (1 << 1) }; static void add_pax_attr(struct archive_string *, const char *key, const char *value); +static void add_pax_attr_binary(struct archive_string *, + const char *key, + const char *value, size_t value_len); static void add_pax_attr_int(struct archive_string *, const char *key, int64_t value); static void add_pax_attr_time(struct archive_string *, @@ -136,6 +143,8 @@ archive_write_set_format_pax(struct archive *_a) "Can't allocate pax data"); return (ARCHIVE_FATAL); } + pax->flags = WRITE_LIBARCHIVE_XATTR | WRITE_SCHILY_XATTR; + a->format_data = pax; a->format_name = "pax"; a->format_options = archive_write_pax_options; @@ -274,6 +283,17 @@ add_pax_attr_int(struct archive_string *as, const char *key, int64_t value) */ static void add_pax_attr(struct archive_string *as, const char *key, const char *value) +{ + add_pax_attr_binary(as, key, value, strlen(value)); +} + +/* + * Add a key/value attribute to the pax header. This function handles + * binary values. + */ +static void +add_pax_attr_binary(struct archive_string *as, const char *key, + const char *value, size_t value_len) { int digits, i, len, next_ten; char tmp[1 + 3 * sizeof(int)]; /* < 3 base-10 digits per byte */ @@ -282,7 +302,7 @@ add_pax_attr(struct archive_string *as, const char *key, const char *value) * PAX attributes have the following layout: * <=> */ - len = 1 + (int)strlen(key) + 1 + (int)strlen(value) + 1; + len = 1 + (int)strlen(key) + 1 + (int)value_len + 1; /* * The field includes the length of the field, so @@ -313,21 +333,47 @@ add_pax_attr(struct archive_string *as, const char *key, const char *value) archive_strappend_char(as, ' '); archive_strcat(as, key); archive_strappend_char(as, '='); - archive_strcat(as, value); + archive_array_append(as, value, value_len); archive_strappend_char(as, '\n'); } +static void +archive_write_pax_header_xattr(struct pax *pax, const char *encoded_name, + const void *value, size_t value_len) +{ + struct archive_string s; + char *encoded_value; + + if (pax->flags & WRITE_LIBARCHIVE_XATTR) { + encoded_value = base64_encode((const char *)value, value_len); + + if (encoded_name != NULL && encoded_value != NULL) { + archive_string_init(&s); + archive_strcpy(&s, "LIBARCHIVE.xattr."); + archive_strcat(&s, encoded_name); + add_pax_attr(&(pax->pax_header), s.s, encoded_value); + archive_string_free(&s); + } + free(encoded_value); + } + if (pax->flags & WRITE_SCHILY_XATTR) { + archive_string_init(&s); + archive_strcpy(&s, "SCHILY.xattr."); + archive_strcat(&s, encoded_name); + add_pax_attr_binary(&(pax->pax_header), s.s, value, value_len); + archive_string_free(&s); + } +} + static int archive_write_pax_header_xattrs(struct archive_write *a, struct pax *pax, struct archive_entry *entry) { - struct archive_string s; int i = archive_entry_xattr_reset(entry); while (i--) { const char *name; const void *value; - char *encoded_value; char *url_encoded_name = NULL, *encoded_name = NULL; size_t size; int r; @@ -348,16 +394,9 @@ archive_write_pax_header_xattrs(struct archive_write *a, } } - encoded_value = base64_encode((const char *)value, size); + archive_write_pax_header_xattr(pax, encoded_name, + value, size); - if (encoded_name != NULL && encoded_value != NULL) { - archive_string_init(&s); - archive_strcpy(&s, "LIBARCHIVE.xattr."); - archive_strcat(&s, encoded_name); - add_pax_attr(&(pax->pax_header), s.s, encoded_value); - archive_string_free(&s); - } - free(encoded_value); } return (ARCHIVE_OK); } diff --git a/contrib/libarchive/libarchive/archive_write_set_format_xar.c b/contrib/libarchive/libarchive/archive_write_set_format_xar.c index ef9ad96ad51f..c9f5ac239582 100644 --- a/contrib/libarchive/libarchive/archive_write_set_format_xar.c +++ b/contrib/libarchive/libarchive/archive_write_set_format_xar.c @@ -2913,7 +2913,7 @@ compression_init_encoder_xz(struct archive *a, *strm = lzma_init_data; #ifdef HAVE_LZMA_STREAM_ENCODER_MT if (threads > 1) { - bzero(&mt_options, sizeof(mt_options)); + memset(&mt_options, 0, sizeof(mt_options)); mt_options.threads = threads; mt_options.timeout = 300; mt_options.filters = lzmafilters; diff --git a/contrib/libarchive/libarchive/archive_write_set_format_zip.c b/contrib/libarchive/libarchive/archive_write_set_format_zip.c index a69ce20526ec..38a9abfca86c 100644 --- a/contrib/libarchive/libarchive/archive_write_set_format_zip.c +++ b/contrib/libarchive/libarchive/archive_write_set_format_zip.c @@ -878,7 +878,7 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) || zip->entry_encryption == ENCRYPTION_WINZIP_AES256)) { memcpy(e, "\001\231\007\000\001\000AE", 8); - /* AES vendoer version AE-2 does not store a CRC. + /* AES vendor version AE-2 does not store a CRC. * WinZip 11 uses AE-1, which does store the CRC, * but it does not store the CRC when the file size * is less than 20 bytes. So we simulate what @@ -1013,7 +1013,7 @@ archive_write_zip_data(struct archive_write *a, const void *buff, size_t s) if (zip->entry_flags & ZIP_ENTRY_FLAG_ENCRYPTED) { switch (zip->entry_encryption) { case ENCRYPTION_TRADITIONAL: - /* Initialize traditoinal PKWARE encryption context. */ + /* Initialize traditional PKWARE encryption context. */ if (!zip->tctx_valid) { ret = init_traditional_pkware_encryption(a); if (ret != ARCHIVE_OK) @@ -1499,7 +1499,7 @@ trad_enc_update_keys(struct trad_enc_ctx *ctx, uint8_t c) } static uint8_t -trad_enc_decypt_byte(struct trad_enc_ctx *ctx) +trad_enc_decrypt_byte(struct trad_enc_ctx *ctx) { unsigned temp = ctx->keys[2] | 2; return (uint8_t)((temp * (temp ^ 1)) >> 8) & 0xff; @@ -1515,7 +1515,7 @@ trad_enc_encrypt_update(struct trad_enc_ctx *ctx, const uint8_t *in, for (i = 0; i < max; i++) { uint8_t t = in[i]; - out[i] = t ^ trad_enc_decypt_byte(ctx); + out[i] = t ^ trad_enc_decrypt_byte(ctx); trad_enc_update_keys(ctx, t); } return i; @@ -1626,7 +1626,7 @@ init_winzip_aes_encryption(struct archive_write *a) return (ARCHIVE_FAILED); } - /* Set a passowrd verification value after the 'salt'. */ + /* Set a password verification value after the 'salt'. */ salt[salt_len] = derived_key[key_len * 2]; salt[salt_len + 1] = derived_key[key_len * 2 + 1]; diff --git a/contrib/libarchive/libarchive/test/test_archive_read_add_passphrase.c b/contrib/libarchive/libarchive/test/test_archive_read_add_passphrase.c index 68dec10d9a12..0ce5a76aedbc 100644 --- a/contrib/libarchive/libarchive/test/test_archive_read_add_passphrase.c +++ b/contrib/libarchive/libarchive/test/test_archive_read_add_passphrase.c @@ -191,7 +191,7 @@ DEFINE_TEST(test_archive_read_add_passphrase_set_callback3) /* Fist call, we should get "passCallBack" as a passphrase. */ assertEqualString("passCallBack", __archive_read_next_passphrase(ar)); __archive_read_reset_passphrase(ar); - /* After reset passphrase, we should get "passCallBack"passphrase. */ + /* After reset passphrase, we should get "passCallBack" passphrase. */ assertEqualString("passCallBack", __archive_read_next_passphrase(ar)); /* Second call, we should get NULL which means all the passphrases * are passed already. */ diff --git a/contrib/libarchive/libarchive/test/test_compat_uudecode.c b/contrib/libarchive/libarchive/test/test_compat_uudecode.c index 95b1c9a8b836..cfb17c85788d 100644 --- a/contrib/libarchive/libarchive/test/test_compat_uudecode.c +++ b/contrib/libarchive/libarchive/test/test_compat_uudecode.c @@ -40,7 +40,7 @@ static char archive_data[] = { }; /* - * Compatibility: uudecode command ignores junk data placed ater the "end" + * Compatibility: uudecode command ignores junk data placed after the "end" * marker. */ DEFINE_TEST(test_compat_uudecode) diff --git a/contrib/libarchive/libarchive/test/test_read_format_cpio_afio.c b/contrib/libarchive/libarchive/test/test_read_format_cpio_afio.c index 16065eb0a512..95d3171e72cc 100644 --- a/contrib/libarchive/libarchive/test/test_read_format_cpio_afio.c +++ b/contrib/libarchive/libarchive/test/test_read_format_cpio_afio.c @@ -27,7 +27,7 @@ __FBSDID("$FreeBSD$"); /* -ecute the following to rebuild the data for this program: +execute the following to rebuild the data for this program: tail -n +33 test_read_format_cpio_afio.c | /bin/sh # How to make a sample data. diff --git a/contrib/libarchive/libarchive/test/test_read_format_zip_traditional_encryption_data.c b/contrib/libarchive/libarchive/test/test_read_format_zip_traditional_encryption_data.c index 2700be15ac7c..305261567ec8 100644 --- a/contrib/libarchive/libarchive/test/test_read_format_zip_traditional_encryption_data.c +++ b/contrib/libarchive/libarchive/test/test_read_format_zip_traditional_encryption_data.c @@ -28,7 +28,7 @@ __FBSDID("$FreeBSD$"); DEFINE_TEST(test_read_format_zip_traditional_encryption_data) { - /* This file is password protected (Traditional PKWARE Enctypted). + /* This file is password protected (Traditional PKWARE Encrypted). The headers are NOT encrypted. Password is "12345678". */ const char *refname = "test_read_format_zip_traditional_encryption_data.zip"; @@ -36,7 +36,7 @@ DEFINE_TEST(test_read_format_zip_traditional_encryption_data) struct archive *a; char buff[512]; - /* Check if running system has cryptographic functionarity. */ + /* Check if running system has cryptographic functionality. */ assert((a = archive_write_new()) != NULL); assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_zip(a)); assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_none(a)); diff --git a/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes.c b/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes.c index 082337d7eabf..cc1e3110d65b 100644 --- a/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes.c +++ b/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes.c @@ -33,7 +33,7 @@ test_winzip_aes(const char *refname, int need_libz) struct archive *a; char buff[512]; - /* Check if running system has cryptographic functionarity. */ + /* Check if running system has cryptographic functionality. */ assert((a = archive_write_new()) != NULL); assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_zip(a)); assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_none(a)); diff --git a/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes_large.c b/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes_large.c index a40d5cf1329a..6c40ae76695a 100644 --- a/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes_large.c +++ b/contrib/libarchive/libarchive/test/test_read_format_zip_winzip_aes_large.c @@ -34,7 +34,7 @@ DEFINE_TEST(test_read_format_zip_winzip_aes256_large) char buff[512]; - /* Check if running system has cryptographic functionarity. */ + /* Check if running system has cryptographic functionality. */ assert((a = archive_write_new()) != NULL); assertEqualIntA(a, ARCHIVE_OK, archive_write_set_format_zip(a)); assertEqualIntA(a, ARCHIVE_OK, archive_write_add_filter_none(a)); diff --git a/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.c b/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.c new file mode 100644 index 000000000000..7554f6d5d12a --- /dev/null +++ b/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.c @@ -0,0 +1,70 @@ +/*- + * Copyright (c) 2016 IBM Corporation + * Copyright (c) 2003-2007 Tim Kientzle + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * This test case's code has been derived from test_entry.c + */ +#include "test.h" + +DEFINE_TEST(test_schily_xattr_pax) +{ + struct archive *a; + struct archive_entry *ae; + const char *refname = "test_read_pax_schily_xattr.tar"; + const char *xname; /* For xattr tests. */ + const void *xval; /* For xattr tests. */ + size_t xsize; /* For xattr tests. */ + const char *string, *array; + + assert((a = archive_read_new()) != NULL); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_filter_all(a)); + assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a)); + + extract_reference_file(refname); + assertEqualIntA(a, ARCHIVE_OK, + archive_read_open_filename(a, refname, 10240)); + + assertEqualInt(ARCHIVE_OK, archive_read_next_header(a, &ae)); + assertEqualInt(2, archive_entry_xattr_count(ae)); + assertEqualInt(2, archive_entry_xattr_reset(ae)); + + assertEqualInt(0, archive_entry_xattr_next(ae, &xname, &xval, &xsize)); + assertEqualString(xname, "security.selinux"); + string = "system_u:object_r:unlabeled_t:s0"; + assertEqualString(xval, string); + /* the xattr's value also contains the terminating \0 */ + assertEqualInt((int)xsize, strlen(string) + 1); + + assertEqualInt(0, archive_entry_xattr_next(ae, &xname, &xval, &xsize)); + assertEqualString(xname, "security.ima"); + assertEqualInt((int)xsize, 265); + /* we only compare the first 12 bytes */ + array = "\x03\x02\x04\xb0\xe9\xd6\x79\x01\x00\x2b\xad\x1e"; + assertEqualMem(xval, array, 12); + + /* Close the archive. */ + assertEqualInt(ARCHIVE_OK, archive_read_close(a)); + assertEqualInt(ARCHIVE_OK, archive_read_free(a)); +} diff --git a/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.tar.uu b/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.tar.uu new file mode 100644 index 000000000000..52f7a8f0dc08 --- /dev/null +++ b/contrib/libarchive/libarchive/test/test_read_pax_schily_xattr.tar.uu @@ -0,0 +1,231 @@ +begin 644 test_schily_xattr_pax.tar +M+B]087A(96%D97)S+C$U,C4O8V]N9F9I;&5S```````````````````````` +M```````````````````````````````````````````````````````````` +M`````````````#`P,#`V-#0`,#`P,#`P,``P,#`P,#`P`#`P,#`P,#`P-C0W +M`#$R-S$R,C$P-3`V`#`Q,C4V-@`@>``````````````````````````````` +M```````````````````````````````````````````````````````````` +M``````````````````````````````````````````!UW6X5O?Y6: +M9^':P2MZR[4)$@W?)B6GX0U@<,0M%6YNMO%OG+IS%/.< +M,"A(N&S.F9]=!*5=\).X."2$GUGJ,0C:@+G#$M_E8UQP,LU-G(8IKW^K^<8* +M*3_.N0'%8.^$8S$`D9XOF+DK<<)U34U'_"O5/22YS96QI;G5X +M/7-Y libarchive_test'. */ diff --git a/contrib/libarchive/libarchive/xxhash.c b/contrib/libarchive/libarchive/xxhash.c index d7f8e96de6fb..6f5ba52fac32 100644 --- a/contrib/libarchive/libarchive/xxhash.c +++ b/contrib/libarchive/libarchive/xxhash.c @@ -29,10 +29,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - xxHash source repository : http://code.google.com/p/xxhash/ */ +#include "archive_platform.h" + #include #include -#include "archive_platform.h" #include "archive_xxhash.h" #ifdef HAVE_LIBLZ4 @@ -60,7 +61,7 @@ You can contact the author at : ** By default, xxHash library provides endian-independent Hash values, based on little-endian convention. ** Results are therefore identical for little-endian and big-endian CPU. ** This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -** Should endian-independance be of no importance for your application, you may set the #define below to 1. +** Should endian-independence be of no importance for your application, you may set the #define below to 1. ** It will improve speed for Big-endian CPU. ** This option has no impact on Little_Endian CPU. */ diff --git a/contrib/netbsd-tests/lib/libc/c063/t_faccessat.c b/contrib/netbsd-tests/lib/libc/c063/t_faccessat.c index 99235eaa53c4..7de4f61637a4 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_faccessat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_faccessat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_faccessat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_faccessat.c,v 1.3 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_faccessat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_faccessat.c,v 1.3 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_faccessat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/faccessat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_fchmodat.c b/contrib/netbsd-tests/lib/libc/c063/t_fchmodat.c index 6598059c5f8e..a7bb6831b86c 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_fchmodat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_fchmodat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_fchmodat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_fchmodat.c,v 1.3 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_fchmodat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_fchmodat.c,v 1.3 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_fchmodat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/fchmodat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_fchownat.c b/contrib/netbsd-tests/lib/libc/c063/t_fchownat.c index 2ca14cf35790..631d55ac521c 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_fchownat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_fchownat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_fchownat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_fchownat.c,v 1.4 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_fchownat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_fchownat.c,v 1.4 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -40,10 +42,6 @@ __RCSID("$NetBSD: t_fchownat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/fchownat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_fexecve.c b/contrib/netbsd-tests/lib/libc/c063/t_fexecve.c index 30b26a76d13e..b9aa0170f885 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_fexecve.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_fexecve.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_fexecve.c,v 1.2 2013/03/17 04:35:59 jmmv Exp $ */ +/* $NetBSD: t_fexecve.c,v 1.3 2017/01/10 15:15:09 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_fexecve.c,v 1.2 2013/03/17 04:35:59 jmmv Exp $"); +__RCSID("$NetBSD: t_fexecve.c,v 1.3 2017/01/10 15:15:09 christos Exp $"); #include @@ -70,9 +70,7 @@ ATF_TC_BODY(fexecve, tc) error = 76; else error = EXIT_FAILURE; -#ifdef __FreeBSD__ (void)close(fd); -#endif err(error, "fexecve"); } } diff --git a/contrib/netbsd-tests/lib/libc/c063/t_fstatat.c b/contrib/netbsd-tests/lib/libc/c063/t_fstatat.c index b23f62583f0a..c17961f26494 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_fstatat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_fstatat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_fstatat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_fstatat.c,v 1.3 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_fstatat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_fstatat.c,v 1.3 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_fstatat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/fstatat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_mkfifoat.c b/contrib/netbsd-tests/lib/libc/c063/t_mkfifoat.c index 3736dfa60203..5c496c8d0987 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_mkfifoat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_mkfifoat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_mkfifoat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_mkfifoat.c,v 1.3 2017/01/10 15:15:09 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_mkfifoat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_mkfifoat.c,v 1.3 2017/01/10 15:15:09 christos Exp $"); #include #include @@ -63,9 +63,7 @@ ATF_TC_BODY(mkfifoat_fd, tc) ATF_REQUIRE((fd = mkfifoat(dfd, BASEFIFO, mode)) != -1); ATF_REQUIRE(close(fd) == 0); ATF_REQUIRE(access(FIFO, F_OK) == 0); -#ifdef __FreeBSD__ (void)close(dfd); -#endif } ATF_TC(mkfifoat_fdcwd); diff --git a/contrib/netbsd-tests/lib/libc/c063/t_mknodat.c b/contrib/netbsd-tests/lib/libc/c063/t_mknodat.c index e8f71cdb8b31..7c3ab9b2edbe 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_mknodat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_mknodat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_mknodat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_mknodat.c,v 1.4 2017/01/10 15:15:09 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_mknodat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_mknodat.c,v 1.4 2017/01/10 15:15:09 christos Exp $"); #include #include @@ -80,9 +80,7 @@ ATF_TC_BODY(mknodat_fd, tc) ATF_REQUIRE((fd = mknodat(dfd, BASEFILE, mode, dev)) != -1); ATF_REQUIRE(close(fd) == 0); ATF_REQUIRE(access(FILE, F_OK) == 0); -#ifdef __FreeBSD__ (void)close(dfd); -#endif } ATF_TC(mknodat_fdcwd); diff --git a/contrib/netbsd-tests/lib/libc/c063/t_o_search.c b/contrib/netbsd-tests/lib/libc/c063/t_o_search.c index d9dbe19e899f..7cefa8bf8822 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_o_search.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_o_search.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_o_search.c,v 1.4 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_o_search.c,v 1.5 2017/01/10 22:25:01 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,9 +29,13 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_o_search.c,v 1.4 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_o_search.c,v 1.5 2017/01/10 22:25:01 christos Exp $"); #include + +#include +#include + #include #include #include @@ -40,7 +44,6 @@ __RCSID("$NetBSD: t_o_search.c,v 1.4 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include /* * dholland 20130112: disable tests that require O_SEARCH semantics diff --git a/contrib/netbsd-tests/lib/libc/c063/t_openat.c b/contrib/netbsd-tests/lib/libc/c063/t_openat.c index 5112efce3b81..f7c8c74b5b15 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_openat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_openat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_openat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_openat.c,v 1.3 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_openat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_openat.c,v 1.3 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_openat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/openat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_readlinkat.c b/contrib/netbsd-tests/lib/libc/c063/t_readlinkat.c index c9bc26762e03..cf14652a9efe 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_readlinkat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_readlinkat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_readlinkat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_readlinkat.c,v 1.4 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_readlinkat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_readlinkat.c,v 1.4 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_readlinkat.c,v 1.3 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/readlinkat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_unlinkat.c b/contrib/netbsd-tests/lib/libc/c063/t_unlinkat.c index 220c4b2fc40d..9897b928c516 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_unlinkat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_unlinkat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_unlinkat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_unlinkat.c,v 1.3 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_unlinkat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_unlinkat.c,v 1.3 2017/01/10 15:13:56 christos Exp $"); +#include +#include #include #include #include @@ -39,10 +41,6 @@ __RCSID("$NetBSD: t_unlinkat.c,v 1.2 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif #define DIR "dir" #define FILE "dir/unlinkat" diff --git a/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c b/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c index bbfa28bd01df..682c2df06974 100644 --- a/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c +++ b/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_utimensat.c,v 1.5 2013/03/17 04:46:06 jmmv Exp $ */ +/* $NetBSD: t_utimensat.c,v 1.6 2017/01/10 15:13:56 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -29,8 +29,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_utimensat.c,v 1.5 2013/03/17 04:46:06 jmmv Exp $"); +__RCSID("$NetBSD: t_utimensat.c,v 1.6 2017/01/10 15:13:56 christos Exp $"); +#include +#include +#include #include #include #include @@ -39,11 +42,6 @@ __RCSID("$NetBSD: t_utimensat.c,v 1.5 2013/03/17 04:46:06 jmmv Exp $"); #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif -#include #define DIR "dir" #define FILE "dir/utimensat" diff --git a/contrib/netbsd-tests/lib/libc/gen/posix_spawn/t_fileactions.c b/contrib/netbsd-tests/lib/libc/gen/posix_spawn/t_fileactions.c index 5bbf337efd9e..74009a805758 100644 --- a/contrib/netbsd-tests/lib/libc/gen/posix_spawn/t_fileactions.c +++ b/contrib/netbsd-tests/lib/libc/gen/posix_spawn/t_fileactions.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_fileactions.c,v 1.5 2012/04/09 19:42:07 martin Exp $ */ +/* $NetBSD: t_fileactions.c,v 1.6 2017/01/10 22:36:29 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. @@ -31,10 +31,11 @@ */ -#ifdef __FreeBSD__ -#include -#endif #include + +#include +#include + #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include ATF_TC(t_spawn_openmode); diff --git a/contrib/netbsd-tests/lib/libc/gen/t_assert.c b/contrib/netbsd-tests/lib/libc/gen/t_assert.c index a09c13054fe4..ce73015142d6 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_assert.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_assert.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_assert.c,v 1.2 2011/06/14 05:28:00 jruoho Exp $ */ +/* $NetBSD: t_assert.c,v 1.3 2017/01/10 15:17:57 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,8 +29,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_assert.c,v 1.2 2011/06/14 05:28:00 jruoho Exp $"); +__RCSID("$NetBSD: t_assert.c,v 1.3 2017/01/10 15:17:57 christos Exp $"); +#include +#include +#include #include #include @@ -40,11 +43,6 @@ __RCSID("$NetBSD: t_assert.c,v 1.2 2011/06/14 05:28:00 jruoho Exp $"); #include #include -#ifdef __FreeBSD__ -#include -#include -#include - static void disable_corefile(void) { @@ -55,7 +53,6 @@ disable_corefile(void) ATF_REQUIRE(setrlimit(RLIMIT_CORE, &limits) == 0); } -#endif static void handler(int); @@ -82,9 +79,7 @@ ATF_TC_BODY(assert_false, tc) if (pid == 0) { -#ifdef __FreeBSD__ disable_corefile(); -#endif (void)closefrom(0); (void)memset(&sa, 0, sizeof(struct sigaction)); @@ -122,9 +117,7 @@ ATF_TC_BODY(assert_true, tc) if (pid == 0) { -#ifdef __FreeBSD__ disable_corefile(); -#endif (void)closefrom(0); (void)memset(&sa, 0, sizeof(struct sigaction)); diff --git a/contrib/netbsd-tests/lib/libc/gen/t_dir.c b/contrib/netbsd-tests/lib/libc/gen/t_dir.c index cc38c3b0e772..c05873870a3b 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_dir.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_dir.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_dir.c,v 1.6 2013/10/19 17:45:00 christos Exp $ */ +/* $NetBSD: t_dir.c,v 1.8 2017/01/11 07:26:17 christos Exp $ */ /*- * Copyright (c) 2010 The NetBSD Foundation, Inc. @@ -26,22 +26,19 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#include - +#include #include +#include #include #include +#include #include #include #include #include #include -#include -#ifdef __FreeBSD__ -#include -#endif ATF_TC(seekdir_basic); ATF_TC_HEAD(seekdir_basic, tc) @@ -58,13 +55,12 @@ ATF_TC_BODY(seekdir_basic, tc) struct dirent *entry; long here; -#ifdef __FreeBSD__ -#define CREAT(x, m) do { \ - int _creat_fd; \ - ATF_REQUIRE_MSG((_creat_fd = creat((x), (m))), \ - "creat(%s, %x) failed: %s", (x), (m), \ - strerror(errno)); \ - (void)close(_creat_fd); \ +#define CREAT(x, m) do { \ + int _creat_fd; \ + ATF_REQUIRE_MSG((_creat_fd = creat((x), (m))) != -1, \ + "creat(%s, %x) failed: %s", (x), (m), \ + strerror(errno)); \ + (void)close(_creat_fd); \ } while(0); ATF_REQUIRE_MSG(mkdir("t", 0755) == 0, @@ -72,12 +68,6 @@ ATF_TC_BODY(seekdir_basic, tc) CREAT("t/a", 0600); CREAT("t/b", 0600); CREAT("t/c", 0600); -#else - mkdir("t", 0755); - creat("t/a", 0600); - creat("t/b", 0600); - creat("t/c", 0600); -#endif dp = opendir("t"); if ( dp == NULL) @@ -90,10 +80,8 @@ ATF_TC_BODY(seekdir_basic, tc) /* get first entry */ entry = readdir(dp); here = telldir(dp); -#ifdef __FreeBSD__ ATF_REQUIRE_MSG(here != -1, "telldir failed: %s", strerror(errno)); -#endif /* get second entry */ entry = readdir(dp); @@ -137,9 +125,7 @@ ATF_TC_BODY(seekdir_basic, tc) atf_tc_fail("3rd seekdir found wrong name"); closedir(dp); -#ifdef __FreeBSD__ free(wasname); -#endif } /* There is no sbrk on AArch64 and RISC-V */ diff --git a/contrib/netbsd-tests/lib/libc/gen/t_ftok.c b/contrib/netbsd-tests/lib/libc/gen/t_ftok.c index 718d310a16b3..4c1ab18950eb 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_ftok.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_ftok.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_ftok.c,v 1.1 2011/11/08 05:47:00 jruoho Exp $ */ +/* $NetBSD: t_ftok.c,v 1.2 2017/01/10 15:19:52 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_ftok.c,v 1.1 2011/11/08 05:47:00 jruoho Exp $"); +__RCSID("$NetBSD: t_ftok.c,v 1.2 2017/01/10 15:19:52 christos Exp $"); #include #include @@ -68,9 +68,7 @@ ATF_TC_BODY(ftok_link, tc) fd = open(path, O_RDONLY | O_CREAT); ATF_REQUIRE(fd >= 0); -#ifdef __FreeBSD__ (void)close(fd); -#endif ATF_REQUIRE(link(path, hlnk) == 0); ATF_REQUIRE(symlink(path, slnk) == 0); diff --git a/contrib/netbsd-tests/lib/libc/gen/t_humanize_number.c b/contrib/netbsd-tests/lib/libc/gen/t_humanize_number.c index 53055d277648..17eac877f5d4 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_humanize_number.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_humanize_number.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_humanize_number.c,v 1.8 2012/03/18 07:14:08 jruoho Exp $ */ +/* $NetBSD: t_humanize_number.c,v 1.9 2017/01/10 15:20:44 christos Exp $ */ /*- * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. @@ -247,9 +247,7 @@ ATF_TC_BODY(humanize_number_basic, tc) newline(); atf_tc_fail_nonfatal("Failed for table entry %d", i); } -#ifdef __FreeBSD__ free(buf); -#endif } ATF_TC(humanize_number_big); diff --git a/contrib/netbsd-tests/lib/libc/gen/t_raise.c b/contrib/netbsd-tests/lib/libc/gen/t_raise.c index d6f888fde127..91a8c37cbfa0 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_raise.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_raise.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_raise.c,v 1.5 2011/05/10 12:43:42 jruoho Exp $ */ +/* $NetBSD: t_raise.c,v 1.6 2016/11/03 22:08:31 kamil Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_raise.c,v 1.5 2011/05/10 12:43:42 jruoho Exp $"); +__RCSID("$NetBSD: t_raise.c,v 1.6 2016/11/03 22:08:31 kamil Exp $"); #include @@ -180,7 +180,7 @@ ATF_TC_BODY(raise_stress, tc) (void)raise(SIGUSR1); if (count != maxiter) - atf_tc_fail("not all signals were catched"); + atf_tc_fail("not all signals were caught"); } ATF_TP_ADD_TCS(tp) diff --git a/contrib/netbsd-tests/lib/libc/gen/t_setdomainname.c b/contrib/netbsd-tests/lib/libc/gen/t_setdomainname.c index f51eb2a9d893..02dd17694740 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_setdomainname.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_setdomainname.c @@ -64,7 +64,7 @@ ATF_TC_BODY(setdomainname_basic, tc) (void)memset(name, 0, sizeof(name)); #ifdef __FreeBSD__ - /* + /* * Sanity checks to ensure that the wrong invariant isn't being * tested for per PR # 181127 */ diff --git a/contrib/netbsd-tests/lib/libc/gen/t_sethostname.c b/contrib/netbsd-tests/lib/libc/gen/t_sethostname.c index 1972f7d5747b..136fa7cdfca0 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_sethostname.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_sethostname.c @@ -66,7 +66,7 @@ ATF_TC_BODY(sethostname_basic, tc) (void)memset(name, 0, sizeof(name)); #ifdef __FreeBSD__ - /* + /* * Sanity checks to ensure that the wrong invariant isn't being * tested for per PR # 181127 */ diff --git a/contrib/netbsd-tests/lib/libc/gen/t_sleep.c b/contrib/netbsd-tests/lib/libc/gen/t_sleep.c index e89df690e4e7..5e364569fbc2 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_sleep.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_sleep.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_sleep.c,v 1.9 2016/08/11 21:34:11 kre Exp $ */ +/* $NetBSD: t_sleep.c,v 1.11 2017/01/10 15:43:59 maya Exp $ */ /*- * Copyright (c) 2006 Frank Kardel @@ -26,8 +26,17 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifdef __FreeBSD__ +#include +#endif +#include +#include +#include +#include /* for TIMESPEC_TO_TIMEVAL on FreeBSD */ + #include #include +#include #include #include #include @@ -35,10 +44,6 @@ #include #include -#include -#include -#include - #include "isqemu.h" #define BILLION 1000000000LL /* nano-seconds per second */ @@ -49,11 +54,6 @@ #define KEVNT_TIMEOUT 10300 /* measured in milli-seconds */ #define FUZZ (40 * MILLION) /* scheduling fuzz accepted - 40 ms */ -#ifdef __FreeBSD__ -#include -#include -#endif - /* * Timer notes * diff --git a/contrib/netbsd-tests/lib/libc/gen/t_time.c b/contrib/netbsd-tests/lib/libc/gen/t_time.c index 290540366d1d..15a8d585c847 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_time.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_time.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_time.c,v 1.3 2014/10/31 12:22:38 justin Exp $ */ +/* $NetBSD: t_time.c,v 1.4 2017/01/10 15:32:46 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,11 +29,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_time.c,v 1.3 2014/10/31 12:22:38 justin Exp $"); +__RCSID("$NetBSD: t_time.c,v 1.4 2017/01/10 15:32:46 christos Exp $"); -#ifdef __FreeBSD__ -#include -#endif #include #include #include @@ -41,6 +38,7 @@ __RCSID("$NetBSD: t_time.c,v 1.3 2014/10/31 12:22:38 justin Exp $"); #include #include #include +#include #include ATF_TC(time_copy); diff --git a/contrib/netbsd-tests/lib/libc/gen/t_ttyname.c b/contrib/netbsd-tests/lib/libc/gen/t_ttyname.c index 1c813fb0ec48..61121b8b9c0d 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_ttyname.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_ttyname.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_ttyname.c,v 1.3 2011/05/01 18:14:01 jruoho Exp $ */ +/* $NetBSD: t_ttyname.c,v 1.4 2017/01/10 15:33:40 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_ttyname.c,v 1.3 2011/05/01 18:14:01 jruoho Exp $"); +__RCSID("$NetBSD: t_ttyname.c,v 1.4 2017/01/10 15:33:40 christos Exp $"); #include #include @@ -78,9 +78,7 @@ ATF_TC_BODY(ttyname_err, tc) ATF_REQUIRE(ttyname(fd) == NULL); ATF_REQUIRE(errno == ENOTTY); -#ifdef __FreeBSD__ (void)close(fd); -#endif } } diff --git a/contrib/netbsd-tests/lib/libc/gen/t_vis.c b/contrib/netbsd-tests/lib/libc/gen/t_vis.c index 76f85f495b3f..adb0930a300a 100644 --- a/contrib/netbsd-tests/lib/libc/gen/t_vis.c +++ b/contrib/netbsd-tests/lib/libc/gen/t_vis.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_vis.c,v 1.8 2015/05/23 14:02:11 christos Exp $ */ +/* $NetBSD: t_vis.c,v 1.9 2017/01/10 15:16:57 christos Exp $ */ /*- * Copyright (c) 2002 The NetBSD Foundation, Inc. @@ -144,9 +144,7 @@ ATF_TC_BODY(strunvis_hex, tc) } } -/* Begin FreeBSD: ^/stable/10 doesn't have VIS_NOLOCALE */ #ifdef VIS_NOLOCALE -/* End FreeBSD */ ATF_TC(strvis_locale); ATF_TC_HEAD(strvis_locale, tc) { @@ -175,9 +173,7 @@ ATF_TC_BODY(strvis_locale, tc) setlocale(LC_CTYPE, ol); free(ol); } -/* Begin FreeBSD: ^/stable/10 doesn't have VIS_NOLOCALE */ #endif /* VIS_NOLOCALE */ -/* End FreeBSD */ ATF_TP_ADD_TCS(tp) { @@ -186,13 +182,9 @@ ATF_TP_ADD_TCS(tp) ATF_TP_ADD_TC(tp, strvis_null); ATF_TP_ADD_TC(tp, strvis_empty); ATF_TP_ADD_TC(tp, strunvis_hex); -/* Begin FreeBSD: ^/stable/10 doesn't have VIS_NOLOCALE */ #ifdef VIS_NOLOCALE -/* End FreeBSD */ ATF_TP_ADD_TC(tp, strvis_locale); -/* Begin FreeBSD: ^/stable/10 doesn't have VIS_NOLOCALE */ #endif /* VIS_NOLOCALE */ -/* End FreeBSD */ return atf_no_error(); } diff --git a/contrib/netbsd-tests/lib/libc/regex/debug.c b/contrib/netbsd-tests/lib/libc/regex/debug.c index 3fc6d5b5867a..22f0b90d2272 100644 --- a/contrib/netbsd-tests/lib/libc/regex/debug.c +++ b/contrib/netbsd-tests/lib/libc/regex/debug.c @@ -48,9 +48,7 @@ #ifdef __NetBSD__ static void s_print(struct re_guts *, FILE *); static char *regchar(int); -#endif -#ifdef __NetBSD__ /* * regprint - print a regexp for debugging */ diff --git a/contrib/netbsd-tests/lib/libc/stdio/t_printf.c b/contrib/netbsd-tests/lib/libc/stdio/t_printf.c index d6ca662273fc..95b4b2c9dffe 100644 --- a/contrib/netbsd-tests/lib/libc/stdio/t_printf.c +++ b/contrib/netbsd-tests/lib/libc/stdio/t_printf.c @@ -36,10 +36,6 @@ #include #include -#ifndef __NetBSD__ -#include -#endif - ATF_TC(snprintf_c99); ATF_TC_HEAD(snprintf_c99, tc) { diff --git a/contrib/netbsd-tests/lib/libc/string/t_strchr.c b/contrib/netbsd-tests/lib/libc/string/t_strchr.c index 4556b2c245ca..5dd9a62213ab 100644 --- a/contrib/netbsd-tests/lib/libc/string/t_strchr.c +++ b/contrib/netbsd-tests/lib/libc/string/t_strchr.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_strchr.c,v 1.1 2011/07/07 08:59:33 jruoho Exp $ */ +/* $NetBSD: t_strchr.c,v 1.2 2017/01/10 15:34:49 christos Exp $ */ /* * Written by J.T. Conklin @@ -58,12 +58,10 @@ ATF_TC_HEAD(strchr_basic, tc) ATF_TC_BODY(strchr_basic, tc) { -#ifdef __FreeBSD__ void *dl_handle; -#endif - unsigned int t, a; char *off; char buf[32]; + unsigned int t, a; const char *tab[] = { "", @@ -248,12 +246,8 @@ ATF_TC_BODY(strchr_basic, tc) "abcdefgh/abcdefgh/", }; -#ifdef __FreeBSD__ dl_handle = dlopen(NULL, RTLD_LAZY); strchr_fn = dlsym(dl_handle, "test_strlen"); -#else - strchr_fn = dlsym(dlopen(0, RTLD_LAZY), "test_strchr"); -#endif if (!strchr_fn) strchr_fn = strchr; @@ -288,9 +282,7 @@ ATF_TC_BODY(strchr_basic, tc) verify_strchr(buf + a, 0xff, t, a); } } -#ifdef __FreeBSD__ (void)dlclose(dl_handle); -#endif } ATF_TP_ADD_TCS(tp) diff --git a/contrib/netbsd-tests/lib/libc/string/t_strerror.c b/contrib/netbsd-tests/lib/libc/string/t_strerror.c index 888a8261f8d5..99b95b4266c6 100644 --- a/contrib/netbsd-tests/lib/libc/string/t_strerror.c +++ b/contrib/netbsd-tests/lib/libc/string/t_strerror.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_strerror.c,v 1.3 2011/05/10 06:55:27 jruoho Exp $ */ +/* $NetBSD: t_strerror.c,v 1.4 2017/01/10 20:35:49 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,18 +29,15 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_strerror.c,v 1.3 2011/05/10 06:55:27 jruoho Exp $"); +__RCSID("$NetBSD: t_strerror.c,v 1.4 2017/01/10 20:35:49 christos Exp $"); #include #include +#include /* Needed for sys_nerr on FreeBSD */ #include #include #include -#ifdef __FreeBSD__ -#include -#endif - ATF_TC(strerror_basic); ATF_TC_HEAD(strerror_basic, tc) { diff --git a/contrib/netbsd-tests/lib/libc/sys/t_access.c b/contrib/netbsd-tests/lib/libc/sys/t_access.c index 69d2df293bc8..e15241febc64 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_access.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_access.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_access.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $ */ +/* $NetBSD: t_access.c,v 1.2 2017/01/10 22:36:29 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,7 +29,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_access.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $"); +__RCSID("$NetBSD: t_access.c,v 1.2 2017/01/10 22:36:29 christos Exp $"); + +#include + +#include #include #include @@ -38,13 +42,6 @@ __RCSID("$NetBSD: t_access.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $"); #include #include -#include - -#ifdef __FreeBSD__ -#include -#include -#endif - static const char path[] = "access"; static const int mode[4] = { R_OK, W_OK, X_OK, F_OK }; diff --git a/contrib/netbsd-tests/lib/libc/sys/t_chroot.c b/contrib/netbsd-tests/lib/libc/sys/t_chroot.c index 651dc10201f3..1f1c49b346ae 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_chroot.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_chroot.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_chroot.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $ */ +/* $NetBSD: t_chroot.c,v 1.2 2017/01/10 22:36:29 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -29,9 +29,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_chroot.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $"); +__RCSID("$NetBSD: t_chroot.c,v 1.2 2017/01/10 22:36:29 christos Exp $"); #include +#include #include #include @@ -42,10 +43,6 @@ __RCSID("$NetBSD: t_chroot.c,v 1.1 2011/07/07 06:57:53 jruoho Exp $"); #include #include -#ifdef __FreeBSD__ -#include -#endif - ATF_TC(chroot_basic); ATF_TC_HEAD(chroot_basic, tc) { diff --git a/contrib/netbsd-tests/lib/libc/sys/t_mincore.c b/contrib/netbsd-tests/lib/libc/sys/t_mincore.c index c31170f94ddd..ab8c33f0082d 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_mincore.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_mincore.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_mincore.c,v 1.8 2012/06/08 07:18:58 martin Exp $ */ +/* $NetBSD: t_mincore.c,v 1.9 2017/01/10 22:36:29 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -59,9 +59,10 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_mincore.c,v 1.8 2012/06/08 07:18:58 martin Exp $"); +__RCSID("$NetBSD: t_mincore.c,v 1.9 2017/01/10 22:36:29 christos Exp $"); #include +#include #include #include @@ -74,10 +75,6 @@ __RCSID("$NetBSD: t_mincore.c,v 1.8 2012/06/08 07:18:58 martin Exp $"); #include #include -#ifdef __FreeBSD__ -#include -#endif - static long page = 0; static const char path[] = "mincore"; static size_t check_residency(void *, size_t); diff --git a/contrib/netbsd-tests/lib/libc/sys/t_mmap.c b/contrib/netbsd-tests/lib/libc/sys/t_mmap.c index 0ce4a84eea2e..738758c637f7 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_mmap.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_mmap.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_mmap.c,v 1.9 2015/02/28 13:57:08 martin Exp $ */ +/* $NetBSD: t_mmap.c,v 1.10 2017/01/10 22:36:29 christos Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. @@ -55,10 +55,11 @@ * SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_mmap.c,v 1.9 2015/02/28 13:57:08 martin Exp $"); +__RCSID("$NetBSD: t_mmap.c,v 1.10 2017/01/10 22:36:29 christos Exp $"); #include #include +#include #include #include #include @@ -78,7 +79,6 @@ __RCSID("$NetBSD: t_mmap.c,v 1.9 2015/02/28 13:57:08 martin Exp $"); #ifdef __FreeBSD__ #include -#include #include #endif diff --git a/contrib/netbsd-tests/lib/libc/sys/t_unlink.c b/contrib/netbsd-tests/lib/libc/sys/t_unlink.c index e72974c7d133..b504bb06debc 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_unlink.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_unlink.c @@ -63,12 +63,7 @@ ATF_TC_BODY(unlink_basic, tc) ATF_REQUIRE(unlink(path) == 0); errno = 0; -#ifdef __FreeBSD__ - ATF_REQUIRE_ERRNO(ENOENT, (fd = open(path, O_RDONLY)) == -1); - (void)close(fd); -#else ATF_REQUIRE_ERRNO(ENOENT, open(path, O_RDONLY) == -1); -#endif } } @@ -128,12 +123,7 @@ ATF_TC_BODY(unlink_fifo, tc) ATF_REQUIRE(unlink(path) == 0); errno = 0; -#ifdef __FreeBSD__ - ATF_REQUIRE_ERRNO(ENOENT, (fd = open(path, O_RDONLY)) == -1); - (void)close(fd); -#else ATF_REQUIRE_ERRNO(ENOENT, open(path, O_RDONLY) == -1); -#endif } ATF_TC_CLEANUP(unlink_fifo, tc) diff --git a/contrib/netbsd-tests/lib/libc/sys/t_wait.c b/contrib/netbsd-tests/lib/libc/sys/t_wait.c index 027c40dd67b6..86532652c6e6 100644 --- a/contrib/netbsd-tests/lib/libc/sys/t_wait.c +++ b/contrib/netbsd-tests/lib/libc/sys/t_wait.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_wait.c,v 1.4 2016/04/27 21:14:24 christos Exp $ */ +/* $NetBSD: t_wait.c,v 1.7 2016/11/06 15:04:14 kamil Exp $ */ /*- * Copyright (c) 2016 The NetBSD Foundation, Inc. @@ -29,7 +29,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ #include -__RCSID("$NetBSD: t_wait.c,v 1.4 2016/04/27 21:14:24 christos Exp $"); +__RCSID("$NetBSD: t_wait.c,v 1.7 2016/11/06 15:04:14 kamil Exp $"); #include #include @@ -64,22 +64,6 @@ ATF_TC_BODY(wait6_invalid, tc) && errno == EINVAL); } -ATF_TC(wait6_noproc); -ATF_TC_HEAD(wait6_noproc, tc) -{ - atf_tc_set_md_var(tc, "descr", - "Test that wait6(2) returns ECHILD with for no processes"); -} - -ATF_TC_BODY(wait6_noproc, tc) -{ - siginfo_t si; - struct wrusage wru; - int st; - ATF_REQUIRE(wait6(P_ALL, 0, &st, WEXITED, &wru, &si) == -1 - && errno == ECHILD); -} - ATF_TC(wait6_exited); ATF_TC_HEAD(wait6_exited, tc) { @@ -96,12 +80,12 @@ ATF_TC_BODY(wait6_exited, tc) switch (pid = fork()) { case -1: - ATF_REQUIRE(pid > 0); + ATF_REQUIRE(pid > 0); case 0: exit(0x5a5a5a5a); /*NOTREACHED*/ default: - ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); ATF_REQUIRE(WIFEXITED(st) && WEXITSTATUS(st) == 0x5a); ATF_REQUIRE(si.si_status = 0x5a5a5a5a); ATF_REQUIRE(si.si_pid == pid); @@ -134,10 +118,10 @@ ATF_TC_BODY(wait6_terminated, tc) sleep(100); /*FALLTHROUGH*/ case -1: - ATF_REQUIRE(pid > 0); + ATF_REQUIRE(pid > 0); default: ATF_REQUIRE(kill(pid, SIGTERM) == 0); - ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); ATF_REQUIRE(WIFSIGNALED(st) && WTERMSIG(st) == SIGTERM); ATF_REQUIRE(si.si_status == SIGTERM); ATF_REQUIRE(si.si_pid == pid); @@ -172,9 +156,9 @@ ATF_TC_BODY(wait6_coredumped, tc) *(char *)8 = 0; /*FALLTHROUGH*/ case -1: - ATF_REQUIRE(pid > 0); + ATF_REQUIRE(pid > 0); default: - ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); ATF_REQUIRE(WIFSIGNALED(st) && WTERMSIG(st) == SIGSEGV && WCOREDUMP(st)); ATF_REQUIRE(si.si_status == SIGSEGV); @@ -210,11 +194,14 @@ ATF_TC_BODY(wait6_stop_and_go, tc) sleep(100); /*FALLTHROUGH*/ case -1: - ATF_REQUIRE(pid > 0); + ATF_REQUIRE(pid > 0); default: ATF_REQUIRE(kill(pid, SIGSTOP) == 0); - ATF_REQUIRE(wait6(P_PID, pid, &st, WSTOPPED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WSTOPPED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); + ATF_REQUIRE(!WIFSIGNALED(st)); ATF_REQUIRE(WIFSTOPPED(st) && WSTOPSIG(st) == SIGSTOP); + ATF_REQUIRE(!WIFCONTINUED(st)); ATF_REQUIRE(si.si_status == SIGSTOP); ATF_REQUIRE(si.si_pid == pid); ATF_REQUIRE(si.si_uid == getuid()); @@ -225,8 +212,11 @@ ATF_TC_BODY(wait6_stop_and_go, tc) #endif ATF_REQUIRE(kill(pid, SIGCONT) == 0); - ATF_REQUIRE(wait6(P_PID, pid, &st, WCONTINUED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WCONTINUED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); + ATF_REQUIRE(!WIFSIGNALED(st)); ATF_REQUIRE(WIFCONTINUED(st)); + ATF_REQUIRE(!WIFSTOPPED(st)); ATF_REQUIRE(si.si_status == SIGCONT); ATF_REQUIRE(si.si_pid == pid); ATF_REQUIRE(si.si_uid == getuid()); @@ -237,8 +227,11 @@ ATF_TC_BODY(wait6_stop_and_go, tc) #endif ATF_REQUIRE(kill(pid, SIGQUIT) == 0); - ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); ATF_REQUIRE(WIFSIGNALED(st) && WTERMSIG(st) == SIGQUIT); + ATF_REQUIRE(!WIFSTOPPED(st)); + ATF_REQUIRE(!WIFCONTINUED(st)); ATF_REQUIRE(si.si_status == SIGQUIT); ATF_REQUIRE(si.si_pid == pid); ATF_REQUIRE(si.si_uid == getuid()); @@ -251,15 +244,80 @@ ATF_TC_BODY(wait6_stop_and_go, tc) } } +ATF_TC(wait6_stopgo_loop); +ATF_TC_HEAD(wait6_stopgo_loop, tc) +{ + atf_tc_set_md_var(tc, "descr", + "Test that wait6(2) handled stopped/continued process loop"); +} + +ATF_TC_BODY(wait6_stopgo_loop, tc) +{ + siginfo_t si; + struct wrusage wru; + int st; + pid_t pid; + static const struct rlimit rl = { 0, 0 }; + size_t N = 100; + + ATF_REQUIRE(setrlimit(RLIMIT_CORE, &rl) == 0); + switch (pid = fork()) { + case 0: + sleep(100); + /*FALLTHROUGH*/ + case -1: + ATF_REQUIRE(pid > 0); + } + + printf("Before loop of SIGSTOP/SIGCONT sequence %zu times\n", N); + while (N --> 0) { + ATF_REQUIRE(kill(pid, SIGSTOP) == 0); + ATF_REQUIRE(wait6(P_PID, pid, &st, WSTOPPED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); + ATF_REQUIRE(!WIFSIGNALED(st)); + ATF_REQUIRE(WIFSTOPPED(st) && WSTOPSIG(st) == SIGSTOP); + ATF_REQUIRE(!WIFCONTINUED(st)); + ATF_REQUIRE(si.si_status == SIGSTOP); + ATF_REQUIRE(si.si_pid == pid); + ATF_REQUIRE(si.si_uid == getuid()); + ATF_REQUIRE(si.si_code == CLD_STOPPED); + + ATF_REQUIRE(kill(pid, SIGCONT) == 0); + ATF_REQUIRE(wait6(P_PID, pid, &st, WCONTINUED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); + ATF_REQUIRE(!WIFSIGNALED(st)); + ATF_REQUIRE(WIFCONTINUED(st)); + ATF_REQUIRE(!WIFSTOPPED(st)); + ATF_REQUIRE(si.si_status == SIGCONT); + ATF_REQUIRE(si.si_pid == pid); + ATF_REQUIRE(si.si_uid == getuid()); + ATF_REQUIRE(si.si_code == CLD_CONTINUED); + } + ATF_REQUIRE(kill(pid, SIGQUIT) == 0); + ATF_REQUIRE(wait6(P_PID, pid, &st, WEXITED, &wru, &si) == pid); + ATF_REQUIRE(!WIFEXITED(st)); + ATF_REQUIRE(WIFSIGNALED(st) && WTERMSIG(st) == SIGQUIT); + ATF_REQUIRE(!WIFSTOPPED(st)); + ATF_REQUIRE(!WIFCONTINUED(st)); + ATF_REQUIRE(si.si_status == SIGQUIT); + ATF_REQUIRE(si.si_pid == pid); + ATF_REQUIRE(si.si_uid == getuid()); + ATF_REQUIRE(si.si_code == CLD_KILLED); +#ifdef __NetBSD__ + printf("user: %ju system: %ju\n", (uintmax_t)si.si_utime, + (uintmax_t)si.si_utime); +#endif +} + ATF_TP_ADD_TCS(tp) { ATF_TP_ADD_TC(tp, wait6_invalid); - ATF_TP_ADD_TC(tp, wait6_noproc); ATF_TP_ADD_TC(tp, wait6_exited); ATF_TP_ADD_TC(tp, wait6_terminated); ATF_TP_ADD_TC(tp, wait6_coredumped); ATF_TP_ADD_TC(tp, wait6_stop_and_go); + ATF_TP_ADD_TC(tp, wait6_stopgo_loop); return atf_no_error(); } diff --git a/contrib/netbsd-tests/lib/libc/t_cdb.c b/contrib/netbsd-tests/lib/libc/t_cdb.c index 5e88e6523a26..97da4a34f412 100644 --- a/contrib/netbsd-tests/lib/libc/t_cdb.c +++ b/contrib/netbsd-tests/lib/libc/t_cdb.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_cdb.c,v 1.1 2012/09/27 00:38:57 joerg Exp $ */ +/* $NetBSD: t_cdb.c,v 1.2 2017/01/10 22:24:29 christos Exp $ */ /*- * Copyright (c) 2012 The NetBSD Foundation, Inc. * All rights reserved. @@ -32,9 +32,12 @@ */ #include -__RCSID("$NetBSD: t_cdb.c,v 1.1 2012/09/27 00:38:57 joerg Exp $"); +__RCSID("$NetBSD: t_cdb.c,v 1.2 2017/01/10 22:24:29 christos Exp $"); #include + +#include + #include #include #include diff --git a/contrib/netbsd-tests/lib/libc/ttyio/t_ttyio.c b/contrib/netbsd-tests/lib/libc/ttyio/t_ttyio.c index 1e42536f81a7..7f60d786a75f 100644 --- a/contrib/netbsd-tests/lib/libc/ttyio/t_ttyio.c +++ b/contrib/netbsd-tests/lib/libc/ttyio/t_ttyio.c @@ -1,4 +1,4 @@ -/* $NetBSD: t_ttyio.c,v 1.2 2011/04/19 20:07:53 martin Exp $ */ +/* $NetBSD: t_ttyio.c,v 1.3 2017/01/10 01:31:40 christos Exp $ */ /* * Copyright (c) 2001, 2008 The NetBSD Foundation, Inc. @@ -32,7 +32,7 @@ #include __COPYRIGHT("@(#) Copyright (c) 2008\ The NetBSD Foundation, inc. All rights reserved."); -__RCSID("$NetBSD: t_ttyio.c,v 1.2 2011/04/19 20:07:53 martin Exp $"); +__RCSID("$NetBSD: t_ttyio.c,v 1.3 2017/01/10 01:31:40 christos Exp $"); #include #include @@ -150,11 +150,9 @@ ATF_TC_BODY(ioctl, tc) /* wait for last child */ sa.sa_handler = SIG_DFL; REQUIRE_ERRNO(sigaction(SIGCHLD, &sa, NULL), -1); - (void) wait(NULL); + (void)wait(NULL); -#ifdef __FreeBSD__ (void)close(s); -#endif ATF_REQUIRE_EQ(rc, 0); } diff --git a/crypto/openssh/serverloop.c b/crypto/openssh/serverloop.c index 80d1db5490bc..f5c362dfcc59 100644 --- a/crypto/openssh/serverloop.c +++ b/crypto/openssh/serverloop.c @@ -995,7 +995,7 @@ server_request_direct_streamlocal(void) /* XXX fine grained permissions */ if ((options.allow_streamlocal_forwarding & FORWARD_LOCAL) != 0 && - !no_port_forwarding_flag) { + !no_port_forwarding_flag && use_privsep) { c = channel_connect_to_path(target, "direct-streamlocal@openssh.com", "direct-streamlocal"); } else { @@ -1279,7 +1279,7 @@ server_input_global_request(int type, u_int32_t seq, void *ctxt) /* check permissions */ if ((options.allow_streamlocal_forwarding & FORWARD_REMOTE) == 0 - || no_port_forwarding_flag) { + || no_port_forwarding_flag || !use_privsep) { success = 0; packet_send_debug("Server has disabled port forwarding."); } else { diff --git a/crypto/openssh/ssh-agent.1 b/crypto/openssh/ssh-agent.1 index b8cd0c554b35..3806b6231976 100644 --- a/crypto/openssh/ssh-agent.1 +++ b/crypto/openssh/ssh-agent.1 @@ -1,4 +1,4 @@ -.\" $OpenBSD: ssh-agent.1,v 1.62 2015/11/15 23:54:15 jmc Exp $ +.\" $OpenBSD: ssh-agent.1,v 1.63 2016/11/30 03:07:37 djm Exp $ .\" $FreeBSD$ .\" .\" Author: Tatu Ylonen @@ -48,6 +48,7 @@ .Op Fl a Ar bind_address .Op Fl E Ar fingerprint_hash .Op Fl t Ar life +.Op Fl P Ar pkcs11_whitelist .Op Ar command Op Ar arg ... .Nm ssh-agent .Op Fl c | s @@ -122,6 +123,18 @@ The default is Kill the current agent (given by the .Ev SSH_AGENT_PID environment variable). +.It Fl P +Specify a pattern-list of acceptable paths for PKCS#11 shared libraries +that may be added using the +.Fl s +option to +.Xr ssh-add 1 . +The default is to allow loading PKCS#11 libraries from +.Dq /usr/lib/*,/usr/local/lib/* . +PKCS#11 libraries that do not match the whitelist will be refused. +See PATTERNS in +.Xr ssh_config 5 +for a description of pattern-list syntax. .It Fl s Generate Bourne shell commands on .Dv stdout . diff --git a/crypto/openssh/ssh-agent.c b/crypto/openssh/ssh-agent.c index 8113022ec723..9cad00a281f9 100644 --- a/crypto/openssh/ssh-agent.c +++ b/crypto/openssh/ssh-agent.c @@ -84,11 +84,16 @@ __RCSID("$FreeBSD$"); #include "misc.h" #include "digest.h" #include "ssherr.h" +#include "match.h" #ifdef ENABLE_PKCS11 #include "ssh-pkcs11.h" #endif +#ifndef DEFAULT_PKCS11_WHITELIST +# define DEFAULT_PKCS11_WHITELIST "/usr/lib/*,/usr/local/lib/*" +#endif + #if defined(HAVE_SYS_PRCTL_H) #include /* For prctl() and PR_SET_DUMPABLE */ #endif @@ -140,6 +145,9 @@ pid_t cleanup_pid = 0; char socket_name[PATH_MAX]; char socket_dir[PATH_MAX]; +/* PKCS#11 path whitelist */ +static char *pkcs11_whitelist; + /* locking */ #define LOCK_SIZE 32 #define LOCK_SALT_SIZE 16 @@ -761,7 +769,7 @@ no_identities(SocketEntry *e, u_int type) static void process_add_smartcard_key(SocketEntry *e) { - char *provider = NULL, *pin; + char *provider = NULL, *pin, canonical_provider[PATH_MAX]; int r, i, version, count = 0, success = 0, confirm = 0; u_int seconds; time_t death = 0; @@ -793,10 +801,21 @@ process_add_smartcard_key(SocketEntry *e) goto send; } } + if (realpath(provider, canonical_provider) == NULL) { + verbose("failed PKCS#11 add of \"%.100s\": realpath: %s", + provider, strerror(errno)); + goto send; + } + if (match_pattern_list(canonical_provider, pkcs11_whitelist, 0) != 1) { + verbose("refusing PKCS#11 add of \"%.100s\": " + "provider not whitelisted", canonical_provider); + goto send; + } + debug("%s: add %.100s", __func__, canonical_provider); if (lifetime && !death) death = monotime() + lifetime; - count = pkcs11_add_provider(provider, pin, &keys); + count = pkcs11_add_provider(canonical_provider, pin, &keys); for (i = 0; i < count; i++) { k = keys[i]; version = k->type == KEY_RSA1 ? 1 : 2; @@ -804,8 +823,8 @@ process_add_smartcard_key(SocketEntry *e) if (lookup_identity(k, version) == NULL) { id = xcalloc(1, sizeof(Identity)); id->key = k; - id->provider = xstrdup(provider); - id->comment = xstrdup(provider); /* XXX */ + id->provider = xstrdup(canonical_provider); + id->comment = xstrdup(canonical_provider); /* XXX */ id->death = death; id->confirm = confirm; TAILQ_INSERT_TAIL(&tab->idlist, id, next); @@ -1200,7 +1219,7 @@ usage(void) { fprintf(stderr, "usage: ssh-agent [-c | -s] [-Dd] [-a bind_address] [-E fingerprint_hash]\n" - " [-t life] [command [arg ...]]\n" + " [-P pkcs11_whitelist] [-t life] [command [arg ...]]\n" " ssh-agent [-c | -s] -k\n"); fprintf(stderr, " -x Exit when the last client disconnects.\n"); exit(1); @@ -1246,7 +1265,7 @@ main(int ac, char **av) __progname = ssh_get_progname(av[0]); seed_rng(); - while ((ch = getopt(ac, av, "cDdksE:a:t:x")) != -1) { + while ((ch = getopt(ac, av, "cDdksE:a:P:t:x")) != -1) { switch (ch) { case 'E': fingerprint_hash = ssh_digest_alg_by_name(optarg); @@ -1261,6 +1280,11 @@ main(int ac, char **av) case 'k': k_flag++; break; + case 'P': + if (pkcs11_whitelist != NULL) + fatal("-P option already specified"); + pkcs11_whitelist = xstrdup(optarg); + break; case 's': if (c_flag) usage(); @@ -1298,6 +1322,9 @@ main(int ac, char **av) if (ac > 0 && (c_flag || k_flag || s_flag || d_flag || D_flag)) usage(); + if (pkcs11_whitelist == NULL) + pkcs11_whitelist = xstrdup(DEFAULT_PKCS11_WHITELIST); + if (ac == 0 && !c_flag && !s_flag) { shell = getenv("SHELL"); if (shell != NULL && (len = strlen(shell)) > 2 && @@ -1445,7 +1472,7 @@ main(int ac, char **av) signal(SIGTERM, cleanup_handler); nalloc = 0; - if (pledge("stdio cpath unix id proc exec", NULL) == -1) + if (pledge("stdio rpath cpath unix id proc exec", NULL) == -1) fatal("%s: pledge: %s", __progname, strerror(errno)); platform_pledge_agent(); diff --git a/crypto/openssh/ssh_config b/crypto/openssh/ssh_config index 8eca3453e298..54dffac92e77 100644 --- a/crypto/openssh/ssh_config +++ b/crypto/openssh/ssh_config @@ -50,4 +50,4 @@ # ProxyCommand ssh -q -W %h:%p gateway.example.com # RekeyLimit 1G 1h # VerifyHostKeyDNS yes -# VersionAddendum FreeBSD-20160310 +# VersionAddendum FreeBSD-20161230 diff --git a/crypto/openssh/ssh_config.5 b/crypto/openssh/ssh_config.5 index 0a21247f0d77..f6b22581e6d6 100644 --- a/crypto/openssh/ssh_config.5 +++ b/crypto/openssh/ssh_config.5 @@ -1727,7 +1727,7 @@ See also VERIFYING HOST KEYS in Specifies a string to append to the regular version string to identify OS- or site-specific modifications. The default is -.Dq FreeBSD-20160310 . +.Dq FreeBSD-20161230 . The value .Dq none may be used to disable this. diff --git a/crypto/openssh/sshd_config b/crypto/openssh/sshd_config index a842e6efef9a..075857c354dc 100644 --- a/crypto/openssh/sshd_config +++ b/crypto/openssh/sshd_config @@ -121,7 +121,7 @@ #PermitTunnel no #ChrootDirectory none #UseBlacklist no -#VersionAddendum FreeBSD-20160310 +#VersionAddendum FreeBSD-20161230 # no default banner path #Banner none diff --git a/crypto/openssh/sshd_config.5 b/crypto/openssh/sshd_config.5 index 54c95ad74220..1359a20bced9 100644 --- a/crypto/openssh/sshd_config.5 +++ b/crypto/openssh/sshd_config.5 @@ -1634,7 +1634,7 @@ The default is Optionally specifies additional text to append to the SSH protocol banner sent by the server upon connection. The default is -.Dq FreeBSD-20160310 . +.Dq FreeBSD-20161230 . The value .Dq none may be used to disable this. diff --git a/crypto/openssh/version.h b/crypto/openssh/version.h index 031bb87ea6b8..540aad8f5df1 100644 --- a/crypto/openssh/version.h +++ b/crypto/openssh/version.h @@ -6,7 +6,7 @@ #define SSH_PORTABLE "p2" #define SSH_RELEASE SSH_VERSION SSH_PORTABLE -#define SSH_VERSION_FREEBSD "FreeBSD-20160310" +#define SSH_VERSION_FREEBSD "FreeBSD-20161230" #ifdef WITH_OPENSSL #define OPENSSL_VERSION SSLeay_version(SSLEAY_VERSION) diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist index 620f95c04665..60e2dbea9ae6 100644 --- a/etc/mtree/BSD.tests.dist +++ b/etc/mtree/BSD.tests.dist @@ -644,6 +644,8 @@ .. soelim .. + tail + .. tar .. timeout diff --git a/etc/rc.d/ntpdate b/etc/rc.d/ntpdate index 8f5b273d0fa2..90f9589ab00f 100755 --- a/etc/rc.d/ntpdate +++ b/etc/rc.d/ntpdate @@ -20,7 +20,7 @@ ntpdate_start() if [ -z "$ntpdate_hosts" -a -f "$ntpdate_config" ]; then ntpdate_hosts=`awk ' /^server[ \t]*127.127/ {next} - /^(server|peer)/ { + /^(server|peer|pool)/ { if ($2 ~/^-/) {print $3} else {print $2}} ' < "$ntpdate_config"` diff --git a/lib/libarchive/tests/Makefile b/lib/libarchive/tests/Makefile index dbf34dd93dbe..1249458b1a20 100644 --- a/lib/libarchive/tests/Makefile +++ b/lib/libarchive/tests/Makefile @@ -196,6 +196,7 @@ TESTS_SRCS= \ test_read_format_zip_winzip_aes_large.c \ test_read_format_zip_zip64.c \ test_read_large.c \ + test_read_pax_schily_xattr.c \ test_read_pax_truncated.c \ test_read_position.c \ test_read_set_format.c \ @@ -549,6 +550,7 @@ ${PACKAGE}FILES+= test_read_large_splitted_rar_ab.uu ${PACKAGE}FILES+= test_read_large_splitted_rar_ac.uu ${PACKAGE}FILES+= test_read_large_splitted_rar_ad.uu ${PACKAGE}FILES+= test_read_large_splitted_rar_ae.uu +${PACKAGE}FILES+= test_read_pax_schily_xattr.tar.uu ${PACKAGE}FILES+= test_read_splitted_rar_aa.uu ${PACKAGE}FILES+= test_read_splitted_rar_ab.uu ${PACKAGE}FILES+= test_read_splitted_rar_ac.uu diff --git a/libexec/rtld-elf/rtld_lock.c b/libexec/rtld-elf/rtld_lock.c index f31546c4ac91..c4d2a0b0a7b1 100644 --- a/libexec/rtld-elf/rtld_lock.c +++ b/libexec/rtld-elf/rtld_lock.c @@ -38,8 +38,8 @@ * In this algorithm the lock is a single word. Its low-order bit is * set when a writer holds the lock. The remaining high-order bits * contain a count of readers desiring the lock. The algorithm requires - * atomic "compare_and_store" and "add" operations, which we implement - * using assembly language sequences in "rtld_start.S". + * atomic "compare_and_store" and "add" operations, which we take + * from machine/atomic.h. */ #include @@ -64,10 +64,10 @@ typedef struct Struct_Lock { } Lock; static sigset_t fullsigmask, oldsigmask; -static int thread_flag; +static int thread_flag, wnested; static void * -def_lock_create() +def_lock_create(void) { void *base; char *p; @@ -117,29 +117,34 @@ def_rlock_acquire(void *lock) static void def_wlock_acquire(void *lock) { - Lock *l = (Lock *)lock; - sigset_t tmp_oldsigmask; + Lock *l; + sigset_t tmp_oldsigmask; - for ( ; ; ) { - sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask); - if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) - break; - sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL); - } - oldsigmask = tmp_oldsigmask; + l = (Lock *)lock; + for (;;) { + sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask); + if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) + break; + sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL); + } + if (atomic_fetchadd_int(&wnested, 1) == 0) + oldsigmask = tmp_oldsigmask; } static void def_lock_release(void *lock) { - Lock *l = (Lock *)lock; + Lock *l; - if ((l->lock & WAFLAG) == 0) - atomic_add_rel_int(&l->lock, -RC_INCR); - else { - atomic_add_rel_int(&l->lock, -WAFLAG); - sigprocmask(SIG_SETMASK, &oldsigmask, NULL); - } + l = (Lock *)lock; + if ((l->lock & WAFLAG) == 0) + atomic_add_rel_int(&l->lock, -RC_INCR); + else { + assert(wnested > 0); + atomic_add_rel_int(&l->lock, -WAFLAG); + if (atomic_fetchadd_int(&wnested, -1) == 1) + sigprocmask(SIG_SETMASK, &oldsigmask, NULL); + } } static int @@ -269,7 +274,7 @@ lock_restart_for_upgrade(RtldLockState *lockstate) } void -lockdflt_init() +lockdflt_init(void) { int i; @@ -373,12 +378,12 @@ _rtld_atfork_pre(int *locks) return; /* - * Warning: this does not work with the rtld compat locks - * above, since the thread signal mask is corrupted (set to - * all signals blocked) if two locks are taken in write mode. - * The caller of the _rtld_atfork_pre() must provide the - * working implementation of the locks, and libthr locks are - * fine. + * Warning: this did not worked well with the rtld compat + * locks above, when the thread signal mask was corrupted (set + * to all signals blocked) if two locks were taken + * simultaneously in the write mode. The caller of the + * _rtld_atfork_pre() must provide the working implementation + * of the locks anyway, and libthr locks are fine. */ wlock_acquire(rtld_phdr_lock, &ls[0]); wlock_acquire(rtld_bind_lock, &ls[1]); diff --git a/sbin/camcontrol/modeedit.c b/sbin/camcontrol/modeedit.c index cbc138caf391..90862db193be 100644 --- a/sbin/camcontrol/modeedit.c +++ b/sbin/camcontrol/modeedit.c @@ -557,7 +557,7 @@ editlist_populate(struct cam_device *device, int dbd, int pc, int page, struct scsi_mode_header_6 *mh; /* Location of mode header. */ struct scsi_mode_page_header *mph; struct scsi_mode_page_header_sp *mphsp; - int len; + size_t len; STAILQ_INIT(&editlist); @@ -575,6 +575,7 @@ editlist_populate(struct cam_device *device, int dbd, int pc, int page, mode_pars = (uint8_t *)(mphsp + 1); len = scsi_2btoul(mphsp->page_length); } + len = MIN(len, sizeof(data) - (mode_pars - data)); /* Decode the value data, creating edit_entries for each value. */ buff_decode_visit(mode_pars, len, format, editentry_create, 0); @@ -594,7 +595,7 @@ editlist_save(struct cam_device *device, int dbd, int pc, int page, struct scsi_mode_header_6 *mh; /* Location of mode header. */ struct scsi_mode_page_header *mph; struct scsi_mode_page_header_sp *mphsp; - int len, hlen; + size_t len, hlen; /* Make sure that something changed before continuing. */ if (! editlist_changed) @@ -617,6 +618,7 @@ editlist_save(struct cam_device *device, int dbd, int pc, int page, mode_pars = (uint8_t *)(mphsp + 1); len = scsi_2btoul(mphsp->page_length); } + len = MIN(len, sizeof(data) - (mode_pars - data)); /* Encode the value data to be passed back to the device. */ buff_encode_visit(mode_pars, len, format, editentry_save, 0); @@ -814,7 +816,7 @@ modepage_dump(struct cam_device *device, int dbd, int pc, int page, int subpage, struct scsi_mode_header_6 *mh; /* Location of mode header. */ struct scsi_mode_page_header *mph; struct scsi_mode_page_header_sp *mphsp; - int indx, len; + size_t indx, len; mode_sense(device, dbd, pc, page, subpage, retries, timeout, data, sizeof(data)); @@ -829,6 +831,7 @@ modepage_dump(struct cam_device *device, int dbd, int pc, int page, int subpage, mode_pars = (uint8_t *)(mphsp + 1); len = scsi_2btoul(mphsp->page_length); } + len = MIN(len, sizeof(data) - (mode_pars - data)); /* Print the raw mode page data with newlines each 8 bytes. */ for (indx = 0; indx < len; indx++) { diff --git a/share/misc/committers-ports.dot b/share/misc/committers-ports.dot index 3da0e274a68a..79c5f2e18e04 100644 --- a/share/misc/committers-ports.dot +++ b/share/misc/committers-ports.dot @@ -29,7 +29,6 @@ node [color=grey62, style=filled, bgcolor=black]; # Alumni go here.. Try to keep things sorted. -adamw [label="Adam Weinberger\nadamw@FreeBSD.org\n2002/10/16\n2006/09/25"] asami [label="Satoshi Asami\nasami@FreeBSD.org\n1994/11/18\n2001/09/11"] billf [label="Bill Fumerola\nbillf@FreeBSD.org\n1998/11/11\n2006/12/14"] jmallett [label="Juli Mallett\njmallett@FreeBSD.org\n2003/01/16\n2006/08/10"] @@ -43,6 +42,7 @@ node [color=lightblue2, style=filled, bgcolor=black]; ache [label="Andrey Chernov\nache@FreeBSD.org\n1994/11/15"] acm [label="Jose Alonso Cardenas Marquez\nacm@FreeBSD.org\n2006/07/18"] +adamw [label="Adam Weinberger\nadamw@FreeBSD.org\n2002/10/16"] ahze [label="Michael Johnson\nahze@FreeBSD.org\n2004/10/29"] ak [label="Alex Kozlov\nak@FreeBSD.org\n2012/02/29"] ale [label="Alex Dupre\nale@FreeBSD.org\n2004/01/12"] @@ -90,6 +90,7 @@ delphij [label="Xin Li\ndelphij@FreeBSD.org\n2006/05/01"] demon [label="Dmitry Sivachenko\ndemon@FreeBSD.org\n2000/11/13"] dhn [label="Dennis Herrmann\ndhn@FreeBSD.org\n2009/03/03"] dryice [label="Dryice Dong Liu\ndryice@FreeBSD.org\n2006/12/25"] +dumbbell [label="Jean-Sebastien Pedron\ndumbbell@FreeBSD.org\n2017/01/10"] dvl [label="Dan Langille\ndvl@FreeBSD.org\n2014/08/10"] eadler [label="Eitan Adler\neadler@FreeBSD.org\n2011/08/17"] edwin [label="Edwin Groothuis\nedwin@FreeBSD.org\n2002/10/22"] @@ -144,6 +145,7 @@ laszlof [label="Frank Laszlo\nlaszlof@FreeBSD.org\n2006/11/07"] lawrance [label="Sam Lawrance\nlawrance@FreeBSD.org\n2005/04/11\n2007/02/21"] lbr [label="Lars Balker Rasmussen\nlbr@FreeBSD.org\n2006/04/30"] leeym [label="Yen-Ming Lee\nleeym@FreeBSD.org\n2002/08/14"] +ler [label="Larry Rosenman\nler@FreeBSD.org\n2017/01/09"] lev [label="Lev Serebryakov\nlev@FreeBSD.org\n2003/06/17"] lifanov [label="Nikolai Lifanov\nlifanov@FreeBSD.org\n2016/12/11"] linimon [label="Mark Linimon\nlinimon@FreeBSD.org\n2003/10/23"] @@ -252,6 +254,7 @@ znerd [label="Ernst de Haan\nznerd@FreeBSD.org\n2001/11/15"] adamw -> ahze adamw -> jylefort +adamw -> ler adamw -> mezz adamw -> pav adamw -> woodsb02 @@ -263,6 +266,8 @@ ahze -> tmclaugh amdmi3 -> jrm +antoine -> dumbbell + araujo -> lippe araujo -> pclin araujo -> pgollucci @@ -281,6 +286,7 @@ bdrewery -> trociny bapt -> bdrewery bapt -> bofh +bapt -> dumbbell bapt -> eadler bapt -> grembo bapt -> jbeich @@ -554,6 +560,7 @@ rene -> bar rene -> cmt rene -> crees rene -> jgh +rene -> ler rene -> olivierd rm -> koobs diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 85202302c9d7..817cdbe97c9e 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -6010,7 +6010,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) pdp_entry_t *pdpe; pd_entry_t oldpde, *pde; pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; - vm_offset_t va_next; + vm_offset_t va, va_next; vm_page_t m; boolean_t anychanged; @@ -6090,11 +6090,11 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } if (va_next > eva) va_next = eva; + va = va_next; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { - if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | - PG_V)) - continue; + if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) + goto maybe_invlrng; else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (advice == MADV_DONTNEED) { /* @@ -6109,12 +6109,22 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } else if ((*pte & PG_A) != 0) atomic_clear_long(pte, PG_A); else - continue; - if ((*pte & PG_G) != 0) - pmap_invalidate_page(pmap, sva); - else + goto maybe_invlrng; + + if ((*pte & PG_G) != 0) { + if (va == va_next) + va = sva; + } else anychanged = TRUE; + continue; +maybe_invlrng: + if (va != va_next) { + pmap_invalidate_range(pmap, va, sva); + va = va_next; + } } + if (va != va_next) + pmap_invalidate_range(pmap, va, sva); } if (anychanged) pmap_invalidate_all(pmap); diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC index db8d07d10b3c..ba5bfe688fc0 100644 --- a/sys/amd64/conf/GENERIC +++ b/sys/amd64/conf/GENERIC @@ -230,7 +230,6 @@ device puc # Multi I/O cards and multi-channel UARTs device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family -device igb # Intel PRO/1000 PCIE Server Gigabit Family device ix # Intel PRO/10GbE PCIE PF Ethernet device ixv # Intel PRO/10GbE PCIE VF Ethernet device ixl # Intel XL710 40Gbe PCIE Ethernet diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC index ee4bc49f3e64..790c02a2d369 100644 --- a/sys/arm64/conf/GENERIC +++ b/sys/arm64/conf/GENERIC @@ -120,7 +120,6 @@ device mii device miibus # MII bus support device awg # Allwinner EMAC Gigabit Ethernet device em # Intel PRO/1000 Gigabit Ethernet Family -device igb # Intel PRO/1000 PCIE Server Gigabit Family device ix # Intel 10Gb Ethernet Family device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet device smc # SMSC LAN91C111 diff --git a/sys/boot/common/reloc_elf.c b/sys/boot/common/reloc_elf.c index 2b60d18b5aff..6d4a00ff8f68 100644 --- a/sys/boot/common/reloc_elf.c +++ b/sys/boot/common/reloc_elf.c @@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include #include #define FREEBSD_ELF diff --git a/sys/boot/efi/Makefile.inc b/sys/boot/efi/Makefile.inc index 671033c58e80..eac5992a4c40 100644 --- a/sys/boot/efi/Makefile.inc +++ b/sys/boot/efi/Makefile.inc @@ -23,4 +23,8 @@ CFLAGS+= -fshort-wchar CFLAGS+= -fPIC .endif +.if ${MACHINE_CPUARCH} == "arm" +CFLAGS+= -fPIC +.endif + .include "../Makefile.inc" diff --git a/sys/boot/efi/boot1/boot_module.h b/sys/boot/efi/boot1/boot_module.h index 296d5a67a10b..3a6b8270a5cc 100644 --- a/sys/boot/efi/boot1/boot_module.h +++ b/sys/boot/efi/boot1/boot_module.h @@ -64,7 +64,7 @@ typedef struct boot_module_t const char *name; /* init is the optional initialiser for the module. */ - void (*init)(); + void (*init)(void); /* * probe checks to see if the module can handle dev. @@ -89,10 +89,10 @@ typedef struct boot_module_t void **buf, size_t *bufsize); /* status outputs information about the probed devices. */ - void (*status)(); + void (*status)(void); /* valid devices as found by probe. */ - dev_info_t *(*devices)(); + dev_info_t *(*devices)(void); } boot_module_t; /* Standard boot modules. */ diff --git a/sys/boot/efi/include/efiapi.h b/sys/boot/efi/include/efiapi.h index b1a7b45eeb53..92eb5131f2d7 100644 --- a/sys/boot/efi/include/efiapi.h +++ b/sys/boot/efi/include/efiapi.h @@ -532,6 +532,7 @@ EFI_STATUS typedef EFI_STATUS (EFIAPI *EFI_RESERVED_SERVICE) ( + VOID ); typedef diff --git a/sys/boot/efi/loader/arch/arm/ldscript.arm b/sys/boot/efi/loader/arch/arm/ldscript.arm index 7ff4cc6a4cb2..9f6e7c29117e 100644 --- a/sys/boot/efi/loader/arch/arm/ldscript.arm +++ b/sys/boot/efi/loader/arch/arm/ldscript.arm @@ -18,7 +18,7 @@ SECTIONS . = ALIGN(16); .data : { - *(.data *.data.*) + *(.data .data.*) *(.gnu.linkonce.d*) *(.rodata) *(.rodata.*) diff --git a/sys/boot/efi/loader/arch/arm/start.S b/sys/boot/efi/loader/arch/arm/start.S index 443de4af1f87..9bc0c9f0f08b 100644 --- a/sys/boot/efi/loader/arch/arm/start.S +++ b/sys/boot/efi/loader/arch/arm/start.S @@ -161,7 +161,7 @@ _start: mov r2, #0 1: cmp r0, r1 - bgt 2f + bge 2f str r2, [r0], #4 b 1b 2: diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c index 0c31ef9e8fff..2efad7c7b7ad 100644 --- a/sys/cam/ctl/ctl.c +++ b/sys/cam/ctl/ctl.c @@ -4593,6 +4593,8 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun, printf("ctl: requested LUN ID %d is already " "in use\n", be_lun->req_lun_id); } +fail: + free(lun->lun_devid, M_CTL); if (lun->flags & CTL_LUN_MALLOCED) free(lun, M_CTL); be_lun->lun_config_status(be_lun->be_lun, @@ -4605,14 +4607,11 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun, if (lun_number == -1) { mtx_unlock(&ctl_softc->ctl_lock); printf("ctl: can't allocate LUN, out of LUNs\n"); - if (lun->flags & CTL_LUN_MALLOCED) - free(lun, M_CTL); - be_lun->lun_config_status(be_lun->be_lun, - CTL_LUN_CONFIG_FAILURE); - return (ENOSPC); + goto fail; } } ctl_set_mask(ctl_softc->ctl_lun_mask, lun_number); + mtx_unlock(&ctl_softc->ctl_lock); mtx_init(&lun->lun_lock, "CTL LUN", NULL, MTX_DEF); lun->lun = lun_number; @@ -4664,22 +4663,6 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun, ctl_init_page_index(lun); ctl_init_log_page_index(lun); - /* - * Now, before we insert this lun on the lun list, set the lun - * inventory changed UA for all other luns. - */ - STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) { - mtx_lock(&nlun->lun_lock); - ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE); - mtx_unlock(&nlun->lun_lock); - } - - STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links); - - ctl_softc->ctl_luns[lun_number] = lun; - - ctl_softc->num_luns++; - /* Setup statistics gathering */ #ifdef CTL_LEGACY_STATS lun->legacy_stats.device_type = be_lun->lun_type; @@ -4692,6 +4675,19 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun, #endif /* CTL_LEGACY_STATS */ lun->stats.item = lun_number; + /* + * Now, before we insert this lun on the lun list, set the lun + * inventory changed UA for all other luns. + */ + mtx_lock(&ctl_softc->ctl_lock); + STAILQ_FOREACH(nlun, &ctl_softc->lun_list, links) { + mtx_lock(&nlun->lun_lock); + ctl_est_ua_all(nlun, -1, CTL_UA_LUN_CHANGE); + mtx_unlock(&nlun->lun_lock); + } + STAILQ_INSERT_TAIL(&ctl_softc->lun_list, lun, links); + ctl_softc->ctl_luns[lun_number] = lun; + ctl_softc->num_luns++; mtx_unlock(&ctl_softc->ctl_lock); lun->be_lun->lun_config_status(lun->be_lun->be_lun, CTL_LUN_CONFIG_OK); diff --git a/sys/cam/ctl/ctl_ha.c b/sys/cam/ctl/ctl_ha.c index d7f21c7a74b6..3402f040ee02 100644 --- a/sys/cam/ctl/ctl_ha.c +++ b/sys/cam/ctl/ctl_ha.c @@ -1001,7 +1001,7 @@ ctl_ha_msg_shutdown(struct ctl_softc *ctl_softc) softc->ha_shutdown = 1; softc->ha_wakeup = 1; wakeup(&softc->ha_wakeup); - while (softc->ha_shutdown < 2) { + while (softc->ha_shutdown < 2 && !SCHEDULER_STOPPED()) { msleep(&softc->ha_wakeup, &softc->ha_lock, 0, "shutdown", hz); } diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c index c47fb9b65398..ed0db960d16a 100644 --- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c +++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c @@ -4335,7 +4335,9 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_SUBR_MUTEX_OWNER: @@ -4345,7 +4347,9 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); regs[rd] = (uintptr_t)lowner; break; @@ -4356,9 +4360,9 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); - /* XXX - should be only LC_SLEEPABLE? */ - regs[rd] = (LOCK_CLASS(l.li)->lc_flags & - (LC_SLEEPLOCK | LC_SLEEPABLE)) != 0; + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); + regs[rd] = (LOCK_CLASS(l.li)->lc_flags & LC_SLEEPLOCK) != 0; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_SUBR_MUTEX_TYPE_SPIN: @@ -4368,7 +4372,9 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = (LOCK_CLASS(l.li)->lc_flags & LC_SPINLOCK) != 0; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_SUBR_RW_READ_HELD: @@ -4379,8 +4385,10 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && lowner == NULL; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_SUBR_RW_WRITE_HELD: @@ -4391,8 +4399,10 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr(tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && lowner != NULL; + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); break; case DIF_SUBR_RW_ISWRITER: @@ -4403,7 +4413,9 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, break; } l.lx = dtrace_loadptr(tupregs[0].dttk_value); + DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); + DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); regs[rd] = (lowner == curthread); break; #endif /* illumos */ diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 0ade0ad97011..711c9a88de4a 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -1972,7 +1972,6 @@ device xmphy # XaQti XMAC II # KNE110TX. # de: Digital Equipment DC21040 # em: Intel Pro/1000 Gigabit Ethernet 82542, 82543, 82544 based adapters. -# igb: Intel Pro/1000 PCI Express Gigabit Ethernet: 82575 and later adapters. # ep: 3Com 3C509, 3C529, 3C556, 3C562D, 3C563D, 3C572, 3C574X, 3C579, 3C589 # and PC Card devices using these chipsets. # ex: Intel EtherExpress Pro/10 and other i82595-based adapters, @@ -2145,7 +2144,6 @@ device cxgbe # Chelsio T4-T6 1/10/25/40/100 Gigabit Ethernet device cxgbev # Chelsio T4-T6 Virtual Functions device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel Pro/1000 Gigabit Ethernet -device igb # Intel Pro/1000 PCIE Gigabit Ethernet device ixgb # Intel Pro/10Gbe PCI-X Ethernet device ix # Intel Pro/10Gbe PCIE Ethernet device ixv # Intel Pro/10Gbe PCIE Ethernet VF diff --git a/sys/conf/files b/sys/conf/files index 3abe344cfa78..c221bbec85bf 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1572,43 +1572,43 @@ dev/eisa/eisa_if.m standard dev/eisa/eisaconf.c optional eisa dev/e1000/if_em.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/if_lem.c optional em \ +dev/e1000/em_txrx.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/if_igb.c optional igb \ +dev/e1000/igb_txrx.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_80003es2lan.c optional em | igb \ +dev/e1000/e1000_80003es2lan.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82540.c optional em | igb \ +dev/e1000/e1000_82540.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82541.c optional em | igb \ +dev/e1000/e1000_82541.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82542.c optional em | igb \ +dev/e1000/e1000_82542.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82543.c optional em | igb \ +dev/e1000/e1000_82543.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82571.c optional em | igb \ +dev/e1000/e1000_82571.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_82575.c optional em | igb \ +dev/e1000/e1000_82575.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_ich8lan.c optional em | igb \ +dev/e1000/e1000_ich8lan.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_i210.c optional em | igb \ +dev/e1000/e1000_i210.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_api.c optional em | igb \ +dev/e1000/e1000_api.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_mac.c optional em | igb \ +dev/e1000/e1000_mac.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_manage.c optional em | igb \ +dev/e1000/e1000_manage.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_nvm.c optional em | igb \ +dev/e1000/e1000_nvm.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_phy.c optional em | igb \ +dev/e1000/e1000_phy.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_vf.c optional em | igb \ +dev/e1000/e1000_vf.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_mbx.c optional em | igb \ +dev/e1000/e1000_mbx.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" -dev/e1000/e1000_osdep.c optional em | igb \ +dev/e1000/e1000_osdep.c optional em \ compile-with "${NORMAL_C} -I$S/dev/e1000" dev/et/if_et.c optional et dev/en/if_en_pci.c optional en pci @@ -2826,6 +2826,7 @@ dev/scc/scc_dev_z8530.c optional scc dev/sdhci/sdhci.c optional sdhci dev/sdhci/sdhci_fdt_gpio.c optional sdhci fdt gpio dev/sdhci/sdhci_if.m optional sdhci +dev/sdhci/sdhci_acpi.c optional sdhci acpi dev/sdhci/sdhci_pci.c optional sdhci pci dev/sf/if_sf.c optional sf pci dev/sge/if_sge.c optional sge pci @@ -3905,9 +3906,9 @@ net/if_tun.c optional tun net/if_tap.c optional tap net/if_vlan.c optional vlan net/if_vxlan.c optional vxlan inet | vxlan inet6 -net/ifdi_if.m optional ether pci -net/iflib.c optional ether pci -net/mp_ring.c optional ether +net/ifdi_if.m optional ether pci +net/iflib.c optional ether pci +net/mp_ring.c optional ether net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 4351bdcf31ec..2ff24eb1ef94 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -145,7 +145,7 @@ armv8_crypto_wrap.o optional armv8crypto \ crypto/blowfish/bf_enc.c optional crypto | ipsec crypto/des/des_enc.c optional crypto | ipsec | netsmb dev/acpica/acpi_if.m optional acpi -dev/ahci/ahci_generic.c optional ahci fdt +dev/ahci/ahci_generic.c optional ahci dev/cpufreq/cpufreq_dt.c optional cpufreq fdt dev/hwpmc/hwpmc_arm64.c optional hwpmc dev/hwpmc/hwpmc_arm64_md.c optional hwpmc diff --git a/sys/conf/makeLINT.mk b/sys/conf/makeLINT.mk index e2c7e5e00aff..04dfcee05b9c 100644 --- a/sys/conf/makeLINT.mk +++ b/sys/conf/makeLINT.mk @@ -38,7 +38,6 @@ LINT: ${NOTES} ../../conf/makeLINT.sed echo "nodevice bxe" >> ${.TARGET}-NOIP echo "nodevice em" >> ${.TARGET}-NOIP echo "nodevice fxp" >> ${.TARGET}-NOIP - echo "nodevice igb" >> ${.TARGET}-NOIP echo "nodevice jme" >> ${.TARGET}-NOIP echo "nodevice msk" >> ${.TARGET}-NOIP echo "nodevice mxge" >> ${.TARGET}-NOIP diff --git a/sys/contrib/dev/acpica/components/namespace/nsxfeval.c b/sys/contrib/dev/acpica/components/namespace/nsxfeval.c index 4c50040156e1..6467d7e2f54f 100644 --- a/sys/contrib/dev/acpica/components/namespace/nsxfeval.c +++ b/sys/contrib/dev/acpica/components/namespace/nsxfeval.c @@ -1022,25 +1022,23 @@ ACPI_EXPORT_SYMBOL (AcpiDetachData) /******************************************************************************* * - * FUNCTION: AcpiGetDataFull + * FUNCTION: AcpiGetData * * PARAMETERS: ObjHandle - Namespace node - * Handle - Handler used in call to attach_data + * Handler - Handler used in call to AttachData * Data - Where the data is returned - * Callback - function to execute before returning * * RETURN: Status * - * DESCRIPTION: Retrieve data that was previously attached to a namespace node - * and execute a callback before returning. + * DESCRIPTION: Retrieve data that was previously attached to a namespace node. * ******************************************************************************/ + ACPI_STATUS -AcpiGetDataFull ( +AcpiGetData ( ACPI_HANDLE ObjHandle, ACPI_OBJECT_HANDLER Handler, - void **Data, - void (*Callback)(void *)) + void **Data) { ACPI_NAMESPACE_NODE *Node; ACPI_STATUS Status; @@ -1071,34 +1069,10 @@ AcpiGetDataFull ( } Status = AcpiNsGetAttachedData (Node, Handler, Data); - if (ACPI_SUCCESS(Status) && Callback) { - Callback(*Data); - } + UnlockAndExit: (void) AcpiUtReleaseMutex (ACPI_MTX_NAMESPACE); return (Status); } -ACPI_EXPORT_SYMBOL (AcpiGetDataFull) -/******************************************************************************* - * - * FUNCTION: AcpiGetData - * - * PARAMETERS: ObjHandle - Namespace node - * Handler - Handler used in call to AttachData - * Data - Where the data is returned - * - * RETURN: Status - * - * DESCRIPTION: Retrieve data that was previously attached to a namespace node. - * - ******************************************************************************/ -ACPI_STATUS -AcpiGetData ( - ACPI_HANDLE ObjHandle, - ACPI_OBJECT_HANDLER Handler, - void **Data) -{ - return (AcpiGetDataFull(ObjHandle, Handler, Data, NULL)); -} ACPI_EXPORT_SYMBOL (AcpiGetData) diff --git a/sys/contrib/dev/acpica/components/tables/tbxface.c b/sys/contrib/dev/acpica/components/tables/tbxface.c index 254958bd4924..6e231cd8fceb 100644 --- a/sys/contrib/dev/acpica/components/tables/tbxface.c +++ b/sys/contrib/dev/acpica/components/tables/tbxface.c @@ -314,12 +314,11 @@ ACPI_EXPORT_SYMBOL (AcpiGetTableHeader) /******************************************************************************* * - * FUNCTION: AcpiGetTableWithSize + * FUNCTION: AcpiGetTable * * PARAMETERS: Signature - ACPI signature of needed table * Instance - Which instance (for SSDTs) * OutTable - Where the pointer to the table is returned - * TblSize - Size of the table * * RETURN: Status and pointer to the requested table * @@ -334,11 +333,10 @@ ACPI_EXPORT_SYMBOL (AcpiGetTableHeader) ******************************************************************************/ ACPI_STATUS -AcpiGetTableWithSize ( +AcpiGetTable ( char *Signature, UINT32 Instance, - ACPI_TABLE_HEADER **OutTable, - ACPI_SIZE *TblSize) + ACPI_TABLE_HEADER **OutTable) { UINT32 i; UINT32 j; @@ -436,38 +434,10 @@ AcpiPutTable ( (void) AcpiUtReleaseMutex (ACPI_MTX_TABLES); return_VOID; } + ACPI_EXPORT_SYMBOL (AcpiPutTable) -/******************************************************************************* - * - * FUNCTION: AcpiGetTable - * - * PARAMETERS: Signature - ACPI signature of needed table - * Instance - Which instance (for SSDTs) - * OutTable - Where the pointer to the table is returned - * - * RETURN: Status and pointer to the requested table - * - * DESCRIPTION: Finds and verifies an ACPI table. Table must be in the - * RSDT/XSDT. - * - ******************************************************************************/ - -ACPI_STATUS -AcpiGetTable ( - char *Signature, - UINT32 Instance, - ACPI_TABLE_HEADER **OutTable) -{ - ACPI_SIZE Size; - - return (AcpiGetTableWithSize(Signature, Instance, OutTable, &Size)); -} - -ACPI_EXPORT_SYMBOL (AcpiGetTable) - - /******************************************************************************* * * FUNCTION: AcpiGetTableByIndex diff --git a/sys/contrib/dev/acpica/include/acpixf.h b/sys/contrib/dev/acpica/include/acpixf.h index 7a0edc091ddd..7234ae8e331d 100644 --- a/sys/contrib/dev/acpica/include/acpixf.h +++ b/sys/contrib/dev/acpica/include/acpixf.h @@ -584,14 +584,6 @@ AcpiGetTableHeader ( UINT32 Instance, ACPI_TABLE_HEADER *OutTableHeader)) -ACPI_EXTERNAL_RETURN_STATUS ( -ACPI_STATUS -AcpiGetTableWithSize ( - ACPI_STRING Signature, - UINT32 Instance, - ACPI_TABLE_HEADER **OutTable, - ACPI_SIZE *TblSize)) - ACPI_EXTERNAL_RETURN_STATUS ( ACPI_STATUS AcpiGetTable ( @@ -678,14 +670,6 @@ AcpiGetData ( ACPI_OBJECT_HANDLER Handler, void **Data)) -ACPI_EXTERNAL_RETURN_STATUS ( -ACPI_STATUS -AcpiGetDataFull ( - ACPI_HANDLE Object, - ACPI_OBJECT_HANDLER Handler, - void **Data, - void (*Callback)(void *))) - ACPI_EXTERNAL_RETURN_STATUS ( ACPI_STATUS AcpiDebugTrace ( diff --git a/sys/dev/ahci/ahci_generic.c b/sys/dev/ahci/ahci_generic.c index 80f7fedee049..a427e7aeaeea 100644 --- a/sys/dev/ahci/ahci_generic.c +++ b/sys/dev/ahci/ahci_generic.c @@ -27,6 +27,9 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_acpi.h" +#include "opt_platform.h" + #include #include #include @@ -44,6 +47,15 @@ __FBSDID("$FreeBSD$"); #include +#ifdef DEV_ACPI +#include +#include + +#include +#include +#endif + +#ifdef FDT #include #include @@ -54,14 +66,7 @@ static struct ofw_compat_data compat_data[] = { }; static int -ahci_gen_ctlr_reset(device_t dev) -{ - - return ahci_ctlr_reset(dev); -} - -static int -ahci_probe(device_t dev) +ahci_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) @@ -73,6 +78,34 @@ ahci_probe(device_t dev) device_set_desc_copy(dev, "AHCI SATA controller"); return (BUS_PROBE_DEFAULT); } +#endif + +#ifdef DEV_ACPI +static int +ahci_acpi_probe(device_t dev) +{ + ACPI_HANDLE h; + + if ((h = acpi_get_handle(dev)) == NULL) + return (ENXIO); + + if (pci_get_class(dev) == PCIC_STORAGE && + pci_get_subclass(dev) == PCIS_STORAGE_SATA && + pci_get_progif(dev) == PCIP_STORAGE_SATA_AHCI_1_0) { + device_set_desc_copy(dev, "AHCI SATA controller"); + return (BUS_PROBE_DEFAULT); + } + + return (ENXIO); +} +#endif + +static int +ahci_gen_ctlr_reset(device_t dev) +{ + + return ahci_ctlr_reset(dev); +} static int ahci_gen_attach(device_t dev) @@ -109,9 +142,10 @@ ahci_gen_detach(device_t dev) return (0); } -static devclass_t ahci_gen_devclass; -static device_method_t ahci_methods[] = { - DEVMETHOD(device_probe, ahci_probe), +#ifdef FDT +static devclass_t ahci_gen_fdt_devclass; +static device_method_t ahci_fdt_methods[] = { + DEVMETHOD(device_probe, ahci_fdt_probe), DEVMETHOD(device_attach, ahci_gen_attach), DEVMETHOD(device_detach, ahci_gen_detach), DEVMETHOD(bus_print_child, ahci_print_child), @@ -123,9 +157,35 @@ static device_method_t ahci_methods[] = { DEVMETHOD(bus_get_dma_tag, ahci_get_dma_tag), DEVMETHOD_END }; -static driver_t ahci_driver = { +static driver_t ahci_fdt_driver = { "ahci", - ahci_methods, + ahci_fdt_methods, sizeof(struct ahci_controller) }; -DRIVER_MODULE(ahci, simplebus, ahci_driver, ahci_gen_devclass, NULL, NULL); +DRIVER_MODULE(ahci_fdt, simplebus, ahci_fdt_driver, ahci_gen_fdt_devclass, + NULL, NULL); +#endif + +#ifdef DEV_ACPI +static devclass_t ahci_gen_acpi_devclass; +static device_method_t ahci_acpi_methods[] = { + DEVMETHOD(device_probe, ahci_acpi_probe), + DEVMETHOD(device_attach, ahci_gen_attach), + DEVMETHOD(device_detach, ahci_gen_detach), + DEVMETHOD(bus_print_child, ahci_print_child), + DEVMETHOD(bus_alloc_resource, ahci_alloc_resource), + DEVMETHOD(bus_release_resource, ahci_release_resource), + DEVMETHOD(bus_setup_intr, ahci_setup_intr), + DEVMETHOD(bus_teardown_intr,ahci_teardown_intr), + DEVMETHOD(bus_child_location_str, ahci_child_location_str), + DEVMETHOD(bus_get_dma_tag, ahci_get_dma_tag), + DEVMETHOD_END +}; +static driver_t ahci_acpi_driver = { + "ahci", + ahci_acpi_methods, + sizeof(struct ahci_controller) +}; +DRIVER_MODULE(ahci_acpi, acpi, ahci_acpi_driver, ahci_gen_acpi_devclass, + NULL, NULL); +#endif diff --git a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c index 448b99348fb2..c2b508039a47 100644 --- a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c +++ b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c @@ -1094,7 +1094,7 @@ int iwch_register_device(struct iwch_dev *dev) memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC)); dev->ibdev.phys_port_cnt = sc->params.nports; dev->ibdev.num_comp_vectors = 1; - dev->ibdev.dma_device = dev->rdev.adap->dev; + dev->ibdev.dma_device = NULL; dev->ibdev.query_device = iwch_query_device; dev->ibdev.query_port = iwch_query_port; dev->ibdev.modify_port = iwch_modify_port; diff --git a/sys/dev/cxgbe/common/t4_msg.h b/sys/dev/cxgbe/common/t4_msg.h index 830743b91a8f..f15c3a89a56a 100644 --- a/sys/dev/cxgbe/common/t4_msg.h +++ b/sys/dev/cxgbe/common/t4_msg.h @@ -106,6 +106,7 @@ enum { CPL_RX_FCOE_DIF = 0x4A, CPL_RX_DATA_DIF = 0x4B, CPL_ERR_NOTIFY = 0x4D, + CPL_RX_TLS_CMP = 0x4E, CPL_RDMA_READ_REQ = 0x60, CPL_RX_ISCSI_DIF = 0x60, @@ -113,6 +114,11 @@ enum { CPL_SET_LE_REQ = 0x80, CPL_PASS_OPEN_REQ6 = 0x81, CPL_ACT_OPEN_REQ6 = 0x83, + CPL_TX_TLS_PDU = 0x88, + CPL_TX_TLS_SFO = 0x89, + + CPL_TX_SEC_PDU = 0x8A, + CPL_TX_TLS_ACK = 0x8B, CPL_RDMA_TERMINATE = 0xA2, CPL_RDMA_WRITE = 0xA4, @@ -129,6 +135,7 @@ enum { CPL_TRACE_PKT = 0xB0, CPL_RX2TX_DATA = 0xB1, + CPL_TLS_DATA = 0xB1, CPL_ISCSI_DATA = 0xB2, CPL_FCOE_DATA = 0xB3, @@ -136,6 +143,7 @@ enum { CPL_FW4_PLD = 0xC1, CPL_FW4_ACK = 0xC3, CPL_SRQ_TABLE_RPL = 0xCC, + CPL_RX_PHYS_DSGL = 0xD0, CPL_FW6_MSG = 0xE0, CPL_FW6_PLD = 0xE1, @@ -200,6 +208,7 @@ enum { ULP_MODE_RDMA = 4, ULP_MODE_TCPDDP = 5, ULP_MODE_FCOE = 6, + ULP_MODE_TLS = 8, }; enum { @@ -993,6 +1002,23 @@ struct cpl_abort_req_rss { __u8 status; }; +struct cpl_abort_req_rss6 { + RSS_HDR + union opcode_tid ot; + __u32 srqidx_status; +}; + +#define S_ABORT_RSS_STATUS 0 +#define M_ABORT_RSS_STATUS 0xff +#define V_ABORT_RSS_STATUS(x) ((x) << S_ABORT_RSS_STATUS) +#define G_ABORT_RSS_STATUS(x) (((x) >> S_ABORT_RSS_STATUS) & M_ABORT_RSS_STATUS) + +#define S_ABORT_RSS_SRQIDX 8 +#define M_ABORT_RSS_SRQIDX 0xffffff +#define V_ABORT_RSS_SRQIDX(x) ((x) << S_ABORT_RSS_SRQIDX) +#define G_ABORT_RSS_SRQIDX(x) (((x) >> S_ABORT_RSS_SRQIDX) & M_ABORT_RSS_SRQIDX) + + /* cpl_abort_req status command code in case of T6, * bit[0] specifies whether to send RST (0) to remote peer or suppress it (1) * bit[1] indicates ABORT_REQ was sent after a CLOSE_CON_REQ @@ -1014,6 +1040,12 @@ struct cpl_abort_rpl_rss { __u8 status; }; +struct cpl_abort_rpl_rss6 { + RSS_HDR + union opcode_tid ot; + __u32 srqidx_status; +}; + struct cpl_abort_rpl { WR_HDR; union opcode_tid ot; @@ -2612,6 +2644,7 @@ enum { FW_TYPE_RSSCPL = 4, FW_TYPE_WRERR_RPL = 5, FW_TYPE_PI_ERR = 6, + FW_TYPE_TLS_KEY = 7, }; struct cpl_fw2_pld { @@ -2712,7 +2745,8 @@ enum { ULP_TX_SC_IMM = 0x81, ULP_TX_SC_DSGL = 0x82, ULP_TX_SC_ISGL = 0x83, - ULP_TX_SC_PICTRL = 0x84 + ULP_TX_SC_PICTRL = 0x84, + ULP_TX_SC_MEMRD = 0x86 }; #define S_ULPTX_CMD 24 @@ -2763,6 +2797,12 @@ struct ulptx_idata { #define S_ULPTX_NSGE 0 #define M_ULPTX_NSGE 0xFFFF #define V_ULPTX_NSGE(x) ((x) << S_ULPTX_NSGE) +#define G_ULPTX_NSGE(x) (((x) >> S_ULPTX_NSGE) & M_ULPTX_NSGE) + +struct ulptx_sc_memrd { + __be32 cmd_to_len; + __be32 addr; +}; struct ulp_mem_io { WR_HDR; @@ -2816,6 +2856,21 @@ struct ulp_txpkt { __be32 len; }; +/* ulp_txpkt.cmd_dest fields */ +#define S_ULP_TXPKT_DATAMODIFY 23 +#define M_ULP_TXPKT_DATAMODIFY 0x1 +#define V_ULP_TXPKT_DATAMODIFY(x) ((x) << S_ULP_TXPKT_DATAMODIFY) +#define G_ULP_TXPKT_DATAMODIFY(x) \ + (((x) >> S_ULP_TXPKT_DATAMODIFY) & M_ULP_TXPKT_DATAMODIFY_) +#define F_ULP_TXPKT_DATAMODIFY V_ULP_TXPKT_DATAMODIFY(1U) + +#define S_ULP_TXPKT_CHANNELID 22 +#define M_ULP_TXPKT_CHANNELID 0x1 +#define V_ULP_TXPKT_CHANNELID(x) ((x) << S_ULP_TXPKT_CHANNELID) +#define G_ULP_TXPKT_CHANNELID(x) \ + (((x) >> S_ULP_TXPKT_CHANNELID) & M_ULP_TXPKT_CHANNELID) +#define F_ULP_TXPKT_CHANNELID V_ULP_TXPKT_CHANNELID(1U) + /* ulp_txpkt.cmd_dest fields */ #define S_ULP_TXPKT_DEST 16 #define M_ULP_TXPKT_DEST 0x3 @@ -3044,4 +3099,542 @@ struct cpl_rx_mps_pkt { #define X_CPL_RX_MPS_PKT_TYPE_QFC (1 << 2) #define X_CPL_RX_MPS_PKT_TYPE_PTP (1 << 3) +struct cpl_tx_tls_sfo { + __be32 op_to_seg_len; + __be32 pld_len; + __be64 rsvd; + __be32 seqno_numivs; + __be32 ivgen_hdrlen; + __be64 scmd1; +}; + +/* cpl_tx_tls_sfo macros */ +#define S_CPL_TX_TLS_SFO_OPCODE 24 +#define M_CPL_TX_TLS_SFO_OPCODE 0xff +#define V_CPL_TX_TLS_SFO_OPCODE(x) ((x) << S_CPL_TX_TLS_SFO_OPCODE) +#define G_CPL_TX_TLS_SFO_OPCODE(x) \ + (((x) >> S_CPL_TX_TLS_SFO_OPCODE) & M_CPL_TX_TLS_SFO_OPCODE) + +#define S_CPL_TX_TLS_SFO_DATA_TYPE 20 +#define M_CPL_TX_TLS_SFO_DATA_TYPE 0xf +#define V_CPL_TX_TLS_SFO_DATA_TYPE(x) ((x) << S_CPL_TX_TLS_SFO_DATA_TYPE) +#define G_CPL_TX_TLS_SFO_DATA_TYPE(x) \ + (((x) >> S_CPL_TX_TLS_SFO_DATA_TYPE) & M_CPL_TX_TLS_SFO_DATA_TYPE) + +#define S_CPL_TX_TLS_SFO_CPL_LEN 16 +#define M_CPL_TX_TLS_SFO_CPL_LEN 0xf +#define V_CPL_TX_TLS_SFO_CPL_LEN(x) ((x) << S_CPL_TX_TLS_SFO_CPL_LEN) +#define G_CPL_TX_TLS_SFO_CPL_LEN(x) \ + (((x) >> S_CPL_TX_TLS_SFO_CPL_LEN) & M_CPL_TX_TLS_SFO_CPL_LEN) +#define S_CPL_TX_TLS_SFO_SEG_LEN 0 +#define M_CPL_TX_TLS_SFO_SEG_LEN 0xffff +#define V_CPL_TX_TLS_SFO_SEG_LEN(x) ((x) << S_CPL_TX_TLS_SFO_SEG_LEN) +#define G_CPL_TX_TLS_SFO_SEG_LEN(x) \ + (((x) >> S_CPL_TX_TLS_SFO_SEG_LEN) & M_CPL_TX_TLS_SFO_SEG_LEN) + +struct cpl_tls_data { + RSS_HDR + __be32 op_tid; + __be32 length_pkd; + __be32 seq; + __be32 r1; +}; + +#define S_CPL_TLS_DATA_OPCODE 24 +#define M_CPL_TLS_DATA_OPCODE 0xff +#define V_CPL_TLS_DATA_OPCODE(x) ((x) << S_CPL_TLS_DATA_OPCODE) +#define G_CPL_TLS_DATA_OPCODE(x) \ + (((x) >> S_CPL_TLS_DATA_OPCODE) & M_CPL_TLS_DATA_OPCODE) + +#define S_CPL_TLS_DATA_TID 0 +#define M_CPL_TLS_DATA_TID 0xffffff +#define V_CPL_TLS_DATA_TID(x) ((x) << S_CPL_TLS_DATA_TID) +#define G_CPL_TLS_DATA_TID(x) \ + (((x) >> S_CPL_TLS_DATA_TID) & M_CPL_TLS_DATA_TID) + +#define S_CPL_TLS_DATA_LENGTH 0 +#define M_CPL_TLS_DATA_LENGTH 0xffff +#define V_CPL_TLS_DATA_LENGTH(x) ((x) << S_CPL_TLS_DATA_LENGTH) +#define G_CPL_TLS_DATA_LENGTH(x) \ + (((x) >> S_CPL_TLS_DATA_LENGTH) & M_CPL_TLS_DATA_LENGTH) + +struct cpl_rx_tls_cmp { + RSS_HDR + __be32 op_tid; + __be32 pdulength_length; + __be32 seq; + __be32 ddp_report; + __be32 r; + __be32 ddp_valid; +}; + +#define S_CPL_RX_TLS_CMP_OPCODE 24 +#define M_CPL_RX_TLS_CMP_OPCODE 0xff +#define V_CPL_RX_TLS_CMP_OPCODE(x) ((x) << S_CPL_RX_TLS_CMP_OPCODE) +#define G_CPL_RX_TLS_CMP_OPCODE(x) \ + (((x) >> S_CPL_RX_TLS_CMP_OPCODE) & M_CPL_RX_TLS_CMP_OPCODE) + +#define S_CPL_RX_TLS_CMP_TID 0 +#define M_CPL_RX_TLS_CMP_TID 0xffffff +#define V_CPL_RX_TLS_CMP_TID(x) ((x) << S_CPL_RX_TLS_CMP_TID) +#define G_CPL_RX_TLS_CMP_TID(x) \ + (((x) >> S_CPL_RX_TLS_CMP_TID) & M_CPL_RX_TLS_CMP_TID) + +#define S_CPL_RX_TLS_CMP_PDULENGTH 16 +#define M_CPL_RX_TLS_CMP_PDULENGTH 0xffff +#define V_CPL_RX_TLS_CMP_PDULENGTH(x) ((x) << S_CPL_RX_TLS_CMP_PDULENGTH) +#define G_CPL_RX_TLS_CMP_PDULENGTH(x) \ + (((x) >> S_CPL_RX_TLS_CMP_PDULENGTH) & M_CPL_RX_TLS_CMP_PDULENGTH) + +#define S_CPL_RX_TLS_CMP_LENGTH 0 +#define M_CPL_RX_TLS_CMP_LENGTH 0xffff +#define V_CPL_RX_TLS_CMP_LENGTH(x) ((x) << S_CPL_RX_TLS_CMP_LENGTH) +#define G_CPL_RX_TLS_CMP_LENGTH(x) \ + (((x) >> S_CPL_RX_TLS_CMP_LENGTH) & M_CPL_RX_TLS_CMP_LENGTH) + +#define S_SCMD_SEQ_NO_CTRL 29 +#define M_SCMD_SEQ_NO_CTRL 0x3 +#define V_SCMD_SEQ_NO_CTRL(x) ((x) << S_SCMD_SEQ_NO_CTRL) +#define G_SCMD_SEQ_NO_CTRL(x) \ + (((x) >> S_SCMD_SEQ_NO_CTRL) & M_SCMD_SEQ_NO_CTRL) + +/* StsFieldPrsnt- Status field at the end of the TLS PDU */ +#define S_SCMD_STATUS_PRESENT 28 +#define M_SCMD_STATUS_PRESENT 0x1 +#define V_SCMD_STATUS_PRESENT(x) ((x) << S_SCMD_STATUS_PRESENT) +#define G_SCMD_STATUS_PRESENT(x) \ + (((x) >> S_SCMD_STATUS_PRESENT) & M_SCMD_STATUS_PRESENT) +#define F_SCMD_STATUS_PRESENT V_SCMD_STATUS_PRESENT(1U) + +/* ProtoVersion - Protocol Version 0: 1.2, 1:1.1, 2:DTLS, 3:Generic, + * 3-15: Reserved. */ +#define S_SCMD_PROTO_VERSION 24 +#define M_SCMD_PROTO_VERSION 0xf +#define V_SCMD_PROTO_VERSION(x) ((x) << S_SCMD_PROTO_VERSION) +#define G_SCMD_PROTO_VERSION(x) \ + (((x) >> S_SCMD_PROTO_VERSION) & M_SCMD_PROTO_VERSION) + +/* EncDecCtrl - Encryption/Decryption Control. 0: Encrypt, 1: Decrypt */ +#define S_SCMD_ENC_DEC_CTRL 23 +#define M_SCMD_ENC_DEC_CTRL 0x1 +#define V_SCMD_ENC_DEC_CTRL(x) ((x) << S_SCMD_ENC_DEC_CTRL) +#define G_SCMD_ENC_DEC_CTRL(x) \ + (((x) >> S_SCMD_ENC_DEC_CTRL) & M_SCMD_ENC_DEC_CTRL) +#define F_SCMD_ENC_DEC_CTRL V_SCMD_ENC_DEC_CTRL(1U) + +/* CipherAuthSeqCtrl - Cipher Authentication Sequence Control. */ +#define S_SCMD_CIPH_AUTH_SEQ_CTRL 22 +#define M_SCMD_CIPH_AUTH_SEQ_CTRL 0x1 +#define V_SCMD_CIPH_AUTH_SEQ_CTRL(x) \ + ((x) << S_SCMD_CIPH_AUTH_SEQ_CTRL) +#define G_SCMD_CIPH_AUTH_SEQ_CTRL(x) \ + (((x) >> S_SCMD_CIPH_AUTH_SEQ_CTRL) & M_SCMD_CIPH_AUTH_SEQ_CTRL) +#define F_SCMD_CIPH_AUTH_SEQ_CTRL V_SCMD_CIPH_AUTH_SEQ_CTRL(1U) + +/* CiphMode - Cipher Mode. 0: NOP, 1:AES-CBC, 2:AES-GCM, 3:AES-CTR, + * 4:Generic-AES, 5-15: Reserved. */ +#define S_SCMD_CIPH_MODE 18 +#define M_SCMD_CIPH_MODE 0xf +#define V_SCMD_CIPH_MODE(x) ((x) << S_SCMD_CIPH_MODE) +#define G_SCMD_CIPH_MODE(x) \ + (((x) >> S_SCMD_CIPH_MODE) & M_SCMD_CIPH_MODE) + +/* AuthMode - Auth Mode. 0: NOP, 1:SHA1, 2:SHA2-224, 3:SHA2-256 + * 4-15: Reserved */ +#define S_SCMD_AUTH_MODE 14 +#define M_SCMD_AUTH_MODE 0xf +#define V_SCMD_AUTH_MODE(x) ((x) << S_SCMD_AUTH_MODE) +#define G_SCMD_AUTH_MODE(x) \ + (((x) >> S_SCMD_AUTH_MODE) & M_SCMD_AUTH_MODE) + +/* HmacCtrl - HMAC Control. 0:NOP, 1:No truncation, 2:Support HMAC Truncation + * per RFC 4366, 3:IPSec 96 bits, 4-7:Reserved + */ +#define S_SCMD_HMAC_CTRL 11 +#define M_SCMD_HMAC_CTRL 0x7 +#define V_SCMD_HMAC_CTRL(x) ((x) << S_SCMD_HMAC_CTRL) +#define G_SCMD_HMAC_CTRL(x) \ + (((x) >> S_SCMD_HMAC_CTRL) & M_SCMD_HMAC_CTRL) + +/* IvSize - IV size in units of 2 bytes */ +#define S_SCMD_IV_SIZE 7 +#define M_SCMD_IV_SIZE 0xf +#define V_SCMD_IV_SIZE(x) ((x) << S_SCMD_IV_SIZE) +#define G_SCMD_IV_SIZE(x) \ + (((x) >> S_SCMD_IV_SIZE) & M_SCMD_IV_SIZE) + +/* NumIVs - Number of IVs */ +#define S_SCMD_NUM_IVS 0 +#define M_SCMD_NUM_IVS 0x7f +#define V_SCMD_NUM_IVS(x) ((x) << S_SCMD_NUM_IVS) +#define G_SCMD_NUM_IVS(x) \ + (((x) >> S_SCMD_NUM_IVS) & M_SCMD_NUM_IVS) + +/* EnbDbgId - If this is enabled upper 20 (63:44) bits if SeqNumber + * (below) are used as Cid (connection id for debug status), these + * bits are padded to zero for forming the 64 bit + * sequence number for TLS + */ +#define S_SCMD_ENB_DBGID 31 +#define M_SCMD_ENB_DBGID 0x1 +#define V_SCMD_ENB_DBGID(x) ((x) << S_SCMD_ENB_DBGID) +#define G_SCMD_ENB_DBGID(x) \ + (((x) >> S_SCMD_ENB_DBGID) & M_SCMD_ENB_DBGID) + +/* IV generation in SW. */ +#define S_SCMD_IV_GEN_CTRL 30 +#define M_SCMD_IV_GEN_CTRL 0x1 +#define V_SCMD_IV_GEN_CTRL(x) ((x) << S_SCMD_IV_GEN_CTRL) +#define G_SCMD_IV_GEN_CTRL(x) \ + (((x) >> S_SCMD_IV_GEN_CTRL) & M_SCMD_IV_GEN_CTRL) +#define F_SCMD_IV_GEN_CTRL V_SCMD_IV_GEN_CTRL(1U) + +/* More frags */ +#define S_SCMD_MORE_FRAGS 20 +#define M_SCMD_MORE_FRAGS 0x1 +#define V_SCMD_MORE_FRAGS(x) ((x) << S_SCMD_MORE_FRAGS) +#define G_SCMD_MORE_FRAGS(x) (((x) >> S_SCMD_MORE_FRAGS) & M_SCMD_MORE_FRAGS) + +/*last frag */ +#define S_SCMD_LAST_FRAG 19 +#define M_SCMD_LAST_FRAG 0x1 +#define V_SCMD_LAST_FRAG(x) ((x) << S_SCMD_LAST_FRAG) +#define G_SCMD_LAST_FRAG(x) (((x) >> S_SCMD_LAST_FRAG) & M_SCMD_LAST_FRAG) + +/* TlsCompPdu */ +#define S_SCMD_TLS_COMPPDU 18 +#define M_SCMD_TLS_COMPPDU 0x1 +#define V_SCMD_TLS_COMPPDU(x) ((x) << S_SCMD_TLS_COMPPDU) +#define G_SCMD_TLS_COMPPDU(x) (((x) >> S_SCMD_TLS_COMPPDU) & M_SCMD_TLS_COMPPDU) + +/* KeyCntxtInline - Key context inline after the scmd OR PayloadOnly*/ +#define S_SCMD_KEY_CTX_INLINE 17 +#define M_SCMD_KEY_CTX_INLINE 0x1 +#define V_SCMD_KEY_CTX_INLINE(x) ((x) << S_SCMD_KEY_CTX_INLINE) +#define G_SCMD_KEY_CTX_INLINE(x) \ + (((x) >> S_SCMD_KEY_CTX_INLINE) & M_SCMD_KEY_CTX_INLINE) +#define F_SCMD_KEY_CTX_INLINE V_SCMD_KEY_CTX_INLINE(1U) + +/* TLSFragEnable - 0: Host created TLS PDUs, 1: TLS Framgmentation in ASIC */ +#define S_SCMD_TLS_FRAG_ENABLE 16 +#define M_SCMD_TLS_FRAG_ENABLE 0x1 +#define V_SCMD_TLS_FRAG_ENABLE(x) ((x) << S_SCMD_TLS_FRAG_ENABLE) +#define G_SCMD_TLS_FRAG_ENABLE(x) \ + (((x) >> S_SCMD_TLS_FRAG_ENABLE) & M_SCMD_TLS_FRAG_ENABLE) +#define F_SCMD_TLS_FRAG_ENABLE V_SCMD_TLS_FRAG_ENABLE(1U) + +/* MacOnly - Only send the MAC and discard PDU. This is valid for hash only + * modes, in this case TLS_TX will drop the PDU and only + * send back the MAC bytes. */ +#define S_SCMD_MAC_ONLY 15 +#define M_SCMD_MAC_ONLY 0x1 +#define V_SCMD_MAC_ONLY(x) ((x) << S_SCMD_MAC_ONLY) +#define G_SCMD_MAC_ONLY(x) \ + (((x) >> S_SCMD_MAC_ONLY) & M_SCMD_MAC_ONLY) +#define F_SCMD_MAC_ONLY V_SCMD_MAC_ONLY(1U) + +/* AadIVDrop - Drop the AAD and IV fields. Useful in protocols + * which have complex AAD and IV formations Eg:AES-CCM + */ +#define S_SCMD_AADIVDROP 14 +#define M_SCMD_AADIVDROP 0x1 +#define V_SCMD_AADIVDROP(x) ((x) << S_SCMD_AADIVDROP) +#define G_SCMD_AADIVDROP(x) \ + (((x) >> S_SCMD_AADIVDROP) & M_SCMD_AADIVDROP) +#define F_SCMD_AADIVDROP V_SCMD_AADIVDROP(1U) + +/* HdrLength - Length of all headers excluding TLS header + * present before start of crypto PDU/payload. */ +#define S_SCMD_HDR_LEN 0 +#define M_SCMD_HDR_LEN 0x3fff +#define V_SCMD_HDR_LEN(x) ((x) << S_SCMD_HDR_LEN) +#define G_SCMD_HDR_LEN(x) \ + (((x) >> S_SCMD_HDR_LEN) & M_SCMD_HDR_LEN) + +struct cpl_tx_sec_pdu { + __be32 op_ivinsrtofst; + __be32 pldlen; + __be32 aadstart_cipherstop_hi; + __be32 cipherstop_lo_authinsert; + __be32 seqno_numivs; + __be32 ivgen_hdrlen; + __be64 scmd1; +}; + +#define S_CPL_TX_SEC_PDU_OPCODE 24 +#define M_CPL_TX_SEC_PDU_OPCODE 0xff +#define V_CPL_TX_SEC_PDU_OPCODE(x) ((x) << S_CPL_TX_SEC_PDU_OPCODE) +#define G_CPL_TX_SEC_PDU_OPCODE(x) \ + (((x) >> S_CPL_TX_SEC_PDU_OPCODE) & M_CPL_TX_SEC_PDU_OPCODE) + +/* RX Channel Id */ +#define S_CPL_TX_SEC_PDU_RXCHID 22 +#define M_CPL_TX_SEC_PDU_RXCHID 0x1 +#define V_CPL_TX_SEC_PDU_RXCHID(x) ((x) << S_CPL_TX_SEC_PDU_RXCHID) +#define G_CPL_TX_SEC_PDU_RXCHID(x) \ +(((x) >> S_CPL_TX_SEC_PDU_RXCHID) & M_CPL_TX_SEC_PDU_RXCHID) +#define F_CPL_TX_SEC_PDU_RXCHID V_CPL_TX_SEC_PDU_RXCHID(1U) + +/* Ack Follows */ +#define S_CPL_TX_SEC_PDU_ACKFOLLOWS 21 +#define M_CPL_TX_SEC_PDU_ACKFOLLOWS 0x1 +#define V_CPL_TX_SEC_PDU_ACKFOLLOWS(x) ((x) << S_CPL_TX_SEC_PDU_ACKFOLLOWS) +#define G_CPL_TX_SEC_PDU_ACKFOLLOWS(x) \ +(((x) >> S_CPL_TX_SEC_PDU_ACKFOLLOWS) & M_CPL_TX_SEC_PDU_ACKFOLLOWS) +#define F_CPL_TX_SEC_PDU_ACKFOLLOWS V_CPL_TX_SEC_PDU_ACKFOLLOWS(1U) + +/* Loopback bit in cpl_tx_sec_pdu */ +#define S_CPL_TX_SEC_PDU_ULPTXLPBK 20 +#define M_CPL_TX_SEC_PDU_ULPTXLPBK 0x1 +#define V_CPL_TX_SEC_PDU_ULPTXLPBK(x) ((x) << S_CPL_TX_SEC_PDU_ULPTXLPBK) +#define G_CPL_TX_SEC_PDU_ULPTXLPBK(x) \ +(((x) >> S_CPL_TX_SEC_PDU_ULPTXLPBK) & M_CPL_TX_SEC_PDU_ULPTXLPBK) +#define F_CPL_TX_SEC_PDU_ULPTXLPBK V_CPL_TX_SEC_PDU_ULPTXLPBK(1U) + +/* Length of cpl header encapsulated */ +#define S_CPL_TX_SEC_PDU_CPLLEN 16 +#define M_CPL_TX_SEC_PDU_CPLLEN 0xf +#define V_CPL_TX_SEC_PDU_CPLLEN(x) ((x) << S_CPL_TX_SEC_PDU_CPLLEN) +#define G_CPL_TX_SEC_PDU_CPLLEN(x) \ + (((x) >> S_CPL_TX_SEC_PDU_CPLLEN) & M_CPL_TX_SEC_PDU_CPLLEN) + +/* PlaceHolder */ +#define S_CPL_TX_SEC_PDU_PLACEHOLDER 10 +#define M_CPL_TX_SEC_PDU_PLACEHOLDER 0x1 +#define V_CPL_TX_SEC_PDU_PLACEHOLDER(x) ((x) << S_CPL_TX_SEC_PDU_PLACEHOLDER) +#define G_CPL_TX_SEC_PDU_PLACEHOLDER(x) \ + (((x) >> S_CPL_TX_SEC_PDU_PLACEHOLDER) & \ + M_CPL_TX_SEC_PDU_PLACEHOLDER) + +/* IvInsrtOffset: Insertion location for IV */ +#define S_CPL_TX_SEC_PDU_IVINSRTOFST 0 +#define M_CPL_TX_SEC_PDU_IVINSRTOFST 0x3ff +#define V_CPL_TX_SEC_PDU_IVINSRTOFST(x) ((x) << S_CPL_TX_SEC_PDU_IVINSRTOFST) +#define G_CPL_TX_SEC_PDU_IVINSRTOFST(x) \ + (((x) >> S_CPL_TX_SEC_PDU_IVINSRTOFST) & \ + M_CPL_TX_SEC_PDU_IVINSRTOFST) + +/* AadStartOffset: Offset in bytes for AAD start from + * the first byte following + * the pkt headers (0-255 + * bytes) */ +#define S_CPL_TX_SEC_PDU_AADSTART 24 +#define M_CPL_TX_SEC_PDU_AADSTART 0xff +#define V_CPL_TX_SEC_PDU_AADSTART(x) ((x) << S_CPL_TX_SEC_PDU_AADSTART) +#define G_CPL_TX_SEC_PDU_AADSTART(x) \ + (((x) >> S_CPL_TX_SEC_PDU_AADSTART) & \ + M_CPL_TX_SEC_PDU_AADSTART) + +/* AadStopOffset: offset in bytes for AAD stop/end from the first byte following + * the pkt headers (0-511 bytes) */ +#define S_CPL_TX_SEC_PDU_AADSTOP 15 +#define M_CPL_TX_SEC_PDU_AADSTOP 0x1ff +#define V_CPL_TX_SEC_PDU_AADSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AADSTOP) +#define G_CPL_TX_SEC_PDU_AADSTOP(x) \ + (((x) >> S_CPL_TX_SEC_PDU_AADSTOP) & M_CPL_TX_SEC_PDU_AADSTOP) + +/* CipherStartOffset: offset in bytes for encryption/decryption start from the + * first byte following the pkt headers (0-1023 + * bytes) */ +#define S_CPL_TX_SEC_PDU_CIPHERSTART 5 +#define M_CPL_TX_SEC_PDU_CIPHERSTART 0x3ff +#define V_CPL_TX_SEC_PDU_CIPHERSTART(x) ((x) << S_CPL_TX_SEC_PDU_CIPHERSTART) +#define G_CPL_TX_SEC_PDU_CIPHERSTART(x) \ + (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTART) & \ + M_CPL_TX_SEC_PDU_CIPHERSTART) + +/* CipherStopOffset: offset in bytes for encryption/decryption end + * from end of the payload of this command (0-511 bytes) */ +#define S_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0 +#define M_CPL_TX_SEC_PDU_CIPHERSTOP_HI 0x1f +#define V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \ + ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_HI) +#define G_CPL_TX_SEC_PDU_CIPHERSTOP_HI(x) \ + (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_HI) & \ + M_CPL_TX_SEC_PDU_CIPHERSTOP_HI) + +#define S_CPL_TX_SEC_PDU_CIPHERSTOP_LO 28 +#define M_CPL_TX_SEC_PDU_CIPHERSTOP_LO 0xf +#define V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \ + ((x) << S_CPL_TX_SEC_PDU_CIPHERSTOP_LO) +#define G_CPL_TX_SEC_PDU_CIPHERSTOP_LO(x) \ + (((x) >> S_CPL_TX_SEC_PDU_CIPHERSTOP_LO) & \ + M_CPL_TX_SEC_PDU_CIPHERSTOP_LO) + +/* AuthStartOffset: offset in bytes for authentication start from + * the first byte following the pkt headers (0-1023) + * */ +#define S_CPL_TX_SEC_PDU_AUTHSTART 18 +#define M_CPL_TX_SEC_PDU_AUTHSTART 0x3ff +#define V_CPL_TX_SEC_PDU_AUTHSTART(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTART) +#define G_CPL_TX_SEC_PDU_AUTHSTART(x) \ + (((x) >> S_CPL_TX_SEC_PDU_AUTHSTART) & \ + M_CPL_TX_SEC_PDU_AUTHSTART) + +/* AuthStopOffset: offset in bytes for authentication + * end from end of the payload of this command (0-511 Bytes) */ +#define S_CPL_TX_SEC_PDU_AUTHSTOP 9 +#define M_CPL_TX_SEC_PDU_AUTHSTOP 0x1ff +#define V_CPL_TX_SEC_PDU_AUTHSTOP(x) ((x) << S_CPL_TX_SEC_PDU_AUTHSTOP) +#define G_CPL_TX_SEC_PDU_AUTHSTOP(x) \ + (((x) >> S_CPL_TX_SEC_PDU_AUTHSTOP) & \ + M_CPL_TX_SEC_PDU_AUTHSTOP) + +/* AuthInsrtOffset: offset in bytes for authentication insertion + * from end of the payload of this command (0-511 bytes) */ +#define S_CPL_TX_SEC_PDU_AUTHINSERT 0 +#define M_CPL_TX_SEC_PDU_AUTHINSERT 0x1ff +#define V_CPL_TX_SEC_PDU_AUTHINSERT(x) ((x) << S_CPL_TX_SEC_PDU_AUTHINSERT) +#define G_CPL_TX_SEC_PDU_AUTHINSERT(x) \ + (((x) >> S_CPL_TX_SEC_PDU_AUTHINSERT) & \ + M_CPL_TX_SEC_PDU_AUTHINSERT) + +struct cpl_rx_phys_dsgl { + __be32 op_to_tid; + __be32 pcirlxorder_to_noofsgentr; + struct rss_header rss_hdr_int; +}; + +#define S_CPL_RX_PHYS_DSGL_OPCODE 24 +#define M_CPL_RX_PHYS_DSGL_OPCODE 0xff +#define V_CPL_RX_PHYS_DSGL_OPCODE(x) ((x) << S_CPL_RX_PHYS_DSGL_OPCODE) +#define G_CPL_RX_PHYS_DSGL_OPCODE(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_OPCODE) & M_CPL_RX_PHYS_DSGL_OPCODE) + +#define S_CPL_RX_PHYS_DSGL_ISRDMA 23 +#define M_CPL_RX_PHYS_DSGL_ISRDMA 0x1 +#define V_CPL_RX_PHYS_DSGL_ISRDMA(x) ((x) << S_CPL_RX_PHYS_DSGL_ISRDMA) +#define G_CPL_RX_PHYS_DSGL_ISRDMA(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_ISRDMA) & M_CPL_RX_PHYS_DSGL_ISRDMA) +#define F_CPL_RX_PHYS_DSGL_ISRDMA V_CPL_RX_PHYS_DSGL_ISRDMA(1U) + +#define S_CPL_RX_PHYS_DSGL_RSVD1 20 +#define M_CPL_RX_PHYS_DSGL_RSVD1 0x7 +#define V_CPL_RX_PHYS_DSGL_RSVD1(x) ((x) << S_CPL_RX_PHYS_DSGL_RSVD1) +#define G_CPL_RX_PHYS_DSGL_RSVD1(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_RSVD1) & M_CPL_RX_PHYS_DSGL_RSVD1) + +#define S_CPL_RX_PHYS_DSGL_PCIRLXORDER 31 +#define M_CPL_RX_PHYS_DSGL_PCIRLXORDER 0x1 +#define V_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \ + ((x) << S_CPL_RX_PHYS_DSGL_PCIRLXORDER) +#define G_CPL_RX_PHYS_DSGL_PCIRLXORDER(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_PCIRLXORDER) & \ + M_CPL_RX_PHYS_DSGL_PCIRLXORDER) +#define F_CPL_RX_PHYS_DSGL_PCIRLXORDER V_CPL_RX_PHYS_DSGL_PCIRLXORDER(1U) + +#define S_CPL_RX_PHYS_DSGL_PCINOSNOOP 30 +#define M_CPL_RX_PHYS_DSGL_PCINOSNOOP 0x1 +#define V_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \ + ((x) << S_CPL_RX_PHYS_DSGL_PCINOSNOOP) +#define G_CPL_RX_PHYS_DSGL_PCINOSNOOP(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_PCINOSNOOP) & \ + M_CPL_RX_PHYS_DSGL_PCINOSNOOP) +#define F_CPL_RX_PHYS_DSGL_PCINOSNOOP V_CPL_RX_PHYS_DSGL_PCINOSNOOP(1U) + +#define S_CPL_RX_PHYS_DSGL_PCITPHNTENB 29 +#define M_CPL_RX_PHYS_DSGL_PCITPHNTENB 0x1 +#define V_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \ + ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNTENB) +#define G_CPL_RX_PHYS_DSGL_PCITPHNTENB(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNTENB) & \ + M_CPL_RX_PHYS_DSGL_PCITPHNTENB) +#define F_CPL_RX_PHYS_DSGL_PCITPHNTENB V_CPL_RX_PHYS_DSGL_PCITPHNTENB(1U) + +#define S_CPL_RX_PHYS_DSGL_PCITPHNT 27 +#define M_CPL_RX_PHYS_DSGL_PCITPHNT 0x3 +#define V_CPL_RX_PHYS_DSGL_PCITPHNT(x) ((x) << S_CPL_RX_PHYS_DSGL_PCITPHNT) +#define G_CPL_RX_PHYS_DSGL_PCITPHNT(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_PCITPHNT) & \ + M_CPL_RX_PHYS_DSGL_PCITPHNT) + +#define S_CPL_RX_PHYS_DSGL_DCAID 16 +#define M_CPL_RX_PHYS_DSGL_DCAID 0x7ff +#define V_CPL_RX_PHYS_DSGL_DCAID(x) ((x) << S_CPL_RX_PHYS_DSGL_DCAID) +#define G_CPL_RX_PHYS_DSGL_DCAID(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_DCAID) & \ + M_CPL_RX_PHYS_DSGL_DCAID) + +#define S_CPL_RX_PHYS_DSGL_NOOFSGENTR 0 +#define M_CPL_RX_PHYS_DSGL_NOOFSGENTR 0xffff +#define V_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \ + ((x) << S_CPL_RX_PHYS_DSGL_NOOFSGENTR) +#define G_CPL_RX_PHYS_DSGL_NOOFSGENTR(x) \ + (((x) >> S_CPL_RX_PHYS_DSGL_NOOFSGENTR) & \ + M_CPL_RX_PHYS_DSGL_NOOFSGENTR) + +/* CPL_TX_TLS_ACK */ +struct cpl_tx_tls_ack { + __be32 op_to_Rsvd2; + __be32 PldLen; + __be64 Rsvd3; +}; + +#define S_CPL_TX_TLS_ACK_OPCODE 24 +#define M_CPL_TX_TLS_ACK_OPCODE 0xff +#define V_CPL_TX_TLS_ACK_OPCODE(x) ((x) << S_CPL_TX_TLS_ACK_OPCODE) +#define G_CPL_TX_TLS_ACK_OPCODE(x) \ + (((x) >> S_CPL_TX_TLS_ACK_OPCODE) & M_CPL_TX_TLS_ACK_OPCODE) + +#define S_CPL_TX_TLS_ACK_RSVD1 23 +#define M_CPL_TX_TLS_ACK_RSVD1 0x1 +#define V_CPL_TX_TLS_ACK_RSVD1(x) ((x) << S_CPL_TX_TLS_ACK_RSVD1) +#define G_CPL_TX_TLS_ACK_RSVD1(x) \ + (((x) >> S_CPL_TX_TLS_ACK_RSVD1) & M_CPL_TX_TLS_ACK_RSVD1) +#define F_CPL_TX_TLS_ACK_RSVD1 V_CPL_TX_TLS_ACK_RSVD1(1U) + +#define S_CPL_TX_TLS_ACK_RXCHID 22 +#define M_CPL_TX_TLS_ACK_RXCHID 0x1 +#define V_CPL_TX_TLS_ACK_RXCHID(x) ((x) << S_CPL_TX_TLS_ACK_RXCHID) +#define G_CPL_TX_TLS_ACK_RXCHID(x) \ + (((x) >> S_CPL_TX_TLS_ACK_RXCHID) & M_CPL_TX_TLS_ACK_RXCHID) +#define F_CPL_TX_TLS_ACK_RXCHID V_CPL_TX_TLS_ACK_RXCHID(1U) + +#define S_CPL_TX_TLS_ACK_FWMSG 21 +#define M_CPL_TX_TLS_ACK_FWMSG 0x1 +#define V_CPL_TX_TLS_ACK_FWMSG(x) ((x) << S_CPL_TX_TLS_ACK_FWMSG) +#define G_CPL_TX_TLS_ACK_FWMSG(x) \ + (((x) >> S_CPL_TX_TLS_ACK_FWMSG) & M_CPL_TX_TLS_ACK_FWMSG) +#define F_CPL_TX_TLS_ACK_FWMSG V_CPL_TX_TLS_ACK_FWMSG(1U) + +#define S_CPL_TX_TLS_ACK_ULPTXLPBK 20 +#define M_CPL_TX_TLS_ACK_ULPTXLPBK 0x1 +#define V_CPL_TX_TLS_ACK_ULPTXLPBK(x) ((x) << S_CPL_TX_TLS_ACK_ULPTXLPBK) +#define G_CPL_TX_TLS_ACK_ULPTXLPBK(x) \ + (((x) >> S_CPL_TX_TLS_ACK_ULPTXLPBK) & M_CPL_TX_TLS_ACK_ULPTXLPBK) +#define F_CPL_TX_TLS_ACK_ULPTXLPBK V_CPL_TX_TLS_ACK_ULPTXLPBK(1U) + +#define S_CPL_TX_TLS_ACK_CPLLEN 16 +#define M_CPL_TX_TLS_ACK_CPLLEN 0xf +#define V_CPL_TX_TLS_ACK_CPLLEN(x) ((x) << S_CPL_TX_TLS_ACK_CPLLEN) +#define G_CPL_TX_TLS_ACK_CPLLEN(x) \ + (((x) >> S_CPL_TX_TLS_ACK_CPLLEN) & M_CPL_TX_TLS_ACK_CPLLEN) + +#define S_CPL_TX_TLS_ACK_COMPLONERR 15 +#define M_CPL_TX_TLS_ACK_COMPLONERR 0x1 +#define V_CPL_TX_TLS_ACK_COMPLONERR(x) ((x) << S_CPL_TX_TLS_ACK_COMPLONERR) +#define G_CPL_TX_TLS_ACK_COMPLONERR(x) \ + (((x) >> S_CPL_TX_TLS_ACK_COMPLONERR) & M_CPL_TX_TLS_ACK_COMPLONERR) +#define F_CPL_TX_TLS_ACK_COMPLONERR V_CPL_TX_TLS_ACK_COMPLONERR(1U) + +#define S_CPL_TX_TLS_ACK_LCB 14 +#define M_CPL_TX_TLS_ACK_LCB 0x1 +#define V_CPL_TX_TLS_ACK_LCB(x) ((x) << S_CPL_TX_TLS_ACK_LCB) +#define G_CPL_TX_TLS_ACK_LCB(x) \ + (((x) >> S_CPL_TX_TLS_ACK_LCB) & M_CPL_TX_TLS_ACK_LCB) +#define F_CPL_TX_TLS_ACK_LCB V_CPL_TX_TLS_ACK_LCB(1U) + +#define S_CPL_TX_TLS_ACK_PHASH 13 +#define M_CPL_TX_TLS_ACK_PHASH 0x1 +#define V_CPL_TX_TLS_ACK_PHASH(x) ((x) << S_CPL_TX_TLS_ACK_PHASH) +#define G_CPL_TX_TLS_ACK_PHASH(x) \ + (((x) >> S_CPL_TX_TLS_ACK_PHASH) & M_CPL_TX_TLS_ACK_PHASH) +#define F_CPL_TX_TLS_ACK_PHASH V_CPL_TX_TLS_ACK_PHASH(1U) + +#define S_CPL_TX_TLS_ACK_RSVD2 0 +#define M_CPL_TX_TLS_ACK_RSVD2 0x1fff +#define V_CPL_TX_TLS_ACK_RSVD2(x) ((x) << S_CPL_TX_TLS_ACK_RSVD2) +#define G_CPL_TX_TLS_ACK_RSVD2(x) \ + (((x) >> S_CPL_TX_TLS_ACK_RSVD2) & M_CPL_TX_TLS_ACK_RSVD2) + #endif /* T4_MSG_H */ diff --git a/sys/dev/cxgbe/iw_cxgbe/provider.c b/sys/dev/cxgbe/iw_cxgbe/provider.c index af9da14ce5b7..423c33b82bce 100644 --- a/sys/dev/cxgbe/iw_cxgbe/provider.c +++ b/sys/dev/cxgbe/iw_cxgbe/provider.c @@ -429,7 +429,7 @@ c4iw_register_device(struct c4iw_dev *dev) strlcpy(ibdev->node_desc, C4IW_NODE_DESC, sizeof(ibdev->node_desc)); ibdev->phys_port_cnt = sc->params.nports; ibdev->num_comp_vectors = 1; - ibdev->dma_device = sc->dev; + ibdev->dma_device = NULL; ibdev->query_device = c4iw_query_device; ibdev->query_port = c4iw_query_port; ibdev->modify_port = c4iw_modify_port; diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 1569b7439ff8..d4048cd4ce3e 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -623,6 +623,7 @@ struct { #endif }, t6_pciids[] = { {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */ + {0x6400, "Chelsio T6225-DBG"}, /* 2 x 10/25G, debug */ {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */ {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */ {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */ diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index bdfaa4db2b00..aa0a8a3b804e 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -2298,7 +2298,7 @@ start_wrq_wr(struct sge_wrq *wrq, int len16, struct wrq_cookie *cookie) w = &eq->desc[eq->pidx]; IDXINCR(eq->pidx, ndesc, eq->sidx); - if (__predict_false(eq->pidx < ndesc - 1)) { + if (__predict_false(cookie->pidx + ndesc > eq->sidx)) { w = &wrq->ss[0]; wrq->ss_pidx = cookie->pidx; wrq->ss_len = len16 * 16; @@ -3305,12 +3305,13 @@ ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); c.physeqid_pkd = htobe32(0); c.fetchszm_to_iqid = - htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | + htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) | V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) | + V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) | V_FW_EQ_CTRL_CMD_EQSIZE(qsize)); c.eqaddr = htobe64(eq->ba); diff --git a/sys/dev/e1000/em_txrx.c b/sys/dev/e1000/em_txrx.c new file mode 100644 index 000000000000..33a9b9d115f4 --- /dev/null +++ b/sys/dev/e1000/em_txrx.c @@ -0,0 +1,746 @@ +/*- + * Copyright (c) 2016 Matt Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $FreeBSD$ */ +#include "if_em.h" + +#ifdef RSS +#include +#include +#endif + +#ifdef VERBOSE_DEBUG +#define DPRINTF device_printf +#else +#define DPRINTF(...) +#endif + +/********************************************************************* + * Local Function prototypes + *********************************************************************/ +static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); +static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); +static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); +static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear); +static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); +static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); +static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, + int budget); +static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); + +static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, + int budget); +static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); +static void em_receive_checksum(uint32_t status, if_rxd_info_t ri); +extern int em_intr(void *arg); + +struct if_txrx em_txrx = { + em_isc_txd_encap, + em_isc_txd_flush, + em_isc_txd_credits_update, + em_isc_rxd_available, + em_isc_rxd_pkt_get, + em_isc_rxd_refill, + em_isc_rxd_flush, + em_intr +}; + +struct if_txrx lem_txrx = { + em_isc_txd_encap, + em_isc_txd_flush, + em_isc_txd_credits_update, + lem_isc_rxd_available, + lem_isc_rxd_pkt_get, + lem_isc_rxd_refill, + em_isc_rxd_flush, + em_intr +}; + +extern if_shared_ctx_t em_sctx; + +/********************************************************************** + * + * Setup work for hardware segmentation offload (TSO) on + * adapters using advanced tx descriptors + * + **********************************************************************/ +static int +em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) +{ + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + struct e1000_context_desc *TXD; + struct em_txbuffer *tx_buffer; + int cur, hdr_len; + + hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; + *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ + E1000_TXD_DTYP_D | /* Data descr type */ + E1000_TXD_CMD_TSE); /* Do TSE on this packet */ + + /* IP and/or TCP header checksum calculation and insertion. */ + *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; + + cur = pi->ipi_pidx; + TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; + tx_buffer = &txr->tx_buffers[cur]; + + /* + * Start offset for header checksum calculation. + * End offset for header checksum calculation. + * Offset of place put the checksum. + */ + TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; + TXD->lower_setup.ip_fields.ipcse = + htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); + TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); + + /* + * Start offset for payload checksum calculation. + * End offset for payload checksum calculation. + * Offset of place to put the checksum. + */ + TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + TXD->upper_setup.tcp_fields.tucse = 0; + TXD->upper_setup.tcp_fields.tucso = + pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); + + /* + * Payload size per packet w/o any headers. + * Length of all headers up to payload. + */ + TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); + TXD->tcp_seg_setup.fields.hdr_len = hdr_len; + + TXD->cmd_and_length = htole32(adapter->txd_cmd | + E1000_TXD_CMD_DEXT | /* Extended descr */ + E1000_TXD_CMD_TSE | /* TSE context */ + E1000_TXD_CMD_IP | /* Do IP csum */ + E1000_TXD_CMD_TCP | /* Do TCP checksum */ + (pi->ipi_len - hdr_len)); /* Total len */ + tx_buffer->eop = -1; + txr->tx_tso = TRUE; + + if (++cur == scctx->isc_ntxd[0]) { + cur = 0; + } + DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); + return (cur); +} + +#define TSO_WORKAROUND 4 +#define DONT_FORCE_CTX 1 + + +/********************************************************************* + * The offload context is protocol specific (TCP/UDP) and thus + * only needs to be set when the protocol changes. The occasion + * of a context change can be a performance detriment, and + * might be better just disabled. The reason arises in the way + * in which the controller supports pipelined requests from the + * Tx data DMA. Up to four requests can be pipelined, and they may + * belong to the same packet or to multiple packets. However all + * requests for one packet are issued before a request is issued + * for a subsequent packet and if a request for the next packet + * requires a context change, that request will be stalled + * until the previous request completes. This means setting up + * a new context effectively disables pipelined Tx data DMA which + * in turn greatly slow down performance to send small sized + * frames. + **********************************************************************/ + +static int +em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) +{ + struct e1000_context_desc *TXD = NULL; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + struct em_txbuffer *tx_buffer; + int csum_flags = pi->ipi_csum_flags; + int cur, hdr_len; + u32 cmd; + + cur = pi->ipi_pidx; + hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; + cmd = adapter->txd_cmd; + + /* + * The 82574L can only remember the *last* context used + * regardless of queue that it was use for. We cannot reuse + * contexts on this hardware platform and must generate a new + * context every time. 82574L hardware spec, section 7.2.6, + * second note. + */ + if (DONT_FORCE_CTX && + adapter->tx_num_queues == 1 && + txr->csum_lhlen == pi->ipi_ehdrlen && + txr->csum_iphlen == pi->ipi_ip_hlen && + txr->csum_flags == csum_flags) { + /* + * Same csum offload context as the previous packets; + * just return. + */ + *txd_upper = txr->csum_txd_upper; + *txd_lower = txr->csum_txd_lower; + return (cur); + } + + TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; + if (csum_flags & CSUM_IP) { + *txd_upper |= E1000_TXD_POPTS_IXSM << 8; + /* + * Start offset for header checksum calculation. + * End offset for header checksum calculation. + * Offset of place to put the checksum. + */ + TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; + TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); + TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); + cmd |= E1000_TXD_CMD_IP; + } + + if (csum_flags & (CSUM_TCP|CSUM_UDP)) { + uint8_t tucso; + + *txd_upper |= E1000_TXD_POPTS_TXSM << 8; + *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; + + if (csum_flags & CSUM_TCP) { + tucso = hdr_len + offsetof(struct tcphdr, th_sum); + cmd |= E1000_TXD_CMD_TCP; + } else + tucso = hdr_len + offsetof(struct udphdr, uh_sum); + TXD->upper_setup.tcp_fields.tucss = hdr_len; + TXD->upper_setup.tcp_fields.tucse = htole16(0); + TXD->upper_setup.tcp_fields.tucso = tucso; + } + + txr->csum_lhlen = pi->ipi_ehdrlen; + txr->csum_iphlen = pi->ipi_ip_hlen; + txr->csum_flags = csum_flags; + txr->csum_txd_upper = *txd_upper; + txr->csum_txd_lower = *txd_lower; + + TXD->tcp_seg_setup.data = htole32(0); + TXD->cmd_and_length = + htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); + + tx_buffer = &txr->tx_buffers[cur]; + tx_buffer->eop = -1; + + if (++cur == scctx->isc_ntxd[0]) { + cur = 0; + } + DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", + csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); + return (cur); +} + +static int +em_isc_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + bus_dma_segment_t *segs = pi->ipi_segs; + int nsegs = pi->ipi_nsegs; + int csum_flags = pi->ipi_csum_flags; + int i, j, first, pidx_last; + u32 txd_upper = 0, txd_lower = 0; + + struct em_txbuffer *tx_buffer; + struct e1000_tx_desc *ctxd = NULL; + bool do_tso, tso_desc; + + i = first = pi->ipi_pidx; + do_tso = (csum_flags & CSUM_TSO); + tso_desc = FALSE; + /* + * TSO Hardware workaround, if this packet is not + * TSO, and is only a single descriptor long, and + * it follows a TSO burst, then we need to add a + * sentinel descriptor to prevent premature writeback. + */ + if ((!do_tso) && (txr->tx_tso == TRUE)) { + if (nsegs == 1) + tso_desc = TRUE; + txr->tx_tso = FALSE; + } + + /* Do hardware assists */ + if (do_tso) { + i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); + tso_desc = TRUE; + } else if (csum_flags & EM_CSUM_OFFLOAD) { + i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); + } + + if (pi->ipi_mflags & M_VLANTAG) { + /* Set the vlan id. */ + txd_upper |= htole16(pi->ipi_vtag) << 16; + /* Tell hardware to add tag */ + txd_lower |= htole32(E1000_TXD_CMD_VLE); + } + + DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); + /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ + + /* Set up our transmit descriptors */ + for (j = 0; j < nsegs; j++) { + bus_size_t seg_len; + bus_addr_t seg_addr; + uint32_t cmd; + + ctxd = &txr->tx_base[i]; + tx_buffer = &txr->tx_buffers[i]; + seg_addr = segs[j].ds_addr; + seg_len = segs[j].ds_len; + cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; + + /* + ** TSO Workaround: + ** If this is the last descriptor, we want to + ** split it so we have a small final sentinel + */ + if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { + seg_len -= TSO_WORKAROUND; + ctxd->buffer_addr = htole64(seg_addr); + ctxd->lower.data = htole32(cmd | txd_lower | seg_len); + ctxd->upper.data = htole32(txd_upper); + + if (++i == scctx->isc_ntxd[0]) + i = 0; + + /* Now make the sentinel */ + ctxd = &txr->tx_base[i]; + tx_buffer = &txr->tx_buffers[i]; + ctxd->buffer_addr = htole64(seg_addr + seg_len); + ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); + ctxd->upper.data = htole32(txd_upper); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) + i = 0; + DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); + } else { + ctxd->buffer_addr = htole64(seg_addr); + ctxd->lower.data = htole32(cmd | txd_lower | seg_len); + ctxd->upper.data = htole32(txd_upper); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) + i = 0; + DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); + } + tx_buffer->eop = -1; + } + + /* + * Last Descriptor of Packet + * needs End Of Packet (EOP) + * and Report Status (RS) + */ + ctxd->lower.data |= + htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); + + tx_buffer = &txr->tx_buffers[first]; + tx_buffer->eop = pidx_last; + DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); + pi->ipi_new_pidx = i; + + return (0); +} + +static void +em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) +{ + struct adapter *adapter = arg; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); +} + +static int +em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + u32 cidx, processed = 0; + int last, done; + struct em_txbuffer *buf; + struct e1000_tx_desc *tx_desc, *eop_desc; + + cidx = cidx_init; + buf = &txr->tx_buffers[cidx]; + tx_desc = &txr->tx_base[cidx]; + last = buf->eop; + eop_desc = &txr->tx_base[last]; + + DPRINTF(iflib_get_dev(adapter->ctx), "credits_update: cidx_init=%d clear=%d last=%d\n", + cidx_init, clear, last); + /* + * What this does is get the index of the + * first descriptor AFTER the EOP of the + * first packet, that way we can do the + * simple comparison on the inner while loop. + */ + if (++last == scctx->isc_ntxd[0]) + last = 0; + done = last; + + + while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { + /* We clean the range of the packet */ + while (cidx != done) { + if (clear) { + tx_desc->upper.data = 0; + tx_desc->lower.data = 0; + tx_desc->buffer_addr = 0; + buf->eop = -1; + } + tx_desc++; + buf++; + processed++; + + /* wrap the ring ? */ + if (++cidx == scctx->isc_ntxd[0]) { + cidx = 0; + } + buf = &txr->tx_buffers[cidx]; + tx_desc = &txr->tx_base[cidx]; + } + /* See if we can continue to the next packet */ + last = buf->eop; + if (last == -1) + break; + eop_desc = &txr->tx_base[last]; + /* Get new done point */ + if (++last == scctx->isc_ntxd[0]) + last = 0; + done = last; + } + + DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed); + return(processed); +} + +static void +lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + int i; + uint32_t next_pidx; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; + rxd->buffer_addr = htole64(paddrs[i]); + /* status bits must be cleared */ + rxd->status = 0; + + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + int i; + uint32_t next_pidx; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = &rxr->rx_base[next_pidx]; + rxd->read.buffer_addr = htole64(paddrs[i]); + /* DD bits must be cleared */ + rxd->wb.upper.status_error = 0; + + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) +{ + struct adapter *sc = arg; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + + E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); +} + +static int +lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + u32 staterr = 0; + int cnt, i; + + for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; + staterr = rxd->status; + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) + i = 0; + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + } + return (cnt); +} + +static int +em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + u32 staterr = 0; + int cnt, i; + + for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { + rxd = &rxr->rx_base[i]; + staterr = le32toh(rxd->wb.upper.status_error); + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) { + i = 0; + } + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + + } + return (cnt); +} + +static int +lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + struct e1000_rx_desc *rxd; + u16 len; + u32 status, errors; + bool eop; + int i, cidx; + + status = errors = i = 0; + cidx = ri->iri_cidx; + + do { + rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; + status = rxd->status; + errors = rxd->errors; + + /* Error Checking then decrement count */ + MPASS ((status & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->length); + ri->iri_len += len; + + eop = (status & E1000_RXD_STAT_EOP) != 0; + + /* Make sure bad packets are discarded */ + if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { + adapter->dropped_pkts++; + /* XXX fixup if common */ + return (EBADMSG); + } + + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + /* Zero out the receive descriptors status. */ + rxd->status = 0; + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + i++; + } while (!eop); + + /* XXX add a faster way to look this up */ + if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM)) + lem_receive_checksum(status, errors, ri); + + if (status & E1000_RXD_STAT_VP) { + ri->iri_vtag = le16toh(rxd->special); + ri->iri_flags |= M_VLANTAG; + } + + ri->iri_nfrags = i; + + return (0); +} + +static int +em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + union e1000_rx_desc_extended *rxd; + + u16 len; + u32 staterr = 0; + bool eop; + int i, cidx, vtag; + + i = vtag = 0; + cidx = ri->iri_cidx; + + do { + rxd = &rxr->rx_base[cidx]; + staterr = le32toh(rxd->wb.upper.status_error); + + /* Error Checking then decrement count */ + MPASS ((staterr & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->wb.upper.length); + ri->iri_len += len; + + eop = (staterr & E1000_RXD_STAT_EOP) != 0; + + /* Make sure bad packets are discarded */ + if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { + adapter->dropped_pkts++; + return EBADMSG; + } + + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + /* Zero out the receive descriptors status. */ + rxd->wb.upper.status_error &= htole32(~0xFF); + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + i++; + } while (!eop); + + /* XXX add a faster way to look this up */ + if (adapter->hw.mac.type >= e1000_82543) + em_receive_checksum(staterr, ri); + + if (staterr & E1000_RXD_STAT_VP) { + vtag = le16toh(rxd->wb.upper.vlan); + } + + ri->iri_vtag = vtag; + ri->iri_nfrags = i; + if (vtag) + ri->iri_flags |= M_VLANTAG; + + return (0); +} + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +lem_receive_checksum(int status, int errors, if_rxd_info_t ri) +{ + /* Did it pass? */ + if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE)) + ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); + + if (status & E1000_RXD_STAT_TCPCS) { + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_TCPE)) { + ri->iri_csum_flags |= + (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } + } +} + +static void +em_receive_checksum(uint32_t status, if_rxd_info_t ri) +{ + ri->iri_csum_flags = 0; + + /* Ignore Checksum bit is set */ + if (status & E1000_RXD_STAT_IXSM) + return; + + /* If the IP checksum exists and there is no IP Checksum error */ + if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == + E1000_RXD_STAT_IPCS) { + ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); + } + + /* TCP or UDP checksum */ + if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == + E1000_RXD_STAT_TCPCS) { + ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } + if (status & E1000_RXD_STAT_UDPCS) { + ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); + ri->iri_csum_data = htons(0xffff); + } +} diff --git a/sys/dev/e1000/if_em.c b/sys/dev/e1000/if_em.c index b1bf35b5e464..629ae91025d3 100644 --- a/sys/dev/e1000/if_em.c +++ b/sys/dev/e1000/if_em.c @@ -1,99 +1,36 @@ -/****************************************************************************** +/*- + * Copyright (c) 2016 Matt Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#include "opt_em.h" -#include "opt_ddb.h" -#include "opt_inet.h" -#include "opt_inet6.h" - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include "opt_device_polling.h" -#endif - -#include -#include -#ifdef DDB -#include -#include -#endif -#if __FreeBSD_version >= 800000 -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "e1000_api.h" -#include "e1000_82571.h" +/* $FreeBSD$ */ #include "if_em.h" +#include +#include + +#define em_mac_min e1000_82547 +#define igb_mac_min e1000_82575 /********************************************************************* * Driver version: @@ -110,184 +47,213 @@ char em_driver_version[] = "7.6.1-k"; * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } *********************************************************************/ -static em_vendor_info_t em_vendor_info_array[] = +static pci_vendor_info_t em_vendor_info_array[] = { - /* Intel(R) PRO/1000 Network Connection */ - { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, + /* Intel(R) PRO/1000 Network Connection - Legacy em*/ + PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"), - { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_ICH10_D_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH2_LV_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPT_I217_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_LM3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_I218_V3, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM4, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V4, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_LM5, - PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_PCH_SPT_I219_V5, PCI_ANY_ID, PCI_ANY_ID, 0}, + PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"), + + PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547EI_MOBILE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82547GI, "Intel(R) PRO/1000 Network Connection"), + + /* Intel(R) PRO/1000 Network Connection - em */ + PVID(0x8086, E1000_DEV_ID_82571EB_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_COPPER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_FIBER, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82572EI_SERDES, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573E_IAMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82573L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82583V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH8_82567V_3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_AMT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_C, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IGP_M_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_GT, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_IFE_G, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH9_BM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574L, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_82574LA, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_R_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_LF, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_ICH10_D_BM_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_M_HV_LC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_D_HV_DC, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH2_LV_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPT_I217_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LPTLP_I218_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_LM3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_I218_V3, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_LM2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_SPT_I219_V2, "Intel(R) PRO/1000 Network Connection"), + PVID(0x8086, E1000_DEV_ID_PCH_LBG_I219_LM3, "Intel(R) PRO/1000 Network Connection"), /* required last entry */ - { 0, 0, 0, 0, 0} + PVID_END }; -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *em_strings[] = { - "Intel(R) PRO/1000 Network Connection" +static pci_vendor_info_t igb_vendor_info_array[] = +{ + /* Intel(R) PRO/1000 Network Connection - em */ + PVID(0x8086, E1000_DEV_ID_82575EB_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_NS_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_SERDES_QUAD, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_QUAD_COPPER_ET2, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82576_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_COPPER_DUAL, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_82580_QUAD_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_SFP, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_DH89XXCC_BACKPLANE, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I350_VF, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_IT, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_OEM1, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_COPPER_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES_FLASHLESS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_FIBER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SERDES, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I210_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I211_COPPER, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_1GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, "Intel(R) PRO/1000 PCI-Express Network Driver"), + PVID(0x8086, E1000_DEV_ID_I354_SGMII, "Intel(R) PRO/1000 PCI-Express Network Driver"), + /* required last entry */ + PVID_END }; /********************************************************************* * Function prototypes *********************************************************************/ -static int em_probe(device_t); -static int em_attach(device_t); -static int em_detach(device_t); -static int em_shutdown(device_t); -static int em_suspend(device_t); -static int em_resume(device_t); -#ifdef EM_MULTIQUEUE -static int em_mq_start(if_t, struct mbuf *); -static int em_mq_start_locked(if_t, - struct tx_ring *); -static void em_qflush(if_t); -#else -static void em_start(if_t); -static void em_start_locked(if_t, struct tx_ring *); -#endif -static int em_ioctl(if_t, u_long, caddr_t); -static uint64_t em_get_counter(if_t, ift_counter); -static void em_init(void *); -static void em_init_locked(struct adapter *); -static void em_stop(void *); -static void em_media_status(if_t, struct ifmediareq *); -static int em_media_change(if_t); -static void em_identify_hardware(struct adapter *); -static int em_allocate_pci_resources(struct adapter *); -static int em_allocate_legacy(struct adapter *); -static int em_allocate_msix(struct adapter *); -static int em_allocate_queues(struct adapter *); -static int em_setup_msix(struct adapter *); -static void em_free_pci_resources(struct adapter *); -static void em_local_timer(void *); -static void em_reset(struct adapter *); -static int em_setup_interface(device_t, struct adapter *); -static void em_flush_desc_rings(struct adapter *); +static void *em_register(device_t dev); +static void *igb_register(device_t dev); +static int em_if_attach_pre(if_ctx_t ctx); +static int em_if_attach_post(if_ctx_t ctx); +static int em_if_detach(if_ctx_t ctx); +static int em_if_shutdown(if_ctx_t ctx); +static int em_if_suspend(if_ctx_t ctx); +static int em_if_resume(if_ctx_t ctx); -static void em_setup_transmit_structures(struct adapter *); -static void em_initialize_transmit_unit(struct adapter *); -static int em_allocate_transmit_buffers(struct tx_ring *); -static void em_free_transmit_structures(struct adapter *); -static void em_free_transmit_buffers(struct tx_ring *); +static int em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); +static int em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets); +static void em_if_queues_free(if_ctx_t ctx); -static int em_setup_receive_structures(struct adapter *); -static int em_allocate_receive_buffers(struct rx_ring *); -static void em_initialize_receive_unit(struct adapter *); -static void em_free_receive_structures(struct adapter *); -static void em_free_receive_buffers(struct rx_ring *); +static uint64_t em_if_get_counter(if_ctx_t, ift_counter); +static void em_if_init(if_ctx_t ctx); +static void em_if_stop(if_ctx_t ctx); +static void em_if_media_status(if_ctx_t, struct ifmediareq *); +static int em_if_media_change(if_ctx_t ctx); +static int em_if_mtu_set(if_ctx_t ctx, uint32_t mtu); +static void em_if_timer(if_ctx_t ctx, uint16_t qid); +static void em_if_vlan_register(if_ctx_t ctx, u16 vtag); +static void em_if_vlan_unregister(if_ctx_t ctx, u16 vtag); -static void em_enable_intr(struct adapter *); -static void em_disable_intr(struct adapter *); +static void em_identify_hardware(if_ctx_t ctx); +static int em_allocate_pci_resources(if_ctx_t ctx); +static void em_free_pci_resources(if_ctx_t ctx); +static void em_reset(if_ctx_t ctx); +static int em_setup_interface(if_ctx_t ctx); +static int em_setup_msix(if_ctx_t ctx); + +static void em_initialize_transmit_unit(if_ctx_t ctx); +static void em_initialize_receive_unit(if_ctx_t ctx); + +static void em_if_enable_intr(if_ctx_t ctx); +static void em_if_disable_intr(if_ctx_t ctx); +static int em_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); +static void em_if_multi_set(if_ctx_t ctx); +static void em_if_update_admin_status(if_ctx_t ctx); static void em_update_stats_counters(struct adapter *); static void em_add_hw_stats(struct adapter *adapter); -static void em_txeof(struct tx_ring *); -static bool em_rxeof(struct rx_ring *, int, int *); -#ifndef __NO_STRICT_ALIGNMENT -static int em_fixup_rx(struct rx_ring *); -#endif -static void em_setup_rxdesc(union e1000_rx_desc_extended *, - const struct em_rxbuffer *rxbuf); -static void em_receive_checksum(uint32_t status, struct mbuf *); -static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, int, - struct ip *, u32 *, u32 *); -static void em_tso_setup(struct tx_ring *, struct mbuf *, int, struct ip *, - struct tcphdr *, u32 *, u32 *); -static void em_set_promisc(struct adapter *); -static void em_disable_promisc(struct adapter *); -static void em_set_multi(struct adapter *); -static void em_update_link_status(struct adapter *); -static void em_refresh_mbufs(struct rx_ring *, int); -static void em_register_vlan(void *, if_t, u16); -static void em_unregister_vlan(void *, if_t, u16); +static int em_if_set_promisc(if_ctx_t ctx, int flags); static void em_setup_vlan_hw_support(struct adapter *); -static int em_xmit(struct tx_ring *, struct mbuf **); -static int em_dma_malloc(struct adapter *, bus_size_t, - struct em_dma_alloc *, int); -static void em_dma_free(struct adapter *, struct em_dma_alloc *); static int em_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); static void em_print_nvm_info(struct adapter *); static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); @@ -301,63 +267,122 @@ static void em_init_manageability(struct adapter *); static void em_release_manageability(struct adapter *); static void em_get_hw_control(struct adapter *); static void em_release_hw_control(struct adapter *); -static void em_get_wakeup(device_t); -static void em_enable_wakeup(device_t); +static void em_get_wakeup(if_ctx_t ctx); +static void em_enable_wakeup(if_ctx_t ctx); static int em_enable_phy_wakeup(struct adapter *); -static void em_led_func(void *, int); static void em_disable_aspm(struct adapter *); -static int em_irq_fast(void *); +int em_intr(void *arg); +static void em_disable_promisc(if_ctx_t ctx); /* MSIX handlers */ -static void em_msix_tx(void *); -static void em_msix_rx(void *); -static void em_msix_link(void *); -static void em_handle_tx(void *context, int pending); -static void em_handle_rx(void *context, int pending); -static void em_handle_link(void *context, int pending); +static int em_if_msix_intr_assign(if_ctx_t, int); +static int em_msix_link(void *); +static void em_handle_link(void *context); -#ifdef EM_MULTIQUEUE -static void em_enable_vectors_82574(struct adapter *); -#endif +static void em_enable_vectors_82574(if_ctx_t); static void em_set_sysctl_value(struct adapter *, const char *, const char *, int *, int); static int em_set_flowcntl(SYSCTL_HANDLER_ARGS); static int em_sysctl_eee(SYSCTL_HANDLER_ARGS); +static void em_if_led_func(if_ctx_t ctx, int onoff); -static __inline void em_rx_discard(struct rx_ring *, int); +static void em_init_tx_ring(struct em_tx_queue *que); +static int em_get_regs(SYSCTL_HANDLER_ARGS); + +static void lem_smartspeed(struct adapter *adapter); +static void igb_configure_queues(struct adapter *adapter); -#ifdef DEVICE_POLLING -static poll_handler_t em_poll; -#endif /* POLLING */ /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ - static device_method_t em_methods[] = { /* Device interface */ - DEVMETHOD(device_probe, em_probe), - DEVMETHOD(device_attach, em_attach), - DEVMETHOD(device_detach, em_detach), - DEVMETHOD(device_shutdown, em_shutdown), - DEVMETHOD(device_suspend, em_suspend), - DEVMETHOD(device_resume, em_resume), - DEVMETHOD_END + DEVMETHOD(device_register, em_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), + DEVMETHOD_END }; +static device_method_t igb_methods[] = { + /* Device interface */ + DEVMETHOD(device_register, igb_register), + DEVMETHOD(device_probe, iflib_device_probe), + DEVMETHOD(device_attach, iflib_device_attach), + DEVMETHOD(device_detach, iflib_device_detach), + DEVMETHOD(device_shutdown, iflib_device_shutdown), + DEVMETHOD(device_suspend, iflib_device_suspend), + DEVMETHOD(device_resume, iflib_device_resume), + DEVMETHOD_END +}; + + static driver_t em_driver = { "em", em_methods, sizeof(struct adapter), }; -devclass_t em_devclass; +static devclass_t em_devclass; DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); + MODULE_DEPEND(em, pci, 1, 1, 1); MODULE_DEPEND(em, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(em, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ +MODULE_DEPEND(em, iflib, 1, 1, 1); + +static driver_t igb_driver = { + "igb", igb_methods, sizeof(struct adapter), +}; + +static devclass_t igb_devclass; +DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); + +MODULE_DEPEND(igb, pci, 1, 1, 1); +MODULE_DEPEND(igb, ether, 1, 1, 1); +MODULE_DEPEND(igb, iflib, 1, 1, 1); + + +static device_method_t em_if_methods[] = { + DEVMETHOD(ifdi_attach_pre, em_if_attach_pre), + DEVMETHOD(ifdi_attach_post, em_if_attach_post), + DEVMETHOD(ifdi_detach, em_if_detach), + DEVMETHOD(ifdi_shutdown, em_if_shutdown), + DEVMETHOD(ifdi_suspend, em_if_suspend), + DEVMETHOD(ifdi_resume, em_if_resume), + DEVMETHOD(ifdi_init, em_if_init), + DEVMETHOD(ifdi_stop, em_if_stop), + DEVMETHOD(ifdi_msix_intr_assign, em_if_msix_intr_assign), + DEVMETHOD(ifdi_intr_enable, em_if_enable_intr), + DEVMETHOD(ifdi_intr_disable, em_if_disable_intr), + DEVMETHOD(ifdi_tx_queues_alloc, em_if_tx_queues_alloc), + DEVMETHOD(ifdi_rx_queues_alloc, em_if_rx_queues_alloc), + DEVMETHOD(ifdi_queues_free, em_if_queues_free), + DEVMETHOD(ifdi_update_admin_status, em_if_update_admin_status), + DEVMETHOD(ifdi_multi_set, em_if_multi_set), + DEVMETHOD(ifdi_media_status, em_if_media_status), + DEVMETHOD(ifdi_media_change, em_if_media_change), + DEVMETHOD(ifdi_mtu_set, em_if_mtu_set), + DEVMETHOD(ifdi_promisc_set, em_if_set_promisc), + DEVMETHOD(ifdi_timer, em_if_timer), + DEVMETHOD(ifdi_vlan_register, em_if_vlan_register), + DEVMETHOD(ifdi_vlan_unregister, em_if_vlan_unregister), + DEVMETHOD(ifdi_get_counter, em_if_get_counter), + DEVMETHOD(ifdi_led_func, em_if_led_func), + DEVMETHOD(ifdi_queue_intr_enable, em_if_queue_intr_enable), + DEVMETHOD_END +}; + + /* + * note that if (adapter->msix_mem) is replaced by: + * if (adapter->intr_type == IFLIB_INTR_MSIX) + */ +static driver_t em_if_driver = { + "em_if", em_if_methods, sizeof(struct adapter) +}; /********************************************************************* * Tunable default values. @@ -365,10 +390,16 @@ MODULE_DEPEND(em, netmap, 1, 1, 1); #define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) #define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) +#define M_TSO_LEN 66 #define MAX_INTS_PER_SEC 8000 #define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256)) +/* Allow common code without TSO */ +#ifndef CSUM_TSO +#define CSUM_TSO 0 +#endif + #define TSO_WORKAROUND 4 static SYSCTL_NODE(_hw, OID_AUTO, em, CTLFLAG_RD, 0, "EM driver parameters"); @@ -393,39 +424,15 @@ SYSCTL_INT(_hw_em, OID_AUTO, rx_abs_int_delay, CTLFLAG_RDTUN, &em_rx_abs_int_delay_dflt, 0, "Default receive interrupt delay limit in usecs"); -static int em_rxd = EM_DEFAULT_RXD; -static int em_txd = EM_DEFAULT_TXD; -SYSCTL_INT(_hw_em, OID_AUTO, rxd, CTLFLAG_RDTUN, &em_rxd, 0, - "Number of receive descriptors per queue"); -SYSCTL_INT(_hw_em, OID_AUTO, txd, CTLFLAG_RDTUN, &em_txd, 0, - "Number of transmit descriptors per queue"); - static int em_smart_pwr_down = FALSE; SYSCTL_INT(_hw_em, OID_AUTO, smart_pwr_down, CTLFLAG_RDTUN, &em_smart_pwr_down, 0, "Set to true to leave smart power down enabled on newer adapters"); /* Controls whether promiscuous also shows bad packets */ -static int em_debug_sbp = FALSE; +static int em_debug_sbp = TRUE; SYSCTL_INT(_hw_em, OID_AUTO, sbp, CTLFLAG_RDTUN, &em_debug_sbp, 0, "Show bad packets in promiscuous mode"); -static int em_enable_msix = TRUE; -SYSCTL_INT(_hw_em, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &em_enable_msix, 0, - "Enable MSI-X interrupts"); - -#ifdef EM_MULTIQUEUE -static int em_num_queues = 1; -SYSCTL_INT(_hw_em, OID_AUTO, num_queues, CTLFLAG_RDTUN, &em_num_queues, 0, - "82574 only: Number of queues to configure, 0 indicates autoconfigure"); -#endif - -/* -** Global variable to store last used CPU when binding queues -** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a -** queue is bound to a cpu. -*/ -static int em_last_bind_cpu = -1; - /* How many packets rxeof tries to clean at a time */ static int em_rx_process_limit = 100; SYSCTL_INT(_hw_em, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, @@ -438,64 +445,258 @@ static int eee_setting = 1; SYSCTL_INT(_hw_em, OID_AUTO, eee_setting, CTLFLAG_RDTUN, &eee_setting, 0, "Enable Energy Efficient Ethernet"); +/* +** Tuneable Interrupt rate +*/ +static int em_max_interrupt_rate = 8000; +SYSCTL_INT(_hw_em, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, + &em_max_interrupt_rate, 0, "Maximum interrupts per second"); + + + /* Global used in WOL setup with multiport cards */ static int global_quad_port_a = 0; -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include -#endif /* DEV_NETMAP */ +extern struct if_txrx igb_txrx; +extern struct if_txrx em_txrx; +extern struct if_txrx lem_txrx; -/********************************************************************* - * Device identification routine - * - * em_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ +static struct if_shared_ctx em_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = em_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, -static int -em_probe(device_t dev) + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; + +if_shared_ctx_t em_sctx = &em_sctx_init; + + +static struct if_shared_ctx igb_sctx_init = { + .isc_magic = IFLIB_MAGIC, + .isc_q_align = PAGE_SIZE, + .isc_tx_maxsize = EM_TSO_SIZE, + .isc_tx_maxsegsize = PAGE_SIZE, + .isc_rx_maxsize = MJUM9BYTES, + .isc_rx_nsegments = 1, + .isc_rx_maxsegsize = MJUM9BYTES, + .isc_nfl = 1, + .isc_nrxqs = 1, + .isc_ntxqs = 1, + .isc_admin_intrcnt = 1, + .isc_vendor_info = igb_vendor_info_array, + .isc_driver_version = em_driver_version, + .isc_driver = &em_if_driver, + .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_TSO_INIT_IP, + + .isc_nrxd_min = {EM_MIN_RXD}, + .isc_ntxd_min = {EM_MIN_TXD}, + .isc_nrxd_max = {EM_MAX_RXD}, + .isc_ntxd_max = {EM_MAX_TXD}, + .isc_nrxd_default = {EM_DEFAULT_RXD}, + .isc_ntxd_default = {EM_DEFAULT_TXD}, +}; + +if_shared_ctx_t igb_sctx = &igb_sctx_init; + +/***************************************************************** + * + * Dump Registers + * + ****************************************************************/ +#define IGB_REGS_LEN 739 + +static int em_get_regs(SYSCTL_HANDLER_ARGS) { - char adapter_name[60]; - uint16_t pci_vendor_id = 0; - uint16_t pci_device_id = 0; - uint16_t pci_subvendor_id = 0; - uint16_t pci_subdevice_id = 0; - em_vendor_info_t *ent; + struct adapter *adapter = (struct adapter *)arg1; + struct e1000_hw *hw = &adapter->hw; - INIT_DEBUGOUT("em_probe: begin"); + struct sbuf *sb; + u32 *regs_buff = (u32 *)malloc(sizeof(u32) * IGB_REGS_LEN, M_DEVBUF, M_NOWAIT); + int rc; - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != EM_VENDOR_ID) - return (ENXIO); + memset(regs_buff, 0, IGB_REGS_LEN * sizeof(u32)); - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); + rc = sysctl_wire_old_buffer(req, 0); + MPASS(rc == 0); + if (rc != 0) + return (rc); - ent = em_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && + sb = sbuf_new_for_sysctl(NULL, NULL, 32*400, req); + MPASS(sb != NULL); + if (sb == NULL) + return (ENOMEM); - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && + /* General Registers */ + regs_buff[0] = E1000_READ_REG(hw, E1000_CTRL); + regs_buff[1] = E1000_READ_REG(hw, E1000_STATUS); + regs_buff[2] = E1000_READ_REG(hw, E1000_CTRL_EXT); + regs_buff[3] = E1000_READ_REG(hw, E1000_ICR); + regs_buff[4] = E1000_READ_REG(hw, E1000_RCTL); + regs_buff[5] = E1000_READ_REG(hw, E1000_RDLEN(0)); + regs_buff[6] = E1000_READ_REG(hw, E1000_RDH(0)); + regs_buff[7] = E1000_READ_REG(hw, E1000_RDT(0)); + regs_buff[8] = E1000_READ_REG(hw, E1000_RXDCTL(0)); + regs_buff[9] = E1000_READ_REG(hw, E1000_RDBAL(0)); + regs_buff[10] = E1000_READ_REG(hw, E1000_RDBAH(0)); + regs_buff[11] = E1000_READ_REG(hw, E1000_TCTL); + regs_buff[12] = E1000_READ_REG(hw, E1000_TDBAL(0)); + regs_buff[13] = E1000_READ_REG(hw, E1000_TDBAH(0)); + regs_buff[14] = E1000_READ_REG(hw, E1000_TDLEN(0)); + regs_buff[15] = E1000_READ_REG(hw, E1000_TDH(0)); + regs_buff[16] = E1000_READ_REG(hw, E1000_TDT(0)); + regs_buff[17] = E1000_READ_REG(hw, E1000_TXDCTL(0)); + regs_buff[18] = E1000_READ_REG(hw, E1000_TDFH); + regs_buff[19] = E1000_READ_REG(hw, E1000_TDFT); + regs_buff[20] = E1000_READ_REG(hw, E1000_TDFHS); + regs_buff[21] = E1000_READ_REG(hw, E1000_TDFPC); + + sbuf_printf(sb, "General Registers\n"); + sbuf_printf(sb, "\tCTRL\t %08x\n", regs_buff[0]); + sbuf_printf(sb, "\tSTATUS\t %08x\n", regs_buff[1]); + sbuf_printf(sb, "\tCTRL_EXIT\t %08x\n\n", regs_buff[2]); - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", - em_strings[ent->index], - em_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; + sbuf_printf(sb, "Interrupt Registers\n"); + sbuf_printf(sb, "\tICR\t %08x\n\n", regs_buff[3]); + + sbuf_printf(sb, "RX Registers\n"); + sbuf_printf(sb, "\tRCTL\t %08x\n", regs_buff[4]); + sbuf_printf(sb, "\tRDLEN\t %08x\n", regs_buff[5]); + sbuf_printf(sb, "\tRDH\t %08x\n", regs_buff[6]); + sbuf_printf(sb, "\tRDT\t %08x\n", regs_buff[7]); + sbuf_printf(sb, "\tRXDCTL\t %08x\n", regs_buff[8]); + sbuf_printf(sb, "\tRDBAL\t %08x\n", regs_buff[9]); + sbuf_printf(sb, "\tRDBAH\t %08x\n\n", regs_buff[10]); + + sbuf_printf(sb, "TX Registers\n"); + sbuf_printf(sb, "\tTCTL\t %08x\n", regs_buff[11]); + sbuf_printf(sb, "\tTDBAL\t %08x\n", regs_buff[12]); + sbuf_printf(sb, "\tTDBAH\t %08x\n", regs_buff[13]); + sbuf_printf(sb, "\tTDLEN\t %08x\n", regs_buff[14]); + sbuf_printf(sb, "\tTDH\t %08x\n", regs_buff[15]); + sbuf_printf(sb, "\tTDT\t %08x\n", regs_buff[16]); + sbuf_printf(sb, "\tTXDCTL\t %08x\n", regs_buff[17]); + sbuf_printf(sb, "\tTDFH\t %08x\n", regs_buff[18]); + sbuf_printf(sb, "\tTDFT\t %08x\n", regs_buff[19]); + sbuf_printf(sb, "\tTDFHS\t %08x\n", regs_buff[20]); + sbuf_printf(sb, "\tTDFPC\t %08x\n\n", regs_buff[21]); + +#ifdef DUMP_DESCS + { + if_softc_ctx_t scctx = adapter->shared; + struct rx_ring *rxr = &rx_que->rxr; + struct tx_ring *txr = &tx_que->txr; + int ntxd = scctx->isc_ntxd[0]; + int nrxd = scctx->isc_nrxd[0]; + int j; + + for (j = 0; j < nrxd; j++) { + u32 staterr = le32toh(rxr->rx_base[j].wb.upper.status_error); + u32 length = le32toh(rxr->rx_base[j].wb.upper.length); + sbuf_printf(sb, "\tReceive Descriptor Address %d: %08" PRIx64 " Error:%d Length:%d\n", j, rxr->rx_base[j].read.buffer_addr, staterr, length); } - return (ENXIO); + for (j = 0; j < min(ntxd, 256); j++) { + struct em_txbuffer *buf = &txr->tx_buffers[j]; + unsigned int *ptr = (unsigned int *)&txr->tx_base[j]; + + sbuf_printf(sb, "\tTXD[%03d] [0]: %08x [1]: %08x [2]: %08x [3]: %08x eop: %d DD=%d\n", + j, ptr[0], ptr[1], ptr[2], ptr[3], buf->eop, + buf->eop != -1 ? txr->tx_base[buf->eop].upper.fields.status & E1000_TXD_STAT_DD : 0); + + } + } +#endif + + rc = sbuf_finish(sb); + sbuf_delete(sb); + return(rc); } +static void * +em_register(device_t dev) +{ + return (em_sctx); +} + +static void * +igb_register(device_t dev) +{ + return (igb_sctx); +} + +static void +em_init_tx_ring(struct em_tx_queue *que) +{ + struct adapter *sc = que->adapter; + if_softc_ctx_t scctx = sc->shared; + struct tx_ring *txr = &que->txr; + struct em_txbuffer *tx_buffer; + + tx_buffer = txr->tx_buffers; + for (int i = 0; i < scctx->isc_ntxd[0]; i++, tx_buffer++) { + tx_buffer->eop = -1; + } +} + +static int +em_set_num_queues(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + int maxqueues; + + /* Sanity check based on HW */ + switch (adapter->hw.mac.type) { + case e1000_82576: + case e1000_82580: + case e1000_i350: + case e1000_i354: + maxqueues = 8; + break; + case e1000_i210: + case e1000_82575: + maxqueues = 4; + break; + case e1000_i211: + case e1000_82574: + maxqueues = 2; + break; + default: + maxqueues = 1; + break; + } + + return (maxqueues); +} + + +#define EM_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | \ + IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; + +#define IGB_CAPS \ + IFCAP_TSO4 | IFCAP_TXCSUM | IFCAP_LRO | IFCAP_RXCSUM | IFCAP_VLAN_HWFILTER | IFCAP_WOL_MAGIC | \ + IFCAP_WOL_MCAST | IFCAP_WOL | IFCAP_VLAN_HWTSO | IFCAP_HWCSUM | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ + IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU | IFCAP_TXCSUM_IPV6 | IFCAP_HWCSUM_IPV6 | IFCAP_JUMBO_MTU; + /********************************************************************* * Device initialization routine * @@ -507,23 +708,30 @@ em_probe(device_t dev) *********************************************************************/ static int -em_attach(device_t dev) +em_if_attach_pre(if_ctx_t ctx) { - struct adapter *adapter; + struct adapter *adapter; + if_softc_ctx_t scctx; + device_t dev; struct e1000_hw *hw; int error = 0; - INIT_DEBUGOUT("em_attach: begin"); + INIT_DEBUGOUT("em_if_attach_pre begin"); + dev = iflib_get_dev(ctx); + adapter = iflib_get_softc(ctx); if (resource_disabled("em", device_get_unit(dev))) { device_printf(dev, "Disabled by device hint\n"); return (ENXIO); } - adapter = device_get_softc(dev); + adapter->ctx = ctx; adapter->dev = adapter->osdep.dev = dev; - hw = &adapter->hw; - EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); + scctx = adapter->shared = iflib_get_softc_ctx(ctx); + adapter->media = iflib_get_media(ctx); + hw = &adapter->hw; + + adapter->tx_process_limit = scctx->isc_ntxd[0]; /* SYSCTL stuff */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), @@ -541,13 +749,56 @@ em_attach(device_t dev) OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, em_set_flowcntl, "I", "Flow Control"); - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); + SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "reg_dump", CTLTYPE_STRING | CTLFLAG_RD, adapter, 0, + em_get_regs, "A", "Dump Registers"); /* Determine hardware and mac info */ - em_identify_hardware(adapter); + em_identify_hardware(ctx); + + /* Set isc_msix_bar */ + scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR); + scctx->isc_tx_nsegments = EM_MAX_SCATTER; + scctx->isc_tx_tso_segments_max = scctx->isc_tx_nsegments; + scctx->isc_tx_tso_size_max = EM_TSO_SIZE; + scctx->isc_tx_tso_segsize_max = EM_TSO_SEG_SIZE; + scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx); + device_printf(dev, "attach_pre capping queues at %d\n", scctx->isc_ntxqsets_max); + + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + + + if (adapter->hw.mac.type >= igb_mac_min) { + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(union e1000_adv_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_adv_rx_desc), EM_DBA_ALIGN); + scctx->isc_txrx = &igb_txrx; + scctx->isc_capenable = IGB_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_TSO | CSUM_IP6_TCP \ + | CSUM_IP6_UDP | CSUM_IP6_TCP; + if (adapter->hw.mac.type != e1000_82575) + scctx->isc_tx_csum_flags |= CSUM_SCTP | CSUM_IP6_SCTP; + + } else if (adapter->hw.mac.type >= em_mac_min) { + scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0]* sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); + scctx->isc_txrx = &em_txrx; + scctx->isc_capenable = EM_CAPS; + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + } else { + scctx->isc_txqsizes[0] = roundup2((scctx->isc_ntxd[0] + 1) * sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); + scctx->isc_rxqsizes[0] = roundup2((scctx->isc_nrxd[0] + 1) * sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_txrx = &lem_txrx; + scctx->isc_capenable = EM_CAPS; + if (adapter->hw.mac.type < e1000_82543) + scctx->isc_capenable &= ~(IFCAP_HWCSUM|IFCAP_VLAN_HWCSUM); + scctx->isc_tx_csum_flags = CSUM_TCP | CSUM_UDP | CSUM_IP_TSO; + scctx->isc_msix_bar = 0; + } /* Setup PCI resources */ - if (em_allocate_pci_resources(adapter)) { + if (em_allocate_pci_resources(ctx)) { device_printf(dev, "Allocation of PCI resources failed\n"); error = ENXIO; goto err_pci; @@ -603,11 +854,7 @@ em_attach(device_t dev) goto err_pci; } - /* - * Setup MSI/X or MSI if PCI Express - */ - adapter->msix = em_setup_msix(adapter); - + em_setup_msix(ctx); e1000_get_bus_info(hw); /* Set up some sysctls for the tunable interrupt delays */ @@ -637,32 +884,15 @@ em_attach(device_t dev) em_set_sysctl_value(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, em_rx_process_limit); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || - (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - EM_DEFAULT_TXD, em_txd); - adapter->num_tx_desc = EM_DEFAULT_TXD; - } else - adapter->num_tx_desc = em_txd; - - if (((em_rxd * sizeof(union e1000_rx_desc_extended)) % EM_DBA_ALIGN) != 0 || - (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - EM_DEFAULT_RXD, em_rxd); - adapter->num_rx_desc = EM_DEFAULT_RXD; - } else - adapter->num_rx_desc = em_rxd; - + hw->mac.autoneg = DO_AUTO_NEG; hw->phy.autoneg_wait_to_complete = FALSE; hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; + if (adapter->hw.mac.type < em_mac_min) { + e1000_init_script_state_82541(&adapter->hw, TRUE); + e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); + } /* Copper options */ if (hw->phy.media_type == e1000_media_type_copper) { hw->phy.mdix = AUTO_ALL_MODES; @@ -683,14 +913,6 @@ em_attach(device_t dev) */ hw->mac.report_tx_early = 1; - /* - ** Get queue/ring memory - */ - if (em_allocate_queues(adapter)) { - error = ENOMEM; - goto err_pci; - } - /* Allocate multicast array memory. */ adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); @@ -720,7 +942,6 @@ em_attach(device_t dev) */ e1000_reset_hw(hw); - /* Make sure we have a good EEPROM before we read from it */ if (e1000_validate_nvm_checksum(hw) < 0) { /* @@ -753,68 +974,58 @@ em_attach(device_t dev) /* Disable ULP support */ e1000_disable_ulp_lpt_lp(hw, TRUE); - /* - ** Do interrupt configuration - */ - if (adapter->msix > 1) /* Do MSIX */ - error = em_allocate_msix(adapter); - else /* MSI or Legacy */ - error = em_allocate_legacy(adapter); - if (error) - goto err_late; - - /* + /* * Get Wake-on-Lan and Management info for later use */ - em_get_wakeup(dev); + em_get_wakeup(ctx); + iflib_set_mac(ctx, hw->mac.addr); + + return (0); + +err_late: + em_release_hw_control(adapter); +err_pci: + em_free_pci_resources(ctx); + free(adapter->mta, M_DEVBUF); + + return (error); +} + +static int +em_if_attach_post(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct e1000_hw *hw = &adapter->hw; + int error = 0; + /* Setup OS specific network interface */ - if (em_setup_interface(dev, adapter) != 0) + error = em_setup_interface(ctx); + if (error != 0) { goto err_late; + } - em_reset(adapter); + em_reset(ctx); /* Initialize statistics */ em_update_stats_counters(adapter); - hw->mac.get_link_status = 1; - em_update_link_status(adapter); - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - + em_if_update_admin_status(ctx); em_add_hw_stats(adapter); /* Non-AMT based hardware can now take control from firmware */ if (adapter->has_manage && !adapter->has_amt) em_get_hw_control(adapter); + + INIT_DEBUGOUT("em_if_attach_post: end"); - /* Tell the stack that the interface is not active */ - if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - adapter->led_dev = led_create(em_led_func, adapter, - device_get_nameunit(dev)); -#ifdef DEV_NETMAP - em_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - - INIT_DEBUGOUT("em_attach: end"); - - return (0); + return (error); err_late: - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); em_release_hw_control(adapter); - if (adapter->ifp != (void *)NULL) - if_free(adapter->ifp); -err_pci: - em_free_pci_resources(adapter); + em_free_pci_resources(ctx); + em_if_queues_free(ctx); free(adapter->mta, M_DEVBUF); - EM_CORE_LOCK_DESTROY(adapter); return (error); } @@ -830,60 +1041,17 @@ em_attach(device_t dev) *********************************************************************/ static int -em_detach(device_t dev) +em_if_detach(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_detach: begin"); - /* Make sure VLANS are not using driver */ - if (if_vlantrunkinuse(ifp)) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef DEVICE_POLLING - if (if_getcapenable(ifp) & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - - EM_CORE_LOCK(adapter); - adapter->in_detach = 1; - em_stop(adapter); - EM_CORE_UNLOCK(adapter); - EM_CORE_LOCK_DESTROY(adapter); - e1000_phy_hw_reset(&adapter->hw); em_release_manageability(adapter); em_release_hw_control(adapter); - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - ether_ifdetach(adapter->ifp); - callout_drain(&adapter->timer); - -#ifdef DEV_NETMAP - netmap_detach(ifp); -#endif /* DEV_NETMAP */ - - em_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - em_free_transmit_structures(adapter); - em_free_receive_structures(adapter); - - em_release_hw_control(adapter); - free(adapter->mta, M_DEVBUF); + em_free_pci_resources(ctx); return (0); } @@ -895,448 +1063,78 @@ em_detach(device_t dev) **********************************************************************/ static int -em_shutdown(device_t dev) +em_if_shutdown(if_ctx_t ctx) { - return em_suspend(dev); + return em_if_suspend(ctx); } /* * Suspend/resume device methods. */ static int -em_suspend(device_t dev) +em_if_suspend(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - - EM_CORE_LOCK(adapter); + struct adapter *adapter = iflib_get_softc(ctx); em_release_manageability(adapter); em_release_hw_control(adapter); - em_enable_wakeup(dev); - - EM_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); -} - -static int -em_resume(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; - - EM_CORE_LOCK(adapter); - if (adapter->hw.mac.type == e1000_pch2lan) - e1000_resume_workarounds_pchlan(&adapter->hw); - em_init_locked(adapter); - em_init_manageability(adapter); - - if ((if_getflags(ifp) & IFF_UP) && - (if_getdrvflags(ifp) & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); - - return bus_generic_resume(dev); -} - - -#ifndef EM_MULTIQUEUE -static void -em_start_locked(if_t ifp, struct tx_ring *txr) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(txr); - - if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - - if (!adapter->link_active) - return; - - while (!if_sendq_empty(ifp)) { - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp,IFF_DRV_OACTIVE, 0); - break; - } - m_head = if_dequeue(ifp); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (em_xmit(txr, &m_head)) { - if (m_head == NULL) - break; - if_sendq_prepend(ifp, m_head); - break; - } - - /* Mark the queue as having work */ - if (txr->busy == EM_TX_IDLE) - txr->busy = EM_TX_BUSY; - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - } - - return; -} - -static void -em_start(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_TX_LOCK(txr); - em_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } - return; -} -#else /* EM_MULTIQUEUE */ -/********************************************************************* - * Multiqueue Transmit routines - * - * em_mq_start is called by the stack to initiate a transmit. - * however, if busy the driver can queue the request rather - * than do an immediate send. It is this that is an advantage - * in this driver, rather than also having multiple tx queues. - **********************************************************************/ -/* -** Multiqueue capable stack interface -*/ -static int -em_mq_start(if_t ifp, struct mbuf *m) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - unsigned int i, error; - - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) - i = m->m_pkthdr.flowid % adapter->num_queues; - else - i = curcpu % adapter->num_queues; - - txr = &adapter->tx_rings[i]; - - error = drbr_enqueue(ifp, txr->br, m); - if (error) - return (error); - - if (EM_TX_TRYLOCK(txr)) { - em_mq_start_locked(ifp, txr); - EM_TX_UNLOCK(txr); - } else - taskqueue_enqueue(txr->tq, &txr->tx_task); - + em_enable_wakeup(ctx); return (0); } static int -em_mq_start_locked(if_t ifp, struct tx_ring *txr) +em_if_resume(if_ctx_t ctx) { - struct adapter *adapter = txr->adapter; - struct mbuf *next; - int err = 0, enq = 0; + struct adapter *adapter = iflib_get_softc(ctx); - EM_TX_LOCK_ASSERT(txr); + if (adapter->hw.mac.type == e1000_pch2lan) + e1000_resume_workarounds_pchlan(&adapter->hw); + em_if_init(ctx); + em_init_manageability(adapter); - if (((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) || - adapter->link_active == 0) { - return (ENETDOWN); - } - - /* Process the queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = em_xmit(txr, &next)) != 0) { - if (next == NULL) { - /* It was freed, move forward */ - drbr_advance(ifp, txr->br); - } else { - /* - * Still have one left, it may not be - * the same since the transmit function - * may have changed it. - */ - drbr_putback(ifp, txr->br, next); - } - break; - } - drbr_advance(ifp, txr->br); - enq++; - if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); - if (next->m_flags & M_MCAST) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - ETHER_BPF_MTAP(ifp, next); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - } - - /* Mark the queue as having work */ - if ((enq > 0) && (txr->busy == EM_TX_IDLE)) - txr->busy = EM_TX_BUSY; - - if (txr->tx_avail < EM_MAX_SCATTER) - em_txeof(txr); - if (txr->tx_avail < EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE,0); - } - return (err); + return(0); } -/* -** Flush all ring buffers -*/ -static void -em_qflush(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - EM_TX_UNLOCK(txr); - } - if_qflush(ifp); -} -#endif /* EM_MULTIQUEUE */ - -/********************************************************************* - * Ioctl entry point - * - * em_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - static int -em_ioctl(if_t ifp, u_long command, caddr_t data) +em_if_mtu_set(if_ctx_t ctx, uint32_t mtu) { - struct adapter *adapter = if_getsoftc(ifp); - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - if_setflagbits(ifp,IFF_UP,0); - if (!(if_getdrvflags(ifp)& IFF_DRV_RUNNING)) - em_init(adapter); -#ifdef INET - if (!(if_getflags(ifp) & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - EM_CORE_LOCK(adapter); - switch (adapter->hw.mac.type) { - case e1000_82571: - case e1000_82572: - case e1000_ich9lan: - case e1000_ich10lan: - case e1000_pch2lan: - case e1000_pch_lpt: - case e1000_pch_spt: - case e1000_82574: - case e1000_82583: - case e1000_80003es2lan: /* 9K Jumbo Frame size */ - max_frame_size = 9234; - break; - case e1000_pchlan: - max_frame_size = 4096; - break; - /* Adapters that do not support jumbo frames */ - case e1000_ich8lan: - max_frame_size = ETHER_MAX_LEN; - break; - default: - max_frame_size = MAX_JUMBO_FRAME_SIZE; - } - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - EM_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - if_setmtu(ifp, ifr->ifr_mtu); - adapter->hw.mac.max_frame_size = - if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - EM_CORE_LOCK(adapter); - if (if_getflags(ifp) & IFF_UP) { - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - if ((if_getflags(ifp) ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - em_disable_promisc(adapter); - em_set_promisc(adapter); - } - } else - em_init_locked(adapter); - } else - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - em_stop(adapter); - adapter->if_flags = if_getflags(ifp); - EM_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - em_set_multi(adapter); -#ifdef DEVICE_POLLING - if (!(if_getcapenable(ifp) & IFCAP_POLLING)) -#endif - em_enable_intr(adapter); - EM_CORE_UNLOCK(adapter); - } - break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - EM_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - EM_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - EM_CORE_UNLOCK(adapter); - /* falls thru */ - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); - break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(em_poll, ifp); - if (error) - return (error); - EM_CORE_LOCK(adapter); - em_disable_intr(adapter); - if_setcapenablebit(ifp, IFCAP_POLLING, 0); - EM_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - EM_CORE_LOCK(adapter); - em_enable_intr(adapter); - if_setcapenablebit(ifp, 0, IFCAP_POLLING); - EM_CORE_UNLOCK(adapter); - } - } -#endif - if (mask & IFCAP_HWCSUM) { - if_togglecapenable(ifp,IFCAP_HWCSUM); - reinit = 1; - } - if (mask & IFCAP_TSO4) { - if_togglecapenable(ifp,IFCAP_TSO4); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - if_togglecapenable(ifp,IFCAP_VLAN_HWTAGGING); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWFILTER) { - if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTSO) { - if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); - reinit = 1; - } - if ((mask & IFCAP_WOL) && - (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { - if (mask & IFCAP_WOL_MCAST) - if_togglecapenable(ifp, IFCAP_WOL_MCAST); - if (mask & IFCAP_WOL_MAGIC) - if_togglecapenable(ifp, IFCAP_WOL_MAGIC); - } - if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - em_init(adapter); - if_vlancap(ifp); - break; - } - - default: - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); + int max_frame_size; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); + + IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); + + switch (adapter->hw.mac.type) { + case e1000_82571: + case e1000_82572: + case e1000_ich9lan: + case e1000_ich10lan: + case e1000_pch2lan: + case e1000_pch_lpt: + case e1000_pch_spt: + case e1000_82574: + case e1000_82583: + case e1000_80003es2lan: /* 9K Jumbo Frame size */ + max_frame_size = 9234; + break; + case e1000_pchlan: + max_frame_size = 4096; + break; + /* Adapters that do not support jumbo frames */ + case e1000_ich8lan: + max_frame_size = ETHER_MAX_LEN; + break; + default: + max_frame_size = MAX_JUMBO_FRAME_SIZE; + } + if (mtu > max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) { + return (EINVAL); + } + + adapter->hw.mac.max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + return (0); } - /********************************************************************* * Init entry point * @@ -1349,20 +1147,15 @@ em_ioctl(if_t ifp, u_long command, caddr_t data) **********************************************************************/ static void -em_init_locked(struct adapter *adapter) +em_if_init(if_ctx_t ctx) { - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); - INIT_DEBUGOUT("em_init: begin"); - - EM_CORE_LOCK_ASSERT(adapter); - - em_disable_intr(adapter); - callout_stop(&adapter->timer); + INIT_DEBUGOUT("em_if_init: begin"); /* Get the latest mac address, User can use a LAA */ - bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, + bcopy(if_getlladdr(ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN); /* Put the address into the Receive Address Array */ @@ -1381,29 +1174,24 @@ em_init_locked(struct adapter *adapter) } /* Initialize the hardware */ - em_reset(adapter); - em_update_link_status(adapter); + em_reset(ctx); + em_if_update_admin_status(ctx); /* Setup VLAN support, basic and offload if available */ E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - /* Set hardware offload abilities */ - if_clearhwassist(ifp); - if (if_getcapenable(ifp) & IFCAP_TXCSUM) - if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); - - if (if_getcapenable(ifp) & IFCAP_TSO4) - if_sethwassistbits(ifp, CSUM_TSO, 0); + /* Clear bad data from Rx FIFOs */ + if (adapter->hw.mac.type >= igb_mac_min) + e1000_rx_fifo_flush_82575(&adapter->hw); /* Configure for OS presence */ em_init_manageability(adapter); /* Prepare transmit descriptors and buffers */ - em_setup_transmit_structures(adapter); - em_initialize_transmit_unit(adapter); + em_initialize_transmit_unit(ctx); /* Setup Multicast table */ - em_set_multi(adapter); + em_if_multi_set(ctx); /* ** Figure out the desired mbuf @@ -1416,13 +1204,7 @@ em_init_locked(struct adapter *adapter) else adapter->rx_mbuf_sz = MJUM9BYTES; - /* Prepare receive descriptors and buffers */ - if (em_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - em_stop(adapter); - return; - } - em_initialize_receive_unit(adapter); + em_initialize_receive_unit(ctx); /* Use real VLAN Filter support? */ if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { @@ -1438,118 +1220,54 @@ em_init_locked(struct adapter *adapter) } /* Don't lose promiscuous settings */ - em_set_promisc(adapter); - - /* Set the interface as ACTIVE */ - if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); + em_if_set_promisc(ctx, IFF_PROMISC); e1000_clear_hw_cntrs_base_generic(&adapter->hw); /* MSI/X configuration for 82574 */ if (adapter->hw.mac.type == e1000_82574) { - int tmp; - tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); /* Set the IVAR - interrupt vector routing. */ E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); - } + } else if (adapter->intr_type == IFLIB_INTR_MSIX) /* Set up queue routing */ + igb_configure_queues(adapter); -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (if_getcapenable(ifp) & IFCAP_POLLING) - em_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - em_enable_intr(adapter); + /* this clears any pending interrupts */ + E1000_READ_REG(&adapter->hw, E1000_ICR); + E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); /* AMT based hardware can now take control from firmware */ if (adapter->has_manage && adapter->has_amt) em_get_hw_control(adapter); -} -static void -em_init(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - - -#ifdef DEVICE_POLLING -/********************************************************************* - * - * Legacy polling routine: note this only works with single queue - * - *********************************************************************/ -static int -em_poll(if_t ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 reg_icr; - int rx_done; - - EM_CORE_LOCK(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { - EM_CORE_UNLOCK(adapter); - return (0); + /* Set Energy Efficient Ethernet */ + if (adapter->hw.mac.type >= igb_mac_min && + adapter->hw.phy.media_type == e1000_media_type_copper) { + if (adapter->hw.mac.type == e1000_i354) + e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); + else + e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); } - - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, - em_local_timer, adapter); - } - } - EM_CORE_UNLOCK(adapter); - - em_rxeof(rxr, count, &rx_done); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - - return (rx_done); } -#endif /* DEVICE_POLLING */ - /********************************************************************* * * Fast Legacy/MSI Combined Interrupt Service routine * *********************************************************************/ -static int -em_irq_fast(void *arg) +int +em_intr(void *arg) { - struct adapter *adapter = arg; - if_t ifp; + struct adapter *adapter = arg; + if_ctx_t ctx = adapter->ctx; u32 reg_icr; - ifp = adapter->ifp; - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); + if (adapter->intr_type != IFLIB_INTR_LEGACY) + goto skip_stray; /* Hot eject? */ if (reg_icr == 0xffffffff) return FILTER_STRAY; @@ -1566,80 +1284,42 @@ em_irq_fast(void *arg) (reg_icr & E1000_ICR_INT_ASSERTED) == 0) return FILTER_STRAY; - em_disable_intr(adapter); - taskqueue_enqueue(adapter->tq, &adapter->que_task); - +skip_stray: /* Link status change */ if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { adapter->hw.mac.get_link_status = 1; - taskqueue_enqueue(taskqueue_fast, &adapter->link_task); + iflib_admin_intr_deferred(ctx); } if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; - return FILTER_HANDLED; + + return (FILTER_SCHEDULE_THREAD); } -/* Combined RX/TX handler, used by Legacy and MSI */ static void -em_handle_que(void *context, int pending) +igb_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) { - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - bool more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - if (more) { - taskqueue_enqueue(adapter->tq, &adapter->que_task); - return; - } - } - - em_enable_intr(adapter); - return; + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxq->eims); } - -/********************************************************************* - * - * MSIX Interrupt Service Routines - * - **********************************************************************/ static void -em_msix_tx(void *arg) +em_enable_queue(struct adapter *adapter, struct em_rx_queue *rxq) { - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; + E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxq->eims); +} - ++txr->tx_irq; - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); - return; +static int +em_if_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) +{ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rxq = &adapter->rx_queues[rxqid]; + + if (adapter->hw.mac.type >= igb_mac_min) + igb_enable_queue(adapter, rxq); + else + em_enable_queue(adapter, rxq); + return (0); } /********************************************************************* @@ -1647,25 +1327,14 @@ em_msix_tx(void *arg) * MSIX RX Interrupt Service routine * **********************************************************************/ - -static void -em_msix_rx(void *arg) +static int +em_msix_que(void *arg) { - struct rx_ring *rxr = arg; - struct adapter *adapter = rxr->adapter; - bool more; - - ++rxr->rx_irq; - if (!(if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING)) - return; - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } - return; + struct em_rx_queue *que = arg; + + ++que->irqs; + + return (FILTER_SCHEDULE_THREAD); } /********************************************************************* @@ -1673,103 +1342,51 @@ em_msix_rx(void *arg) * MSIX Link Fast Interrupt Service routine * **********************************************************************/ -static void +static int em_msix_link(void *arg) { struct adapter *adapter = arg; u32 reg_icr; ++adapter->link_irq; + MPASS(adapter->hw.back != NULL); reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); if (reg_icr & E1000_ICR_RXO) adapter->rx_overruns++; if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - adapter->hw.mac.get_link_status = 1; - em_handle_link(adapter, 0); - } else + em_handle_link(adapter->ctx); + } else { E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); + EM_MSIX_LINK | E1000_IMS_LSC); + if (adapter->hw.mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); + + } + /* ** Because we must read the ICR for this interrupt ** it may clear other causes using autoclear, for ** this reason we simply create a soft interrupt ** for all these vectors. */ - if (reg_icr) { + if (reg_icr && adapter->hw.mac.type < igb_mac_min) { E1000_WRITE_REG(&adapter->hw, E1000_ICS, adapter->ims); } - return; + + return (FILTER_HANDLED); } static void -em_handle_rx(void *context, int pending) +em_handle_link(void *context) { - struct rx_ring *rxr = context; - struct adapter *adapter = rxr->adapter; - bool more; + if_ctx_t ctx = context; + struct adapter *adapter = iflib_get_softc(ctx); - more = em_rxeof(rxr, adapter->rx_process_limit, NULL); - if (more) - taskqueue_enqueue(rxr->tq, &rxr->rx_task); - else { - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); - } -} - -static void -em_handle_tx(void *context, int pending) -{ - struct tx_ring *txr = context; - struct adapter *adapter = txr->adapter; - if_t ifp = adapter->ifp; - - EM_TX_LOCK(txr); - em_txeof(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (!if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); - EM_TX_UNLOCK(txr); -} - -static void -em_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - struct tx_ring *txr = adapter->tx_rings; - if_t ifp = adapter->ifp; - - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - return; - - EM_CORE_LOCK(adapter); - callout_stop(&adapter->timer); - em_update_link_status(adapter); - callout_reset(&adapter->timer, hz, em_local_timer, adapter); - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - EM_MSIX_LINK | E1000_IMS_LSC); - if (adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); -#ifdef EM_MULTIQUEUE - if (!drbr_empty(ifp, txr->br)) - em_mq_start_locked(ifp, txr); -#else - if (if_sendq_empty(ifp)) - em_start_locked(ifp, txr); -#endif - EM_TX_UNLOCK(txr); - } - } - EM_CORE_UNLOCK(adapter); + adapter->hw.mac.get_link_status = 1; + iflib_admin_intr_deferred(ctx); } @@ -1782,21 +1399,19 @@ em_handle_link(void *context, int pending) * **********************************************************************/ static void -em_media_status(if_t ifp, struct ifmediareq *ifmr) +em_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) { - struct adapter *adapter = if_getsoftc(ifp); - u_char fiber_type = IFM_1000_SX; + struct adapter *adapter = iflib_get_softc(ctx); + u_char fiber_type = IFM_1000_SX; + + INIT_DEBUGOUT("em_if_media_status: begin"); - INIT_DEBUGOUT("em_media_status: begin"); - - EM_CORE_LOCK(adapter); - em_update_link_status(adapter); + iflib_admin_intr_deferred(ctx); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!adapter->link_active) { - EM_CORE_UNLOCK(adapter); return; } @@ -1804,6 +1419,8 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; ifmr->ifm_active |= fiber_type | IFM_FDX; } else { switch (adapter->link_speed) { @@ -1822,7 +1439,6 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) else ifmr->ifm_active |= IFM_HDX; } - EM_CORE_UNLOCK(adapter); } /********************************************************************* @@ -1834,17 +1450,16 @@ em_media_status(if_t ifp, struct ifmediareq *ifmr) * **********************************************************************/ static int -em_media_change(if_t ifp) +em_if_media_change(if_ctx_t ctx) { - struct adapter *adapter = if_getsoftc(ifp); - struct ifmedia *ifm = &adapter->media; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifmedia *ifm = iflib_get_media(ctx); - INIT_DEBUGOUT("em_media_change: begin"); + INIT_DEBUGOUT("em_if_media_change: begin"); if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return (EINVAL); - EM_CORE_LOCK(adapter); switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_AUTO: adapter->hw.mac.autoneg = DO_AUTO_NEG; @@ -1876,356 +1491,40 @@ em_media_change(if_t ifp) device_printf(adapter->dev, "Unsupported media type\n"); } - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); + em_if_init(ctx); return (0); } -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors. - * - * return 0 on success, positive on failure - **********************************************************************/ - static int -em_xmit(struct tx_ring *txr, struct mbuf **m_headp) +em_if_set_promisc(if_ctx_t ctx, int flags) { - struct adapter *adapter = txr->adapter; - bus_dma_segment_t segs[EM_MAX_SCATTER]; - bus_dmamap_t map; - struct em_txbuffer *tx_buffer, *tx_buffer_mapped; - struct e1000_tx_desc *ctxd = NULL; - struct mbuf *m_head; - struct ether_header *eh; - struct ip *ip = NULL; - struct tcphdr *tp = NULL; - u32 txd_upper = 0, txd_lower = 0; - int ip_off, poff; - int nsegs, i, j, first, last = 0; - int error; - bool do_tso, tso_desc, remap = TRUE; - - m_head = *m_headp; - do_tso = (m_head->m_pkthdr.csum_flags & CSUM_TSO); - tso_desc = FALSE; - ip_off = poff = 0; - - /* - * Intel recommends entire IP/TCP header length reside in a single - * buffer. If multiple descriptors are used to describe the IP and - * TCP header, each descriptor should describe one or more - * complete headers; descriptors referencing only parts of headers - * are not supported. If all layer headers are not coalesced into - * a single buffer, each buffer should not cross a 4KB boundary, - * or be larger than the maximum read request size. - * Controller also requires modifing IP/TCP header to make TSO work - * so we firstly get a writable mbuf chain then coalesce ethernet/ - * IP/TCP header into a single buffer to meet the requirement of - * controller. This also simplifies IP/TCP/UDP checksum offloading - * which also has similar restrictions. - */ - if (do_tso || m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) { - if (do_tso || (m_head->m_next != NULL && - m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)) { - if (M_WRITABLE(*m_headp) == 0) { - m_head = m_dup(*m_headp, M_NOWAIT); - m_freem(*m_headp); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m_head; - } - } - /* - * XXX - * Assume IPv4, we don't have TSO/checksum offload support - * for IPv6 yet. - */ - ip_off = sizeof(struct ether_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - eh = mtod(m_head, struct ether_header *); - if (eh->ether_type == htons(ETHERTYPE_VLAN)) { - ip_off = sizeof(struct ether_vlan_header); - if (m_head->m_len < ip_off) { - m_head = m_pullup(m_head, ip_off); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - } - if (m_head->m_len < ip_off + sizeof(struct ip)) { - m_head = m_pullup(m_head, ip_off + sizeof(struct ip)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - poff = ip_off + (ip->ip_hl << 2); - - if (do_tso || (m_head->m_pkthdr.csum_flags & CSUM_TCP)) { - if (m_head->m_len < poff + sizeof(struct tcphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct tcphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - /* - * TSO workaround: - * pull 4 more bytes of data into it. - */ - if (m_head->m_len < poff + (tp->th_off << 2)) { - m_head = m_pullup(m_head, poff + - (tp->th_off << 2) + - TSO_WORKAROUND); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - tp = (struct tcphdr *)(mtod(m_head, char *) + poff); - if (do_tso) { - ip->ip_len = htons(m_head->m_pkthdr.tso_segsz + - (ip->ip_hl << 2) + - (tp->th_off << 2)); - ip->ip_sum = 0; - /* - * The pseudo TCP checksum does not include TCP - * payload length so driver should recompute - * the checksum here what hardware expect to - * see. This is adherence of Microsoft's Large - * Send specification. - */ - tp->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - } - } else if (m_head->m_pkthdr.csum_flags & CSUM_UDP) { - if (m_head->m_len < poff + sizeof(struct udphdr)) { - m_head = m_pullup(m_head, poff + - sizeof(struct udphdr)); - if (m_head == NULL) { - *m_headp = NULL; - return (ENOBUFS); - } - } - ip = (struct ip *)(mtod(m_head, char *) + ip_off); - } - *m_headp = m_head; - } - - /* - * Map the packet for DMA - * - * Capture the first descriptor index, - * this descriptor will have the index - * of the EOP which is the only one that - * now gets a DONE bit writeback. - */ - first = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[first]; - tx_buffer_mapped = tx_buffer; - map = tx_buffer->map; - -retry: - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - /* - * There are two types of errors we can (try) to handle: - * - EFBIG means the mbuf chain was too long and bus_dma ran - * out of segments. Defragment the mbuf chain and try again. - * - ENOMEM means bus_dma could not obtain enough bounce buffers - * at this point in time. Defer sending and try again later. - * All other errors, in particular EINVAL, are fatal and prevent the - * mbuf chain from ever going through. Drop it and report error. - */ - if (error == EFBIG && remap) { - struct mbuf *m; - - m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again, but only once */ - remap = FALSE; - goto retry; - } else if (error != 0) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* - * TSO Hardware workaround, if this packet is not - * TSO, and is only a single descriptor long, and - * it follows a TSO burst, then we need to add a - * sentinel descriptor to prevent premature writeback. - */ - if ((!do_tso) && (txr->tx_tso == TRUE)) { - if (nsegs == 1) - tso_desc = TRUE; - txr->tx_tso = FALSE; - } - - if (txr->tx_avail < (nsegs + EM_MAX_SCATTER)) { - txr->no_desc_avail++; - bus_dmamap_unload(txr->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* Do hardware assists */ - if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { - em_tso_setup(txr, m_head, ip_off, ip, tp, - &txd_upper, &txd_lower); - /* we need to make a final sentinel transmit desc */ - tso_desc = TRUE; - } else if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) - em_transmit_checksum_setup(txr, m_head, - ip_off, ip, &txd_upper, &txd_lower); - - if (m_head->m_flags & M_VLANTAG) { - /* Set the vlan id. */ - txd_upper |= htole16(if_getvtag(m_head)) << 16; - /* Tell hardware to add tag */ - txd_lower |= htole32(E1000_TXD_CMD_VLE); - } - - i = txr->next_avail_desc; - - /* Set up our transmit descriptors */ - for (j = 0; j < nsegs; j++) { - bus_size_t seg_len; - bus_addr_t seg_addr; - - tx_buffer = &txr->tx_buffers[i]; - ctxd = &txr->tx_base[i]; - seg_addr = segs[j].ds_addr; - seg_len = segs[j].ds_len; - /* - ** TSO Workaround: - ** If this is the last descriptor, we want to - ** split it so we have a small final sentinel - */ - if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { - seg_len -= TSO_WORKAROUND; - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - if (++i == adapter->num_tx_desc) - i = 0; - - /* Now make the sentinel */ - txr->tx_avail--; - ctxd = &txr->tx_base[i]; - tx_buffer = &txr->tx_buffers[i]; - ctxd->buffer_addr = - htole64(seg_addr + seg_len); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | TSO_WORKAROUND); - ctxd->upper.data = - htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } else { - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - } - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - } - - txr->next_avail_desc = i; - txr->tx_avail -= nsegs; - - tx_buffer->m_head = m_head; - /* - ** Here we swap the map so the last descriptor, - ** which gets the completion interrupt has the - ** real map, and the first descriptor gets the - ** unused map from this descriptor. - */ - tx_buffer_mapped->map = tx_buffer->map; - tx_buffer->map = map; - bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - - /* - * Last Descriptor of Packet - * needs End Of Packet (EOP) - * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - /* - * Keep track in the first buffer which - * descriptor will be written back - */ - tx_buffer = &txr->tx_buffers[first]; - tx_buffer->next_eop = last; - - /* - * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 - * that this frame is available to transmit. - */ - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); - - return (0); -} - -static void -em_set_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); u32 reg_rctl; + em_disable_promisc(ctx); + reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (if_getflags(ifp) & IFF_PROMISC) { + if (flags & IFF_PROMISC) { reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); /* Turn this on if you want to see bad packets */ if (em_debug_sbp) reg_rctl |= E1000_RCTL_SBP; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (if_getflags(ifp) & IFF_ALLMULTI) { + } else if (flags & IFF_ALLMULTI) { reg_rctl |= E1000_RCTL_MPE; reg_rctl &= ~E1000_RCTL_UPE; E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); } + return (0); } static void -em_disable_promisc(struct adapter *adapter) +em_disable_promisc(if_ctx_t ctx) { - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); u32 reg_rctl; int mcnt = 0; @@ -2251,9 +1550,10 @@ em_disable_promisc(struct adapter *adapter) **********************************************************************/ static void -em_set_multi(struct adapter *adapter) +em_if_multi_set(if_ctx_t ctx) { - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); u32 reg_rctl = 0; u8 *mta; /* Multicast array memory */ int mcnt = 0; @@ -2302,17 +1602,14 @@ em_set_multi(struct adapter *adapter) **********************************************************************/ static void -em_local_timer(void *arg) +em_if_timer(if_ctx_t ctx, uint16_t qid) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - u32 trigger = 0; + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que; + int i; + int trigger = 0; - EM_CORE_LOCK_ASSERT(adapter); - - em_update_link_status(adapter); + em_if_update_admin_status(ctx); em_update_stats_counters(adapter); /* Reset LAA into RAR[0] on 82571 */ @@ -2320,53 +1617,26 @@ em_local_timer(void *arg) e1000_get_laa_state_82571(&adapter->hw)) e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - /* Mask to use in the irq trigger */ - if (adapter->msix_mem) { - for (int i = 0; i < adapter->num_queues; i++, rxr++) - trigger |= rxr->ims; - rxr = adapter->rx_rings; - } else - trigger = E1000_ICS_RXDMT0; + if (adapter->hw.mac.type < em_mac_min) + lem_smartspeed(adapter); - /* - ** Check on the state of the TX queue(s), this - ** can be done without the lock because its RO - ** and the HUNG state will be static if set. - */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - if (txr->busy == EM_TX_HUNG) - goto hung; - if (txr->busy >= EM_TX_MAXTRIES) - txr->busy = EM_TX_HUNG; - /* Schedule a TX tasklet if needed */ - if (txr->tx_avail <= EM_MAX_SCATTER) - taskqueue_enqueue(txr->tq, &txr->tx_task); + /* Mask to use in the irq trigger */ + if (adapter->intr_type == IFLIB_INTR_MSIX) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) + trigger |= que->eims; + } else { + trigger = E1000_ICS_RXDMT0; } - - callout_reset(&adapter->timer, hz, em_local_timer, adapter); -#ifndef DEVICE_POLLING - /* Trigger an RX interrupt to guarantee mbuf refresh */ - E1000_WRITE_REG(&adapter->hw, E1000_ICS, trigger); -#endif - return; -hung: - /* Looks like we're hung */ - device_printf(adapter->dev, "Watchdog timeout Queue[%d]-- resetting\n", - txr->me); - em_print_debug_info(adapter); - if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING); - adapter->watchdog_events++; - em_init_locked(adapter); } static void -em_update_link_status(struct adapter *adapter) +em_if_update_admin_status(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; + struct ifnet *ifp = iflib_get_ifp(ctx); + device_t dev = iflib_get_dev(ctx); u32 link_check = 0; /* Get the cached link value or read phy for real */ @@ -2380,8 +1650,9 @@ em_update_link_status(struct adapter *adapter) link_check = !hw->mac.get_link_status; if (link_check) /* ESB2 fix */ e1000_cfg_on_link_up(hw); - } else + } else { link_check = TRUE; + } break; case e1000_media_type_fiber: e1000_check_for_link(hw); @@ -2401,18 +1672,6 @@ em_update_link_status(struct adapter *adapter) if (link_check && (adapter->link_active == 0)) { e1000_get_speed_and_duplex(hw, &adapter->link_speed, &adapter->link_duplex); - /* - ** There have proven to be problems with TSO when not - ** at full gigabit speed, so disable the assist automatically - ** when at lower speeds. -jfv - */ - if (adapter->link_speed != SPEED_1000) { - if_sethwassistbits(ifp, 0, CSUM_TSO); - if_setcapenablebit(ifp, 0, IFCAP_TSO4); - if_setcapabilitiesbit(ifp, 0, IFCAP_TSO4); - - } - /* Check if we must disable SPEED_MODE bit on PCI-E */ if ((adapter->link_speed != SPEED_1000) && ((hw->mac.type == e1000_82571) || @@ -2430,7 +1689,8 @@ em_update_link_status(struct adapter *adapter) adapter->link_active = 1; adapter->smartspeed = 0; if_setbaudrate(ifp, adapter->link_speed * 1000000); - if_link_state_change(ifp, LINK_STATE_UP); + iflib_link_state_change(ctx, LINK_STATE_UP, ifp->if_baudrate); + printf("Link state changed to up\n"); } else if (!link_check && (adapter->link_active == 1)) { if_setbaudrate(ifp, 0); adapter->link_speed = 0; @@ -2438,11 +1698,11 @@ em_update_link_status(struct adapter *adapter) if (bootverbose) device_printf(dev, "Link is Down\n"); adapter->link_active = 0; - /* Link down, disable hang detection */ - for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->busy = EM_TX_IDLE; - if_link_state_change(ifp, LINK_STATE_DOWN); + iflib_link_state_change(ctx, LINK_STATE_DOWN, ifp->if_baudrate); + printf("link state changed to down\n"); } + + E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_LINK | E1000_IMS_LSC); } /********************************************************************* @@ -2455,35 +1715,15 @@ em_update_link_status(struct adapter *adapter) **********************************************************************/ static void -em_stop(void *arg) +em_if_stop(if_ctx_t ctx) { - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - - EM_CORE_LOCK_ASSERT(adapter); + struct adapter *adapter = iflib_get_softc(ctx); INIT_DEBUGOUT("em_stop: begin"); - - em_disable_intr(adapter); - callout_stop(&adapter->timer); - - /* Tell the stack that the interface is no longer active */ - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); - - /* Disarm Hang Detection. */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - txr->busy = EM_TX_IDLE; - EM_TX_UNLOCK(txr); - } - - /* I219 needs some special flushing to avoid hangs */ - if (adapter->hw.mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - + e1000_reset_hw(&adapter->hw); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); + if (adapter->hw.mac.type >= e1000_82544) + E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); @@ -2496,12 +1736,12 @@ em_stop(void *arg) * **********************************************************************/ static void -em_identify_hardware(struct adapter *adapter) +em_identify_hardware(if_ctx_t ctx) { - device_t dev = adapter->dev; - + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); /* Save off the information about this board */ @@ -2521,10 +1761,11 @@ em_identify_hardware(struct adapter *adapter) } static int -em_allocate_pci_resources(struct adapter *adapter) +em_allocate_pci_resources(if_ctx_t ctx) { - device_t dev = adapter->dev; - int rid; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + int rid, val; rid = PCIR_BAR(0); adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, @@ -2539,489 +1780,388 @@ em_allocate_pci_resources(struct adapter *adapter) rman_get_bushandle(adapter->memory); adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; + /* Only older adapters use IO mapping */ + if (adapter->hw.mac.type < em_mac_min && + adapter->hw.mac.type > e1000_82543) { + /* Figure our where our IO BAR is ? */ + for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { + val = pci_read_config(dev, rid, 4); + if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { + adapter->io_rid = rid; + break; + } + rid += 4; + /* check for 64bit BAR */ + if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) + rid += 4; + } + if (rid >= PCIR_CIS) { + device_printf(dev, "Unable to locate IO BAR\n"); + return (ENXIO); + } + adapter->ioport = bus_alloc_resource_any(dev, + SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); + if (adapter->ioport == NULL) { + device_printf(dev, "Unable to allocate bus resource: " + "ioport\n"); + return (ENXIO); + } + adapter->hw.io_base = 0; + adapter->osdep.io_bus_space_tag = + rman_get_bustag(adapter->ioport); + adapter->osdep.io_bus_space_handle = + rman_get_bushandle(adapter->ioport); + } + adapter->hw.back = &adapter->osdep; return (0); } -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -int -em_allocate_legacy(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - int error, rid = 0; - - /* Manually turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - if (adapter->msix == 1) /* using MSI */ - rid = 1; - /* We allocate a single interrupt resource */ - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); - } - - /* - * Allocate a fast interrupt and the associated - * deferred processing contexts. - */ - TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); - adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &adapter->tq); - taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); - /* Use a TX only tasklet for local timer */ - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", - device_get_nameunit(adapter->dev)); - TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); - if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, - em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(adapter->tq); - adapter->tq = NULL; - return (error); - } - - return (0); -} - /********************************************************************* * * Setup the MSIX Interrupt handlers - * This is not really Multiqueue, rather - * its just separate interrupt vectors - * for TX, RX, and Link. * **********************************************************************/ -int -em_allocate_msix(struct adapter *adapter) +static int +em_if_msix_intr_assign(if_ctx_t ctx, int msix) { - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - int error, rid, vector = 0; - int cpu_id = 0; - - - /* Make sure all interrupts are disabled */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *rx_que = adapter->rx_queues; + struct em_tx_queue *tx_que = adapter->tx_queues; + int error, rid, i, vector = 0; + char buf[16]; /* First set up ring resources */ - for (int i = 0; i < adapter->num_queues; i++, rxr++, vector++) { - - /* RX ring */ - rid = vector + 1; - - rxr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (rxr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "RX MSIX Interrupt %d\n", i); - return (ENXIO); + for (i = 0; i < adapter->rx_num_queues; i++, rx_que++, vector++) { + rid = vector +1; + snprintf(buf, sizeof(buf), "rxq%d", i); + error = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RX, em_msix_que, rx_que, rx_que->me, buf); + if (error) { + device_printf(iflib_get_dev(ctx), "Failed to allocate que int %d err: %d", i, error); + adapter->rx_num_queues = i + 1; + goto fail; } - if ((error = bus_setup_intr(dev, rxr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, - rxr, &rxr->tag)) != 0) { - device_printf(dev, "Failed to register RX handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, rxr->res, rxr->tag, "rx%d", i); -#endif - rxr->msix = vector; - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, rxr->res, cpu_id); - - TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); - rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, - taskqueue_thread_enqueue, &rxr->tq); - taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); + rx_que->msix = vector; + /* ** Set the bit to enable interrupt ** in E1000_IMS -- bits 20 and 21 ** are for RX0 and RX1, note this has ** NOTHING to do with the MSIX vector */ - rxr->ims = 1 << (20 + i); - adapter->ims |= rxr->ims; - adapter->ivars |= (8 | rxr->msix) << (i * 4); - - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + if (adapter->hw.mac.type == e1000_82574) { + rx_que->eims = 1 << (20 + i); + adapter->ims |= rx_que->eims; + adapter->ivars |= (8 | rx_que->msix) << (i * 4); + } else if (adapter->hw.mac.type == e1000_82575) + rx_que->eims = E1000_EICR_TX_QUEUE0 << vector; + else + rx_que->eims = 1 << vector; } - for (int i = 0; i < adapter->num_queues; i++, txr++, vector++) { - /* TX ring */ + for (i = 0; i < adapter->tx_num_queues; i++, tx_que++) { rid = vector + 1; - txr->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_ACTIVE); - if (txr->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "TX MSIX Interrupt %d\n", i); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, txr->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, - txr, &txr->tag)) != 0) { - device_printf(dev, "Failed to register TX handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, txr->res, txr->tag, "tx%d", i); -#endif - txr->msix = vector; + snprintf(buf, sizeof(buf), "txq%d", i); + tx_que = &adapter->tx_queues[i]; + iflib_softirq_alloc_generic(ctx, rid, IFLIB_INTR_TX, tx_que, tx_que->me, buf); - if (em_last_bind_cpu < 0) - em_last_bind_cpu = CPU_FIRST(); - cpu_id = em_last_bind_cpu; - bus_bind_intr(dev, txr->res, cpu_id); + tx_que->msix = vector; - TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); - txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, - taskqueue_thread_enqueue, &txr->tq); - taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq (cpuid %d)", - device_get_nameunit(adapter->dev), cpu_id); - /* + /* ** Set the bit to enable interrupt ** in E1000_IMS -- bits 22 and 23 ** are for TX0 and TX1, note this has ** NOTHING to do with the MSIX vector */ - txr->ims = 1 << (22 + i); - adapter->ims |= txr->ims; - adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); - - em_last_bind_cpu = CPU_NEXT(em_last_bind_cpu); + if (adapter->hw.mac.type < igb_mac_min) { + tx_que->eims = 1 << (22 + i); + adapter->ims |= tx_que->eims; + adapter->ivars |= (8 | tx_que->msix) << (8 + (i * 4)); + } if (adapter->hw.mac.type == e1000_82575) + tx_que->eims = E1000_EICR_TX_QUEUE0 << (i % adapter->tx_num_queues); + else + tx_que->eims = 1 << (i % adapter->tx_num_queues); } - + /* Link interrupt */ rid = vector + 1; - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!adapter->res) { - device_printf(dev,"Unable to allocate " - "bus resource: Link interrupt [%d]\n", rid); - return (ENXIO); - } - /* Set the link handler function */ - error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - em_msix_link, adapter, &adapter->tag); + error = iflib_irq_alloc_generic(ctx, &adapter->irq, rid, IFLIB_INTR_ADMIN, em_msix_link, adapter, 0, "aq"); + if (error) { - adapter->res = NULL; - device_printf(dev, "Failed to register LINK handler"); - return (error); + device_printf(iflib_get_dev(ctx), "Failed to register admin handler"); + goto fail; } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); -#endif adapter->linkvec = vector; - adapter->ivars |= (8 | vector) << 16; - adapter->ivars |= 0x80000000; - + if (adapter->hw.mac.type < igb_mac_min) { + adapter->ivars |= (8 | vector) << 16; + adapter->ivars |= 0x80000000; + } return (0); + fail: + iflib_irq_free(ctx, &adapter->irq); + rx_que = adapter->rx_queues; + for (int i = 0; i < adapter->rx_num_queues; i++, rx_que++) + iflib_irq_free(ctx, &rx_que->que_irq); + return (error); } - static void -em_free_pci_resources(struct adapter *adapter) +igb_configure_queues(struct adapter *adapter) { - device_t dev = adapter->dev; - struct tx_ring *txr; - struct rx_ring *rxr; - int rid; + struct e1000_hw *hw = &adapter->hw; + struct em_rx_queue *rx_que; + struct em_tx_queue *tx_que; + u32 tmp, ivar = 0, newitr = 0; + /* First turn on RSS capability */ + if (adapter->hw.mac.type != e1000_82575) + E1000_WRITE_REG(hw, E1000_GPIE, + E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | + E1000_GPIE_PBA | E1000_GPIE_NSICR); - /* - ** Release all the queue interrupt resources: - */ - for (int i = 0; i < adapter->num_queues; i++) { - txr = &adapter->tx_rings[i]; - /* an early abort? */ - if (txr == NULL) - break; - rid = txr->msix +1; - if (txr->tag != NULL) { - bus_teardown_intr(dev, txr->res, txr->tag); - txr->tag = NULL; + /* Turn on MSIX */ + switch (adapter->hw.mac.type) { + case e1000_82580: + case e1000_i350: + case e1000_i354: + case e1000_i210: + case e1000_i211: + case e1000_vfadapt: + case e1000_vfadapt_i350: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i & 1) { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } else { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); } - if (txr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, txr->res); - - rxr = &adapter->rx_rings[i]; - /* an early abort? */ - if (rxr == NULL) - break; - rid = rxr->msix +1; - if (rxr->tag != NULL) { - bus_teardown_intr(dev, rxr->res, rxr->tag); - rxr->tag = NULL; + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i >> 1; + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i & 1) { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } else { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; } - if (rxr->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, - rid, rxr->res); + + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + case e1000_82576: + /* RX entries */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + rx_que = &adapter->rx_queues[i]; + if (i < 8) { + ivar &= 0xFFFFFF00; + ivar |= rx_que->msix | E1000_IVAR_VALID; + } else { + ivar &= 0xFF00FFFF; + ivar |= (rx_que->msix | E1000_IVAR_VALID) << 16; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= rx_que->eims; + } + /* TX entries */ + for (int i = 0; i < adapter->tx_num_queues; i++) { + u32 index = i & 0x7; /* Each IVAR has two entries */ + ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); + tx_que = &adapter->tx_queues[i]; + if (i < 8) { + ivar &= 0xFFFF00FF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 8; + } else { + ivar &= 0x00FFFFFF; + ivar |= (tx_que->msix | E1000_IVAR_VALID) << 24; + } + E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); + adapter->que_mask |= tx_que->eims; + } + + /* And for the link interrupt */ + ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; + adapter->link_mask = 1 << adapter->linkvec; + E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); + break; + + case e1000_82575: + /* enable MSI-X support*/ + tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); + tmp |= E1000_CTRL_EXT_PBA_CLR; + /* Auto-Mask interrupts upon ICR read. */ + tmp |= E1000_CTRL_EXT_EIAME; + tmp |= E1000_CTRL_EXT_IRCA; + E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); + + /* Queues */ + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + tmp = E1000_EICR_RX_QUEUE0 << i; + tmp |= E1000_EICR_TX_QUEUE0 << i; + rx_que->eims = tmp; + E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), + i, rx_que->eims); + adapter->que_mask |= rx_que->eims; + } + + /* Link */ + E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), + E1000_EIMS_OTHER); + adapter->link_mask |= E1000_EIMS_OTHER; + default: + break; } - if (adapter->linkvec) /* we are doing MSIX */ - rid = adapter->linkvec + 1; + /* Set the starting interrupt rate */ + if (em_max_interrupt_rate > 0) + newitr = (4000000 / em_max_interrupt_rate) & 0x7FFC; + + if (hw->mac.type == e1000_82575) + newitr |= newitr << 16; else - (adapter->msix != 0) ? (rid = 1):(rid = 0); + newitr |= E1000_EITR_CNT_IGNR; - if (adapter->tag != NULL) { - bus_teardown_intr(dev, adapter->res, adapter->tag); - adapter->tag = NULL; + for (int i = 0; i < adapter->rx_num_queues; i++) { + rx_que = &adapter->rx_queues[i]; + E1000_WRITE_REG(hw, E1000_EITR(rx_que->msix), newitr); } - if (adapter->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); - - - if (adapter->msix) - pci_release_msi(dev); - - if (adapter->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); - - if (adapter->memory != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); - - if (adapter->flash != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - EM_FLASH, adapter->flash); + return; } -/* - * Setup MSI or MSI/X - */ -static int -em_setup_msix(struct adapter *adapter) +static void +em_free_pci_resources(if_ctx_t ctx) { - device_t dev = adapter->dev; - int val; + struct adapter *adapter = iflib_get_softc(ctx); + struct em_rx_queue *que = adapter->rx_queues; + device_t dev = iflib_get_dev(ctx); - /* Nearly always going to use one queue */ - adapter->num_queues = 1; + /* Release all msix queue resources */ + if (adapter->intr_type == IFLIB_INTR_MSIX) + iflib_irq_free(ctx, &adapter->irq); - /* - ** Try using MSI-X for Hartwell adapters - */ - if ((adapter->hw.mac.type == e1000_82574) && - (em_enable_msix == TRUE)) { -#ifdef EM_MULTIQUEUE - adapter->num_queues = (em_num_queues == 1) ? 1 : 2; - if (adapter->num_queues > 1) - em_enable_vectors_82574(adapter); -#endif - /* Map the MSIX BAR */ - int rid = PCIR_BAR(EM_MSIX_BAR); - adapter->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (adapter->msix_mem == NULL) { - /* May not be enabled */ - device_printf(adapter->dev, - "Unable to map MSIX table \n"); - goto msi; - } - val = pci_msix_count(dev); - -#ifdef EM_MULTIQUEUE - /* We need 5 vectors in the multiqueue case */ - if (adapter->num_queues > 1 ) { - if (val >= 5) - val = 5; - else { - adapter->num_queues = 1; - device_printf(adapter->dev, - "Insufficient MSIX vectors for >1 queue, " - "using single queue...\n"); - goto msix_one; - } - } else { -msix_one: -#endif - if (val >= 3) - val = 3; - else { - device_printf(adapter->dev, - "Insufficient MSIX vectors, using MSI\n"); - goto msi; - } -#ifdef EM_MULTIQUEUE - } -#endif - - if ((pci_alloc_msix(dev, &val) == 0)) { - device_printf(adapter->dev, - "Using MSIX interrupts " - "with %d vectors\n", val); - return (val); - } - - /* - ** If MSIX alloc failed or provided us with - ** less than needed, free and fall through to MSI - */ - pci_release_msi(dev); + for (int i = 0; i < adapter->rx_num_queues; i++, que++) { + iflib_irq_free(ctx, &que->que_irq); } -msi: - if (adapter->msix_mem != NULL) { + + + /* First release all the interrupt resources */ + if (adapter->memory != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); - adapter->msix_mem = NULL; + PCIR_BAR(0), adapter->memory); + adapter->memory = NULL; + } + + if (adapter->flash != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, + EM_FLASH, adapter->flash); + adapter->flash = NULL; + } + if (adapter->ioport != NULL) + bus_release_resource(dev, SYS_RES_IOPORT, + adapter->io_rid, adapter->ioport); +} + +/* Setup MSI or MSI/X */ +static int +em_setup_msix(if_ctx_t ctx) +{ + struct adapter *adapter = iflib_get_softc(ctx); + + if (adapter->hw.mac.type == e1000_82574) { + em_enable_vectors_82574(ctx); } - val = 1; - if (pci_alloc_msi(dev, &val) == 0) { - device_printf(adapter->dev, "Using an MSI interrupt\n"); - return (val); - } - /* Should only happen due to manual configuration */ - device_printf(adapter->dev,"No MSI/MSIX using a Legacy IRQ\n"); return (0); } - -/* -** The 3 following flush routines are used as a workaround in the -** I219 client parts and only for them. -** -** em_flush_tx_ring - remove all descriptors from the tx_ring -** -** We want to clear all pending descriptors from the TX ring. -** zeroing happens when the HW reads the regs. We assign the ring itself as -** the data of the next descriptor. We don't care about the data we are about -** to reset the HW. -*/ -static void -em_flush_tx_ring(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct tx_ring *txr = adapter->tx_rings; - struct e1000_tx_desc *txd; - u32 tctl, txd_lower = E1000_TXD_CMD_IFCS; - u16 size = 512; - - tctl = E1000_READ_REG(hw, E1000_TCTL); - E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); - - txd = &txr->tx_base[txr->next_avail_desc++]; - if (txr->next_avail_desc == adapter->num_tx_desc) - txr->next_avail_desc = 0; - - /* Just use the ring as a dummy buffer addr */ - txd->buffer_addr = txr->txdma.dma_paddr; - txd->lower.data = htole32(txd_lower | size); - txd->upper.data = 0; - - /* flush descriptors to memory before notifying the HW */ - wmb(); - - E1000_WRITE_REG(hw, E1000_TDT(0), txr->next_avail_desc); - mb(); - usec_delay(250); -} - -/* -** em_flush_rx_ring - remove all descriptors from the rx_ring -** -** Mark all descriptors in the RX ring as consumed and disable the rx ring -*/ -static void -em_flush_rx_ring(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxdctl; - - rctl = E1000_READ_REG(hw, E1000_RCTL); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); - /* zero the lower 14 bits (prefetch and host thresholds) */ - rxdctl &= 0xffffc000; - /* - * update thresholds: prefetch threshold to 31, host threshold to 1 - * and make sure the granularity is "descriptors" and not "cache lines" - */ - rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); - E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl); - - /* momentarily enable the RX ring for the changes to take effect */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); - E1000_WRITE_FLUSH(hw); - usec_delay(150); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); -} - -/* -** em_flush_desc_rings - remove all descriptors from the descriptor rings -** -** In i219, the descriptor rings must be emptied before resetting the HW -** or before changing the device state to D3 during runtime (runtime PM). -** -** Failure to do this will cause the HW to enter a unit hang state which can -** only be released by PCI reset on the device -** -*/ -static void -em_flush_desc_rings(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; - u16 hang_state; - u32 fext_nvm11, tdlen; - - /* First, disable MULR fix in FEXTNVM11 */ - fext_nvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); - fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX; - E1000_WRITE_REG(hw, E1000_FEXTNVM11, fext_nvm11); - - /* do nothing if we're not in faulty state, or if the queue is empty */ - tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen) - return; - em_flush_tx_ring(adapter); - - /* recheck, maybe the fault is caused by the rx ring */ - hang_state = pci_read_config(dev, PCICFG_DESC_RING_STATUS, 2); - if (hang_state & FLUSH_DESC_REQUIRED) - em_flush_rx_ring(adapter); -} - - /********************************************************************* * * Initialize the hardware to a configuration * as specified by the adapter structure. * **********************************************************************/ + static void -em_reset(struct adapter *adapter) +lem_smartspeed(struct adapter *adapter) { - device_t dev = adapter->dev; - if_t ifp = adapter->ifp; + u16 phy_tmp; + + if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || + adapter->hw.mac.autoneg == 0 || + (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) + return; + + if (adapter->smartspeed == 0) { + /* If Master/Slave config fault is asserted twice, + * we assume back-to-back */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) + return; + e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); + if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { + e1000_read_phy_reg(&adapter->hw, + PHY_1000T_CTRL, &phy_tmp); + if(phy_tmp & CR_1000T_MS_ENABLE) { + phy_tmp &= ~CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, + PHY_1000T_CTRL, phy_tmp); + adapter->smartspeed++; + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, + PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, + PHY_CONTROL, phy_tmp); + } + } + } + return; + } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { + /* If still no link, perhaps using 2/3 pair cable */ + e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); + phy_tmp |= CR_1000T_MS_ENABLE; + e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); + if(adapter->hw.mac.autoneg && + !e1000_copper_link_autoneg(&adapter->hw) && + !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { + phy_tmp |= (MII_CR_AUTO_NEG_EN | + MII_CR_RESTART_AUTO_NEG); + e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); + } + } + /* Restart process after EM_SMARTSPEED_MAX iterations */ + if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) + adapter->smartspeed = 0; +} + + +static void +em_reset(if_ctx_t ctx) +{ + device_t dev = iflib_get_dev(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); struct e1000_hw *hw = &adapter->hw; u16 rx_buffer_size; u32 pba; @@ -3156,10 +2296,6 @@ em_reset(struct adapter *adapter) break; } - /* I219 needs some special flushing to avoid hangs */ - if (hw->mac.type == e1000_pch_spt) - em_flush_desc_rings(adapter); - /* Issue a global reset */ e1000_reset_hw(hw); E1000_WRITE_REG(hw, E1000_WUC, 0); @@ -3173,7 +2309,145 @@ em_reset(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); e1000_get_phy_info(hw); e1000_check_for_link(hw); - return; +} + +#define RSSKEYLEN 10 +static void +em_initialize_rss_mapping(struct adapter *adapter) +{ + uint8_t rss_key[4 * RSSKEYLEN]; + uint32_t reta = 0; + struct e1000_hw *hw = &adapter->hw; + int i; + + /* + * Configure RSS key + */ + arc4rand(rss_key, sizeof(rss_key), 0); + for (i = 0; i < RSSKEYLEN; ++i) { + uint32_t rssrk = 0; + + rssrk = EM_RSSRK_VAL(rss_key, i); + E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); + } + + /* + * Configure RSS redirect table in following fashion: + * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] + */ + for (i = 0; i < sizeof(reta); ++i) { + uint32_t q; + + q = (i % adapter->rx_num_queues) << 7; + reta |= q << (8 * i); + } + + for (i = 0; i < 32; ++i) + E1000_WRITE_REG(hw, E1000_RETA(i), reta); + + E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | + E1000_MRQC_RSS_FIELD_IPV4_TCP | + E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | + E1000_MRQC_RSS_FIELD_IPV6_EX | + E1000_MRQC_RSS_FIELD_IPV6); + +} + +static void +igb_initialize_rss_mapping(struct adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + int i; + int queue_id; + u32 reta; + u32 rss_key[10], mrqc, shift = 0; + + /* XXX? */ + if (adapter->hw.mac.type == e1000_82575) + shift = 6; + + /* + * The redirection table controls which destination + * queue each bucket redirects traffic to. + * Each DWORD represents four queues, with the LSB + * being the first queue in the DWORD. + * + * This just allocates buckets to queues using round-robin + * allocation. + * + * NOTE: It Just Happens to line up with the default + * RSS allocation method. + */ + + /* Warning FM follows */ + reta = 0; + for (i = 0; i < 128; i++) { +#ifdef RSS + queue_id = rss_get_indirection_to_bucket(i); + /* + * If we have more queues than buckets, we'll + * end up mapping buckets to a subset of the + * queues. + * + * If we have more buckets than queues, we'll + * end up instead assigning multiple buckets + * to queues. + * + * Both are suboptimal, but we need to handle + * the case so we don't go out of bounds + * indexing arrays and such. + */ + queue_id = queue_id % adapter->rx_num_queues; +#else + queue_id = (i % adapter->rx_num_queues); +#endif + /* Adjust if required */ + queue_id = queue_id << shift; + + /* + * The low 8 bits are for hash value (n+0); + * The next 8 bits are for hash value (n+1), etc. + */ + reta = reta >> 8; + reta = reta | ( ((uint32_t) queue_id) << 24); + if ((i & 3) == 3) { + E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); + reta = 0; + } + } + + /* Now fill in hash table */ + + /* + * MRQC: Multiple Receive Queues Command + * Set queuing to RSS control, number depends on the device. + */ + mrqc = E1000_MRQC_ENABLE_RSS_8Q; + +#ifdef RSS + /* XXX ew typecasting */ + rss_getkey((uint8_t *) &rss_key); +#else + arc4rand(&rss_key, sizeof(rss_key), 0); +#endif + for (i = 0; i < 10; i++) + E1000_WRITE_REG_ARRAY(hw, + E1000_RSSRK(0), i, rss_key[i]); + + /* + * Configure the RSS fields to hash upon. + */ + mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | + E1000_MRQC_RSS_FIELD_IPV4_TCP); + mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | + E1000_MRQC_RSS_FIELD_IPV6_TCP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | + E1000_MRQC_RSS_FIELD_IPV6_UDP); + mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | + E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); + + E1000_WRITE_REG(hw, E1000_MRQC, mrqc); } /********************************************************************* @@ -3182,57 +2456,36 @@ em_reset(struct adapter *adapter) * **********************************************************************/ static int -em_setup_interface(device_t dev, struct adapter *adapter) +em_setup_interface(if_ctx_t ctx) { - if_t ifp; - + struct ifnet *ifp = iflib_get_ifp(ctx); + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + uint64_t cap = 0; + INIT_DEBUGOUT("em_setup_interface: begin"); - ifp = adapter->ifp = if_gethandle(IFT_ETHER); - if (ifp == 0) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - if_setdev(ifp, dev); - if_setinitfn(ifp, em_init); - if_setsoftc(ifp, adapter); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); - if_setioctlfn(ifp, em_ioctl); - if_setgetcounterfn(ifp, em_get_counter); - /* TSO parameters */ ifp->if_hw_tsomax = IP_MAXPACKET; /* Take m_pullup(9)'s in em_xmit() w/ TSO into acount. */ ifp->if_hw_tsomaxsegcount = EM_MAX_SCATTER - 5; ifp->if_hw_tsomaxsegsize = EM_TSO_SEG_SIZE; -#ifdef EM_MULTIQUEUE - /* Multiqueue stack interface */ - if_settransmitfn(ifp, em_mq_start); - if_setqflushfn(ifp, em_qflush); -#else - if_setstartfn(ifp, em_start); - if_setsendqlen(ifp, adapter->num_tx_desc - 1); - if_setsendqready(ifp); -#endif + /* Single Queue */ + if (adapter->tx_num_queues == 1) { + if_setsendqlen(ifp, scctx->isc_ntxd[0] - 1); + if_setsendqready(ifp); + } - ether_ifattach(ifp, adapter->hw.mac.addr); + cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | IFCAP_TSO4; + cap |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | IFCAP_VLAN_MTU; - if_setcapabilities(ifp, 0); - if_setcapenable(ifp, 0); - - - if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | - IFCAP_TSO4, 0); /* * Tell the upper layer(s) we * support full VLAN capability */ if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | - IFCAP_VLAN_MTU, 0); - if_setcapenable(ifp, if_getcapabilities(ifp)); + if_setcapabilitiesbit(ifp, cap, 0); /* ** Don't turn this on by default, if vlans are @@ -3244,10 +2497,6 @@ em_setup_interface(device_t dev, struct adapter *adapter) */ if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER,0); -#ifdef DEVICE_POLLING - if_setcapabilitiesbit(ifp, IFCAP_POLLING,0); -#endif - /* Enable only WOL MAGIC by default */ if (adapter->wol) { if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); @@ -3258,396 +2507,146 @@ em_setup_interface(device_t dev, struct adapter *adapter) * Specify the media types supported by this adapter and register * callbacks to update media and link information */ - ifmedia_init(&adapter->media, IFM_IMASK, - em_media_change, em_media_status); if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { u_char fiber_type = IFM_1000_SX; /* default type */ - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); + if (adapter->hw.mac.type == e1000_82545) + fiber_type = IFM_1000_LX; + ifmedia_add(adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | fiber_type, 0, NULL); } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); + ifmedia_add(adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL); } } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); + ifmedia_add(adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(adapter->media, IFM_ETHER | IFM_AUTO); return (0); } - -/* - * Manage DMA'able memory. - */ -static void -em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - static int -em_dma_malloc(struct adapter *adapter, bus_size_t size, - struct em_dma_alloc *dma, int mapflags) +em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { - int error; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + int error = E1000_SUCCESS; + struct em_tx_queue *que; + int i; - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - EM_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; + MPASS(adapter->tx_num_queues > 0); + MPASS(adapter->tx_num_queues == ntxqsets); + + /* First allocate the top level queue structs */ + if (!(adapter->tx_queues = + (struct em_tx_queue *) malloc(sizeof(struct em_tx_queue) * + adapter->tx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); + return(ENOMEM); } - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } + for (i = 0, que = adapter->tx_queues; i < adapter->tx_num_queues; i++, que++) { + /* Set up some basics */ + struct tx_ring *txr = &que->txr; + txr->adapter = que->adapter = adapter; + txr->que = que; + que->me = txr->me = i; - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } + /* Allocate transmit buffer memory */ + if (!(txr->tx_buffers = (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * scctx->isc_ntxd[0], M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "failed to allocate tx_buffer memory\n"); + error = ENOMEM; + goto fail; + } + /* get the virtual and physical address of the hardware queues */ + txr->tx_base = (struct e1000_tx_desc *)vaddrs[i*ntxqs]; + txr->tx_paddr = paddrs[i*ntxqs]; + + } + + device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues); return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - + fail: + em_if_queues_free(ctx); return (error); } -static void -em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for the transmit and receive rings, and then - * the descriptors associated with each, called only once at attach. - * - **********************************************************************/ static int -em_allocate_queues(struct adapter *adapter) +em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { - device_t dev = adapter->dev; - struct tx_ring *txr = NULL; - struct rx_ring *rxr = NULL; - int rsize, tsize, error = E1000_SUCCESS; - int txconf = 0, rxconf = 0; + struct adapter *adapter = iflib_get_softc(ctx); + int error = E1000_SUCCESS; + struct em_rx_queue *que; + int i; + MPASS(adapter->rx_num_queues > 0); + MPASS(adapter->rx_num_queues == nrxqsets); - /* Allocate the TX ring struct memory */ - if (!(adapter->tx_rings = - (struct tx_ring *) malloc(sizeof(struct tx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate TX ring memory\n"); + /* First allocate the top level queue structs */ + if (!(adapter->rx_queues = + (struct em_rx_queue *) malloc(sizeof(struct em_rx_queue) * + adapter->rx_num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { + device_printf(iflib_get_dev(ctx), "Unable to allocate queue memory\n"); error = ENOMEM; - goto fail; + goto fail; } - /* Now allocate the RX */ - if (!(adapter->rx_rings = - (struct rx_ring *) malloc(sizeof(struct rx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate RX ring memory\n"); - error = ENOMEM; - goto rx_fail; - } - - tsize = roundup2(adapter->num_tx_desc * - sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); - /* - * Now set up the TX queues, txconf is needed to handle the - * possibility that things fail midcourse and we need to - * undo memory gracefully - */ - for (int i = 0; i < adapter->num_queues; i++, txconf++) { + for (i = 0, que = adapter->rx_queues; i < nrxqsets; i++, que++) { /* Set up some basics */ - txr = &adapter->tx_rings[i]; - txr->adapter = adapter; - txr->me = i; + struct rx_ring *rxr = &que->rxr; + rxr->adapter = que->adapter = adapter; + rxr->que = que; + que->me = rxr->me = i; - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, tsize, - &txr->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; - bzero((void *)txr->tx_base, tsize); - - if (em_allocate_transmit_buffers(txr)) { - device_printf(dev, - "Critical Failure setting up transmit buffers\n"); - error = ENOMEM; - goto err_tx_desc; - } -#if __FreeBSD_version >= 800000 - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(4096, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); -#endif + /* get the virtual and physical address of the hardware queues */ + rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; + rxr->rx_paddr = paddrs[i*nrxqs]; } - /* - * Next the RX queues... - */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - for (int i = 0; i < adapter->num_queues; i++, rxconf++) { - rxr = &adapter->rx_rings[i]; - rxr->adapter = adapter; - rxr->me = i; - - /* Initialize the RX lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (em_dma_malloc(adapter, rsize, - &rxr->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate RxDescriptor memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - rxr->rx_base = (union e1000_rx_desc_extended *)rxr->rxdma.dma_vaddr; - bzero((void *)rxr->rx_base, rsize); - - /* Allocate receive buffers for the ring*/ - if (em_allocate_receive_buffers(rxr)) { - device_printf(dev, - "Critical Failure setting up receive buffers\n"); - error = ENOMEM; - goto err_rx_desc; - } - } + device_printf(iflib_get_dev(ctx), "allocated for %d rx_queues\n", adapter->rx_num_queues); return (0); - -err_rx_desc: - for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) - em_dma_free(adapter, &rxr->rxdma); -err_tx_desc: - for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) - em_dma_free(adapter, &txr->txdma); - free(adapter->rx_rings, M_DEVBUF); -rx_fail: -#if __FreeBSD_version >= 800000 - buf_ring_free(txr->br, M_DEVBUF); -#endif - free(adapter->tx_rings, M_DEVBUF); fail: + em_if_queues_free(ctx); return (error); } - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ -static int -em_allocate_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - device_t dev = adapter->dev; - struct em_txbuffer *txbuf; - int error, i; - - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - EM_TSO_SIZE, /* maxsize */ - EM_MAX_SCATTER, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->txtag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } - - if (!(txr->tx_buffers = - (struct em_txbuffer *) malloc(sizeof(struct em_txbuffer) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer dma maps */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - } - - return 0; -fail: - /* We free all, it handles case where we are in the middle */ - em_free_transmit_structures(adapter); - return (error); -} - -/********************************************************************* - * - * Initialize a transmit ring. - * - **********************************************************************/ static void -em_setup_transmit_ring(struct tx_ring *txr) +em_if_queues_free(if_ctx_t ctx) { - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - int i; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif /* DEV_NETMAP */ + struct adapter *adapter = iflib_get_softc(ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; - /* Clear the old descriptor contents */ - EM_TX_LOCK(txr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_TX, txr->me, 0); -#endif /* DEV_NETMAP */ + if (tx_que != NULL) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; + if (txr->tx_buffers == NULL) + break; - bzero((void *)txr->tx_base, - (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); - /* Reset indices */ - txr->next_avail_desc = 0; - txr->next_to_clean = 0; + free(txr->tx_buffers, M_DEVBUF); + txr->tx_buffers = NULL; + } + free(adapter->tx_queues, M_DEVBUF); + adapter->tx_queues = NULL; + } - /* Free any existing tx buffers. */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - } -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); - uint64_t paddr; - void *addr; + if (rx_que != NULL) { + free(adapter->rx_queues, M_DEVBUF); + adapter->rx_queues = NULL; + } - addr = PNMB(na, slot + si, &paddr); - txr->tx_base[i].buffer_addr = htole64(paddr); - /* reload the map for netmap mode */ - netmap_load_map(na, txr->txtag, txbuf->map, addr); - } -#endif /* DEV_NETMAP */ + em_release_hw_control(adapter); - /* clear the watch index */ - txbuf->next_eop = -1; - } - - /* Set number of descriptors available */ - txr->tx_avail = adapter->num_tx_desc; - txr->busy = EM_TX_IDLE; - - /* Clear checksum offload context. */ - txr->last_hw_offload = 0; - txr->last_hw_ipcss = 0; - txr->last_hw_ipcso = 0; - txr->last_hw_tucss = 0; - txr->last_hw_tucso = 0; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - EM_TX_UNLOCK(txr); -} - -/********************************************************************* - * - * Initialize all transmit rings. - * - **********************************************************************/ -static void -em_setup_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) - em_setup_transmit_ring(txr); - - return; + if (adapter->mta != NULL) { + free(adapter->mta, M_DEVBUF); + } } /********************************************************************* @@ -3656,19 +2655,36 @@ em_setup_transmit_structures(struct adapter *adapter) * **********************************************************************/ static void -em_initialize_transmit_unit(struct adapter *adapter) +em_initialize_transmit_unit(if_ctx_t ctx) { - struct tx_ring *txr = adapter->tx_rings; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que; + struct tx_ring *txr; struct e1000_hw *hw = &adapter->hw; - u32 tctl, txdctl = 0, tarc, tipg = 0; + u32 tctl, txdctl = 0, tarc, tipg = 0; INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); - for (int i = 0; i < adapter->num_queues; i++, txr++) { - u64 bus_addr = txr->txdma.dma_paddr; + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { + u64 bus_addr; + caddr_t offp, endp; + + que = &adapter->tx_queues[i]; + txr = &que->txr; + bus_addr = txr->tx_paddr; + + /*Enable all queues */ + em_init_tx_ring(que); + + /* Clear checksum offload context. */ + offp = (caddr_t)&txr->csum_flags; + endp = (caddr_t)(txr + 1); + bzero(offp, endp - offp); + /* Base and Len of TX Ring */ E1000_WRITE_REG(hw, E1000_TDLEN(i), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); + scctx->isc_ntxd[0] * sizeof(struct e1000_tx_desc)); E1000_WRITE_REG(hw, E1000_TDBAH(i), (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_TDBAL(i), @@ -3681,7 +2697,6 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); - txr->busy = EM_TX_IDLE; txdctl = 0; /* clear txdctl */ txdctl |= 0x1f; /* PTHRESH */ txdctl |= 1 << 8; /* HTHRESH */ @@ -3700,6 +2715,11 @@ em_initialize_transmit_unit(struct adapter *adapter) tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; break; + case e1000_82542: + tipg = DEFAULT_82542_TIPG_IPGT; + tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; + tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; + break; default: if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || (adapter->hw.phy.media_type == @@ -3734,7 +2754,7 @@ em_initialize_transmit_unit(struct adapter *adapter) } else if (adapter->hw.mac.type == e1000_82574) { tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); tarc |= TARC_ERRATA_BIT; - if ( adapter->num_queues > 1) { + if ( adapter->tx_num_queues > 1) { tarc |= (TARC_COMPENSATION_MODE | TARC_MQ_FIX); E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); @@ -3742,7 +2762,6 @@ em_initialize_transmit_unit(struct adapter *adapter) E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); } - adapter->txd_cmd = E1000_TXD_CMD_IFCS; if (adapter->tx_int_delay.value > 0) adapter->txd_cmd |= E1000_TXD_CMD_IDE; @@ -3769,780 +2788,6 @@ em_initialize_transmit_unit(struct adapter *adapter) } } - -/********************************************************************* - * - * Free all transmit rings. - * - **********************************************************************/ -static void -em_free_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - EM_TX_LOCK(txr); - em_free_transmit_buffers(txr); - em_dma_free(adapter, &txr->txdma); - EM_TX_UNLOCK(txr); - EM_TX_LOCK_DESTROY(txr); - } - - free(adapter->tx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -static void -em_free_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct em_txbuffer *txbuf; - - INIT_DEBUGOUT("free_transmit_ring: begin"); - - if (txr->tx_buffers == NULL) - return; - - for (int i = 0; i < adapter->num_tx_desc; i++) { - txbuf = &txr->tx_buffers[i]; - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - if (txbuf->map != NULL) { - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } else if (txbuf->map != NULL) { - bus_dmamap_unload(txr->txtag, - txbuf->map); - bus_dmamap_destroy(txr->txtag, - txbuf->map); - txbuf->map = NULL; - } - } -#if __FreeBSD_version >= 800000 - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); -#endif - if (txr->tx_buffers != NULL) { - free(txr->tx_buffers, M_DEVBUF); - txr->tx_buffers = NULL; - } - if (txr->txtag != NULL) { - bus_dma_tag_destroy(txr->txtag); - txr->txtag = NULL; - } - return; -} - - -/********************************************************************* - * The offload context is protocol specific (TCP/UDP) and thus - * only needs to be set when the protocol changes. The occasion - * of a context change can be a performance detriment, and - * might be better just disabled. The reason arises in the way - * in which the controller supports pipelined requests from the - * Tx data DMA. Up to four requests can be pipelined, and they may - * belong to the same packet or to multiple packets. However all - * requests for one packet are issued before a request is issued - * for a subsequent packet and if a request for the next packet - * requires a context change, that request will be stalled - * until the previous request completes. This means setting up - * a new context effectively disables pipelined Tx data DMA which - * in turn greatly slow down performance to send small sized - * frames. - **********************************************************************/ -static void -em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD = NULL; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - u32 cmd = 0; - u16 offload = 0; - u8 ipcso, ipcss, tucso, tucss; - - ipcss = ipcso = tucss = tucso = 0; - hdr_len = ip_off + (ip->ip_hl << 2); - cur = txr->next_avail_desc; - - /* Setup of IP header checksum. */ - if (mp->m_pkthdr.csum_flags & CSUM_IP) { - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; - offload |= CSUM_IP; - ipcss = ip_off; - ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->lower_setup.ip_fields.ipcss = ipcss; - TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); - TXD->lower_setup.ip_fields.ipcso = ipcso; - cmd |= E1000_TXD_CMD_IP; - } - - if (mp->m_pkthdr.csum_flags & CSUM_TCP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - offload |= CSUM_TCP; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct tcphdr, th_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - cmd |= E1000_TXD_CMD_TCP; - } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - tucss = hdr_len; - tucso = hdr_len + offsetof(struct udphdr, uh_sum); - /* - * The 82574L can only remember the *last* context used - * regardless of queue that it was use for. We cannot reuse - * contexts on this hardware platform and must generate a new - * context every time. 82574L hardware spec, section 7.2.6, - * second note. - */ - if (adapter->num_queues < 2) { - /* - * Setting up new checksum offload context for every - * frames takes a lot of processing time for hardware. - * This also reduces performance a lot for small sized - * frames so avoid it if driver can use previously - * configured checksum offload context. - */ - if (txr->last_hw_offload == offload) { - if (offload & CSUM_IP) { - if (txr->last_hw_ipcss == ipcss && - txr->last_hw_ipcso == ipcso && - txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } else { - if (txr->last_hw_tucss == tucss && - txr->last_hw_tucso == tucso) - return; - } - } - txr->last_hw_offload = offload; - txr->last_hw_tucss = tucss; - txr->last_hw_tucso = tucso; - } - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; - TXD->upper_setup.tcp_fields.tucss = tucss; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = tucso; - } - - if (offload & CSUM_IP) { - txr->last_hw_ipcss = ipcss; - txr->last_hw_ipcso = ipcso; - } - - TXD->tcp_seg_setup.data = htole32(0); - TXD->cmd_and_length = - htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &txr->tx_buffers[cur]; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; -} - - -/********************************************************************** - * - * Setup work for hardware segmentation offload (TSO) - * - **********************************************************************/ -static void -em_tso_setup(struct tx_ring *txr, struct mbuf *mp, int ip_off, - struct ip *ip, struct tcphdr *tp, u32 *txd_upper, u32 *txd_lower) -{ - struct adapter *adapter = txr->adapter; - struct e1000_context_desc *TXD; - struct em_txbuffer *tx_buffer; - int cur, hdr_len; - - /* - * In theory we can use the same TSO context if and only if - * frame is the same type(IP/TCP) and the same MSS. However - * checking whether a frame has the same IP/TCP structure is - * hard thing so just ignore that and always restablish a - * new TSO context. - */ - hdr_len = ip_off + (ip->ip_hl << 2) + (tp->th_off << 2); - *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ - E1000_TXD_DTYP_D | /* Data descr type */ - E1000_TXD_CMD_TSE); /* Do TSE on this packet */ - - /* IP and/or TCP header checksum calculation and insertion. */ - *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; - - cur = txr->next_avail_desc; - tx_buffer = &txr->tx_buffers[cur]; - TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; - - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place put the checksum. - */ - TXD->lower_setup.ip_fields.ipcss = ip_off; - TXD->lower_setup.ip_fields.ipcse = - htole16(ip_off + (ip->ip_hl << 2) - 1); - TXD->lower_setup.ip_fields.ipcso = ip_off + offsetof(struct ip, ip_sum); - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD->upper_setup.tcp_fields.tucss = ip_off + (ip->ip_hl << 2); - TXD->upper_setup.tcp_fields.tucse = 0; - TXD->upper_setup.tcp_fields.tucso = - ip_off + (ip->ip_hl << 2) + offsetof(struct tcphdr, th_sum); - /* - * Payload size per packet w/o any headers. - * Length of all headers up to payload. - */ - TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); - TXD->tcp_seg_setup.fields.hdr_len = hdr_len; - - TXD->cmd_and_length = htole32(adapter->txd_cmd | - E1000_TXD_CMD_DEXT | /* Extended descr */ - E1000_TXD_CMD_TSE | /* TSE context */ - E1000_TXD_CMD_IP | /* Do IP csum */ - E1000_TXD_CMD_TCP | /* Do TCP checksum */ - (mp->m_pkthdr.len - (hdr_len))); /* Total len */ - - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++cur == adapter->num_tx_desc) - cur = 0; - - txr->tx_avail--; - txr->next_avail_desc = cur; - txr->tx_tso = TRUE; -} - - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -static void -em_txeof(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - int first, last, done, processed; - struct em_txbuffer *tx_buffer; - struct e1000_tx_desc *tx_desc, *eop_desc; - if_t ifp = adapter->ifp; - - EM_TX_LOCK_ASSERT(txr); -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, txr->me)) - return; -#endif /* DEV_NETMAP */ - - /* No work, make sure hang detection is disabled */ - if (txr->tx_avail == adapter->num_tx_desc) { - txr->busy = EM_TX_IDLE; - return; - } - - processed = 0; - first = txr->next_to_clean; - tx_desc = &txr->tx_base[first]; - tx_buffer = &txr->tx_buffers[first]; - last = tx_buffer->next_eop; - eop_desc = &txr->tx_base[last]; - - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == adapter->num_tx_desc) - last = 0; - done = last; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_POSTREAD); - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++txr->tx_avail; - ++processed; - - if (tx_buffer->m_head) { - bus_dmamap_sync(txr->txtag, - tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } - tx_buffer->next_eop = -1; - - if (++first == adapter->num_tx_desc) - first = 0; - - tx_buffer = &txr->tx_buffers[first]; - tx_desc = &txr->tx_base[first]; - } - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - /* See if we can continue to the next packet */ - last = tx_buffer->next_eop; - if (last != -1) { - eop_desc = &txr->tx_base[last]; - /* Get new done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - txr->next_to_clean = first; - - /* - ** Hang detection: we know there's work outstanding - ** or the entry return would have been taken, so no - ** descriptor processed here indicates a potential hang. - ** The local timer will examine this and do a reset if needed. - */ - if (processed == 0) { - if (txr->busy != EM_TX_HUNG) - ++txr->busy; - } else /* At least one descriptor was cleaned */ - txr->busy = EM_TX_BUSY; /* note this clears HUNG */ - - /* - * If we have a minimum free, clear IFF_DRV_OACTIVE - * to tell the stack that it is OK to send packets. - * Notice that all writes of OACTIVE happen under the - * TX lock which, with a single queue, guarantees - * sanity. - */ - if (txr->tx_avail >= EM_MAX_SCATTER) { - if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); - } - - /* Disable hang detection if all clean */ - if (txr->tx_avail == adapter->num_tx_desc) - txr->busy = EM_TX_IDLE; -} - -/********************************************************************* - * - * Refresh RX descriptor mbufs from system mbuf buffer pool. - * - **********************************************************************/ -static void -em_refresh_mbufs(struct rx_ring *rxr, int limit) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m; - bus_dma_segment_t segs; - struct em_rxbuffer *rxbuf; - int i, j, error, nsegs; - bool cleaned = FALSE; - - i = j = rxr->next_to_refresh; - /* - ** Get one descriptor beyond - ** our work mark to control - ** the loop. - */ - if (++j == adapter->num_rx_desc) - j = 0; - - while (j != limit) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head == NULL) { - m = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - /* - ** If we have a temporary resource shortage - ** that causes a failure, just abort refresh - ** for now, we will return to this point when - ** reinvoked from em_rxeof. - */ - if (m == NULL) - goto update; - } else - m = rxbuf->m_head; - - m->m_len = m->m_pkthdr.len = adapter->rx_mbuf_sz; - m->m_flags |= M_PKTHDR; - m->m_data = m->m_ext.ext_buf; - - /* Use bus_dma machinery to setup the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxbuf->map, - m, &segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(m); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = m; - rxbuf->paddr = segs.ds_addr; - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - em_setup_rxdesc(&rxr->rx_base[i], rxbuf); - cleaned = TRUE; - - i = j; /* Next is precalulated for us */ - rxr->next_to_refresh = i; - /* Calculate next controlling index */ - if (++j == adapter->num_rx_desc) - j = 0; - } -update: - /* - ** Update the tail pointer only if, - ** and as far as we have refreshed. - */ - if (cleaned) - E1000_WRITE_REG(&adapter->hw, - E1000_RDT(rxr->me), rxr->next_to_refresh); - - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -em_allocate_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - device_t dev = adapter->dev; - struct em_rxbuffer *rxbuf; - int error; - - rxr->rx_buffers = malloc(sizeof(struct em_rxbuffer) * - adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (rxr->rx_buffers == NULL) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - return (ENOMEM); - } - - error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM9BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM9BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &rxr->rxtag); - if (error) { - device_printf(dev, "%s: bus_dma_tag_create failed %d\n", - __func__, error); - goto fail; - } - - rxbuf = rxr->rx_buffers; - for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { - rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->rxtag, 0, &rxbuf->map); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - } - - return (0); - -fail: - em_free_receive_structures(adapter); - return (error); -} - - -/********************************************************************* - * - * Initialize a receive ring and its buffers. - * - **********************************************************************/ -static int -em_setup_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf; - bus_dma_segment_t seg[1]; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_slot *slot; - struct netmap_adapter *na = netmap_getna(adapter->ifp); -#endif - - - /* Clear the ring contents */ - EM_RX_LOCK(rxr); - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_rx_desc_extended), EM_DBA_ALIGN); - bzero((void *)rxr->rx_base, rsize); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, rxr->me, 0); -#endif - - /* - ** Free current RX buffer structs and their mbufs - */ - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; /* mark as freed */ - } - } - - /* Now replenish the mbufs */ - for (int j = 0; j != adapter->num_rx_desc; ++j) { - rxbuf = &rxr->rx_buffers[j]; -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->rx_rings[rxr->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - netmap_load_map(na, rxr->rxtag, rxbuf->map, addr); - rxbuf->paddr = paddr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - continue; - } -#endif /* DEV_NETMAP */ - rxbuf->m_head = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - rxbuf->m_head->m_len = adapter->rx_mbuf_sz; - rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ - rxbuf->m_head->m_pkthdr.len = adapter->rx_mbuf_sz; - - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->rxtag, - rxbuf->map, rxbuf->m_head, seg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - goto fail; - } - bus_dmamap_sync(rxr->rxtag, - rxbuf->map, BUS_DMASYNC_PREREAD); - - rxbuf->paddr = seg[0].ds_addr; - em_setup_rxdesc(&rxr->rx_base[j], rxbuf); - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -fail: - EM_RX_UNLOCK(rxr); - return (error); -} - -/********************************************************************* - * - * Initialize all receive rings. - * - **********************************************************************/ -static int -em_setup_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - int q; - - for (q = 0; q < adapter->num_queues; q++, rxr++) - if (em_setup_receive_ring(rxr)) - goto fail; - - return (0); -fail: - /* - * Free RX buffers allocated so far, we will only handle - * the rings that completed, the failing case will have - * cleaned up for itself. 'q' failed, so its the terminus. - */ - for (int i = 0; i < q; ++i) { - rxr = &adapter->rx_rings[i]; - for (int n = 0; n < adapter->num_rx_desc; n++) { - struct em_rxbuffer *rxbuf; - rxbuf = &rxr->rx_buffers[n]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - return (ENOBUFS); -} - -/********************************************************************* - * - * Free all receive rings. - * - **********************************************************************/ -static void -em_free_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - em_free_receive_buffers(rxr); - /* Free the ring memory as well */ - em_dma_free(adapter, &rxr->rxdma); - EM_RX_LOCK_DESTROY(rxr); - } - - free(adapter->rx_rings, M_DEVBUF); -} - - -/********************************************************************* - * - * Free receive ring data structures - * - **********************************************************************/ -static void -em_free_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct em_rxbuffer *rxbuf = NULL; - - INIT_DEBUGOUT("free_receive_buffers: begin"); - - if (rxr->rx_buffers != NULL) { - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->map != NULL) { - bus_dmamap_sync(rxr->rxtag, rxbuf->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->rxtag, rxbuf->map); - bus_dmamap_destroy(rxr->rxtag, rxbuf->map); - } - if (rxbuf->m_head != NULL) { - m_freem(rxbuf->m_head); - rxbuf->m_head = NULL; - } - } - free(rxr->rx_buffers, M_DEVBUF); - rxr->rx_buffers = NULL; - rxr->next_to_check = 0; - rxr->next_to_refresh = 0; - } - - if (rxr->rxtag != NULL) { - bus_dma_tag_destroy(rxr->rxtag); - rxr->rxtag = NULL; - } - - return; -} - - /********************************************************************* * * Enable receive unit. @@ -4550,11 +2795,14 @@ em_free_receive_buffers(struct rx_ring *rxr) **********************************************************************/ static void -em_initialize_receive_unit(struct adapter *adapter) +em_initialize_receive_unit(if_ctx_t ctx) { - struct rx_ring *rxr = adapter->rx_rings; - if_t ifp = adapter->ifp; + struct adapter *adapter = iflib_get_softc(ctx); + if_softc_ctx_t scctx = adapter->shared; + struct ifnet *ifp = iflib_get_ifp(ctx); struct e1000_hw *hw = &adapter->hw; + struct em_rx_queue *que; + int i; u32 rctl, rxcsum, rfctl; INIT_DEBUGOUT("em_initialize_receive_units: begin"); @@ -4587,16 +2835,18 @@ em_initialize_receive_unit(struct adapter *adapter) if (!em_disable_crc_stripping) rctl |= E1000_RCTL_SECRC; - E1000_WRITE_REG(&adapter->hw, E1000_RADV, - adapter->rx_abs_int_delay.value); + if (adapter->hw.mac.type >= e1000_82540) { + E1000_WRITE_REG(&adapter->hw, E1000_RADV, + adapter->rx_abs_int_delay.value); + /* + * Set the interrupt throttling rate. Value is calculated + * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) + */ + E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); + } E1000_WRITE_REG(&adapter->hw, E1000_RDTR, adapter->rx_int_delay.value); - /* - * Set the interrupt throttling rate. Value is calculated - * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) - */ - E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); /* Use extended rx descriptor formats */ rfctl = E1000_READ_REG(hw, E1000_RFCTL); @@ -4615,60 +2865,37 @@ em_initialize_receive_unit(struct adapter *adapter) E1000_WRITE_REG(hw, E1000_RFCTL, rfctl); rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (if_getcapenable(ifp) & IFCAP_RXCSUM) { -#ifdef EM_MULTIQUEUE - rxcsum |= E1000_RXCSUM_TUOFL | - E1000_RXCSUM_IPOFL | - E1000_RXCSUM_PCSD; -#else - rxcsum |= E1000_RXCSUM_TUOFL; -#endif + if (if_getcapenable(ifp) & IFCAP_RXCSUM && + adapter->hw.mac.type >= e1000_82543) { + if (adapter->tx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) { + rxcsum |= E1000_RXCSUM_PCSD; + if (hw->mac.type != e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } else + rxcsum |= E1000_RXCSUM_TUOFL | + E1000_RXCSUM_IPOFL | + E1000_RXCSUM_PCSD; + } else { + if (adapter->hw.mac.type >= igb_mac_min) + rxcsum |= E1000_RXCSUM_IPPCSE; + else + rxcsum |= E1000_RXCSUM_TUOFL | E1000_RXCSUM_IPOFL; + if (adapter->hw.mac.type > e1000_82575) + rxcsum |= E1000_RXCSUM_CRCOFL; + } } else rxcsum &= ~E1000_RXCSUM_TUOFL; E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); -#ifdef EM_MULTIQUEUE -#define RSSKEYLEN 10 - if (adapter->num_queues > 1) { - uint8_t rss_key[4 * RSSKEYLEN]; - uint32_t reta = 0; - int i; - - /* - * Configure RSS key - */ - arc4rand(rss_key, sizeof(rss_key), 0); - for (i = 0; i < RSSKEYLEN; ++i) { - uint32_t rssrk = 0; - - rssrk = EM_RSSRK_VAL(rss_key, i); - E1000_WRITE_REG(hw,E1000_RSSRK(i), rssrk); - } - - /* - * Configure RSS redirect table in following fashion: - * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)] - */ - for (i = 0; i < sizeof(reta); ++i) { - uint32_t q; - - q = (i % adapter->num_queues) << 7; - reta |= q << (8 * i); - } - - for (i = 0; i < 32; ++i) { - E1000_WRITE_REG(hw, E1000_RETA(i), reta); - } - - E1000_WRITE_REG(hw, E1000_MRQC, E1000_MRQC_RSS_ENABLE_2Q | - E1000_MRQC_RSS_FIELD_IPV4_TCP | - E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX | - E1000_MRQC_RSS_FIELD_IPV6_EX | - E1000_MRQC_RSS_FIELD_IPV6); + if (adapter->rx_num_queues > 1) { + if (adapter->hw.mac.type >= igb_mac_min) + igb_initialize_rss_mapping(adapter); + else + em_initialize_rss_mapping(adapter); } -#endif + /* ** XXX TEMPORARY WORKAROUND: on some systems with 82573 ** long latencies are observed, like Lenovo X60. This @@ -4679,28 +2906,21 @@ em_initialize_receive_unit(struct adapter *adapter) if (hw->mac.type == e1000_82573) E1000_WRITE_REG(hw, E1000_RDTR, 0x20); - for (int i = 0; i < adapter->num_queues; i++, rxr++) { + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; /* Setup the Base and Length of the Rx Descriptor Ring */ - u64 bus_addr = rxr->rxdma.dma_paddr; - u32 rdt = adapter->num_rx_desc - 1; /* default */ + u64 bus_addr = rxr->rx_paddr; +#if 0 + u32 rdt = adapter->rx_num_queues -1; /* default */ +#endif E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(union e1000_rx_desc_extended)); + scctx->isc_nrxd[0] * sizeof(union e1000_rx_desc_extended)); E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); /* Setup the Head and Tail Descriptor Pointers */ E1000_WRITE_REG(hw, E1000_RDH(i), 0); -#ifdef DEV_NETMAP - /* - * an init() while a netmap client is active must - * preserve the rx buffers passed to userspace. - */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - struct netmap_adapter *na = netmap_getna(adapter->ifp); - rdt -= nm_kr_rxspace(&na->rx_rings[i]); - } -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(hw, E1000_RDT(i), rdt); + E1000_WRITE_REG(hw, E1000_RDT(i), 0); } /* @@ -4710,6 +2930,7 @@ em_initialize_receive_unit(struct adapter *adapter) * Only write to RXDCTL(1) if there is a need for different * settings. */ + if (((adapter->hw.mac.type == e1000_ich9lan) || (adapter->hw.mac.type == e1000_pch2lan) || (adapter->hw.mac.type == e1000_ich10lan)) && @@ -4717,17 +2938,82 @@ em_initialize_receive_unit(struct adapter *adapter) u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(0)); E1000_WRITE_REG(hw, E1000_RXDCTL(0), rxdctl | 3); } else if (adapter->hw.mac.type == e1000_82574) { - for (int i = 0; i < adapter->num_queues; i++) { + for (int i = 0; i < adapter->rx_num_queues; i++) { u32 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= 0x20; /* PTHRESH */ rxdctl |= 4 << 8; /* HTHRESH */ rxdctl |= 4 << 16;/* WTHRESH */ rxdctl |= 1 << 24; /* Switch to granularity */ E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); } + } else if (adapter->hw.mac.type >= igb_mac_min) { + u32 psize, srrctl = 0; + + if (ifp->if_mtu > ETHERMTU) { + rctl |= E1000_RCTL_LPE; + + /* Set maximum packet len */ + psize = scctx->isc_max_frame_size; + if (psize <= 4096) { + srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; + } else if (psize > 4096) { + srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; + } + + /* are we on a vlan? */ + if (ifp->if_vlantrunk != NULL) + psize += VLAN_TAG_SIZE; + E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); + } else { + rctl &= ~E1000_RCTL_LPE; + srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; + rctl |= E1000_RCTL_SZ_2048; + } + + /* + * If TX flow control is disabled and there's >1 queue defined, + * enable DROP. + * + * This drops frames rather than hanging the RX MAC for all queues. + */ + if ((adapter->rx_num_queues > 1) && + (adapter->fc == e1000_fc_none || + adapter->fc == e1000_fc_rx_pause)) { + srrctl |= E1000_SRRCTL_DROP_EN; + } + /* Setup the Base and Length of the Rx Descriptor Rings */ + for (i = 0, que = adapter->rx_queues; i < adapter->rx_num_queues; i++, que++) { + struct rx_ring *rxr = &que->rxr; + u64 bus_addr = rxr->rx_paddr; + u32 rxdctl; + +#ifdef notyet + /* Configure for header split? -- ignore for now */ + rxr->hdr_split = igb_header_split; +#else + srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; +#endif + + + E1000_WRITE_REG(hw, E1000_RDLEN(i), + scctx->isc_nrxd[0] * sizeof(struct e1000_rx_desc)); + E1000_WRITE_REG(hw, E1000_RDBAH(i), + (uint32_t)(bus_addr >> 32)); + E1000_WRITE_REG(hw, E1000_RDBAL(i), + (uint32_t)bus_addr); + E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); + /* Enable this Queue */ + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; + rxdctl &= 0xFFF00000; + rxdctl |= IGB_RX_PTHRESH; + rxdctl |= IGB_RX_HTHRESH << 8; + rxdctl |= IGB_RX_WTHRESH << 16; + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + } } - if (adapter->hw.mac.type >= e1000_pch2lan) { if (if_getmtu(ifp) > ETHERMTU) e1000_lv_jumbo_workaround_ich8lan(hw, TRUE); @@ -4753,323 +3039,28 @@ em_initialize_receive_unit(struct adapter *adapter) return; } - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * For polling we also now return the number of cleaned packets - *********************************************************************/ -static bool -em_rxeof(struct rx_ring *rxr, int count, int *done) -{ - struct adapter *adapter = rxr->adapter; - if_t ifp = adapter->ifp; - struct mbuf *mp, *sendmp; - u32 status = 0; - u16 len; - int i, processed, rxdone = 0; - bool eop; - union e1000_rx_desc_extended *cur; - - EM_RX_LOCK(rxr); - - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, rxr->me, &processed)) { - EM_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - for (i = rxr->next_to_check, processed = 0; count != 0;) { - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) - break; - - cur = &rxr->rx_base[i]; - status = le32toh(cur->wb.upper.status_error); - mp = sendmp = NULL; - - if ((status & E1000_RXD_STAT_DD) == 0) - break; - - len = le16toh(cur->wb.upper.length); - eop = (status & E1000_RXD_STAT_EOP) != 0; - - if ((status & E1000_RXDEXT_ERR_FRAME_ERR_MASK) || - (rxr->discard == TRUE)) { - adapter->dropped_pkts++; - ++rxr->rx_discarded; - if (!eop) /* Catch subsequent segs */ - rxr->discard = TRUE; - else - rxr->discard = FALSE; - em_rx_discard(rxr, i); - goto next_desc; - } - bus_dmamap_unload(rxr->rxtag, rxr->rx_buffers[i].map); - - /* Assign correct length to the current fragment */ - mp = rxr->rx_buffers[i].m_head; - mp->m_len = len; - - /* Trigger for refresh */ - rxr->rx_buffers[i].m_head = NULL; - - /* First segment? */ - if (rxr->fmp == NULL) { - mp->m_pkthdr.len = len; - rxr->fmp = rxr->lmp = mp; - } else { - /* Chain mbuf's together */ - mp->m_flags &= ~M_PKTHDR; - rxr->lmp->m_next = mp; - rxr->lmp = mp; - rxr->fmp->m_pkthdr.len += len; - } - - if (eop) { - --count; - sendmp = rxr->fmp; - if_setrcvif(sendmp, ifp); - if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); - em_receive_checksum(status, sendmp); -#ifndef __NO_STRICT_ALIGNMENT - if (adapter->hw.mac.max_frame_size > - (MCLBYTES - ETHER_ALIGN) && - em_fixup_rx(rxr) != 0) - goto skip; -#endif - if (status & E1000_RXD_STAT_VP) { - if_setvtag(sendmp, - le16toh(cur->wb.upper.vlan)); - sendmp->m_flags |= M_VLANTAG; - } -#ifndef __NO_STRICT_ALIGNMENT -skip: -#endif - rxr->fmp = rxr->lmp = NULL; - } -next_desc: - /* Sync the ring */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - - /* Zero out the receive descriptors status. */ - cur->wb.upper.status_error &= htole32(~0xFF); - ++rxdone; /* cumulative for POLL */ - ++processed; - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - - /* Send to the stack */ - if (sendmp != NULL) { - rxr->next_to_check = i; - EM_RX_UNLOCK(rxr); - if_input(ifp, sendmp); - EM_RX_LOCK(rxr); - i = rxr->next_to_check; - } - - /* Only refresh mbufs every 8 descriptors */ - if (processed == 8) { - em_refresh_mbufs(rxr, i); - processed = 0; - } - } - - /* Catch any remaining refresh work */ - if (e1000_rx_unrefreshed(rxr)) - em_refresh_mbufs(rxr, i); - - rxr->next_to_check = i; - if (done != NULL) - *done = rxdone; - EM_RX_UNLOCK(rxr); - - return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -static __inline void -em_rx_discard(struct rx_ring *rxr, int i) -{ - struct em_rxbuffer *rbuf; - - rbuf = &rxr->rx_buffers[i]; - bus_dmamap_unload(rxr->rxtag, rbuf->map); - - /* Free any previous pieces */ - if (rxr->fmp != NULL) { - rxr->fmp->m_flags |= M_PKTHDR; - m_freem(rxr->fmp); - rxr->fmp = NULL; - rxr->lmp = NULL; - } - /* - ** Free buffer and allow em_refresh_mbufs() - ** to clean up and recharge buffer. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - } - return; -} - -#ifndef __NO_STRICT_ALIGNMENT -/* - * When jumbo frames are enabled we should realign entire payload on - * architecures with strict alignment. This is serious design mistake of 8254x - * as it nullifies DMA operations. 8254x just allows RX buffer size to be - * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its - * payload. On architecures without strict alignment restrictions 8254x still - * performs unaligned memory access which would reduce the performance too. - * To avoid copying over an entire frame to align, we allocate a new mbuf and - * copy ethernet header to the new mbuf. The new mbuf is prepended into the - * existing mbuf chain. - * - * Be aware, best performance of the 8254x is achived only when jumbo frame is - * not used at all on architectures with strict alignment. - */ -static int -em_fixup_rx(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct mbuf *m, *n; - int error; - - error = 0; - m = rxr->fmp; - if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { - bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); - m->m_data += ETHER_HDR_LEN; - } else { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n != NULL) { - bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); - m->m_data += ETHER_HDR_LEN; - m->m_len -= ETHER_HDR_LEN; - n->m_len = ETHER_HDR_LEN; - M_MOVE_PKTHDR(n, m); - n->m_next = m; - rxr->fmp = n; - } else { - adapter->dropped_pkts++; - m_freem(rxr->fmp); - rxr->fmp = NULL; - error = ENOMEM; - } - } - - return (error); -} -#endif - static void -em_setup_rxdesc(union e1000_rx_desc_extended *rxd, const struct em_rxbuffer *rxbuf) +em_if_vlan_register(if_ctx_t ctx, u16 vtag) { - rxd->read.buffer_addr = htole64(rxbuf->paddr); - /* DD bits must be cleared */ - rxd->wb.upper.status_error= 0; -} - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -em_receive_checksum(uint32_t status, struct mbuf *mp) -{ - mp->m_pkthdr.csum_flags = 0; - - /* Ignore Checksum bit is set */ - if (status & E1000_RXD_STAT_IXSM) - return; - - /* If the IP checksum exists and there is no IP Checksum error */ - if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == - E1000_RXD_STAT_IPCS) { - mp->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); - } - - /* TCP or UDP checksum */ - if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == - E1000_RXD_STAT_TCPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } - if (status & E1000_RXD_STAT_UDPCS) { - mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } -} - -/* - * This routine is run via an vlan - * config EVENT - */ -static void -em_register_vlan(void *arg, if_t ifp, u16 vtag) -{ - struct adapter *adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); u32 index, bit; - if ((void*)adapter != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ - return; - - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] |= (1 << bit); ++adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } -/* - * This routine is run via an vlan - * unconfig EVENT - */ static void -em_unregister_vlan(void *arg, if_t ifp, u16 vtag) +em_if_vlan_unregister(if_ctx_t ctx, u16 vtag) { - struct adapter *adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); u32 index, bit; - if (adapter != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - EM_CORE_LOCK(adapter); index = (vtag >> 5) & 0x7F; bit = vtag & 0x1F; adapter->shadow_vfta[index] &= ~(1 << bit); --adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); } static void @@ -5108,25 +3099,38 @@ em_setup_vlan_hw_support(struct adapter *adapter) } static void -em_enable_intr(struct adapter *adapter) +em_if_enable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; u32 ims_mask = IMS_ENABLE_MASK; if (hw->mac.type == e1000_82574) { E1000_WRITE_REG(hw, EM_EIAC, adapter->ims); ims_mask |= adapter->ims; - } + } if (adapter->intr_type == IFLIB_INTR_MSIX && hw->mac.type >= igb_mac_min) { + u32 mask = (adapter->que_mask | adapter->link_mask); + + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); + E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); + ims_mask = E1000_IMS_LSC; + } + E1000_WRITE_REG(hw, E1000_IMS, ims_mask); } static void -em_disable_intr(struct adapter *adapter) +em_if_disable_intr(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - if (hw->mac.type == e1000_82574) - E1000_WRITE_REG(hw, EM_EIAC, 0); + if (adapter->intr_type == IFLIB_INTR_MSIX) { + if (hw->mac.type >= igb_mac_min) + E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); + E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); + } E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); } @@ -5245,15 +3249,34 @@ em_is_valid_ether_addr(u8 *addr) ** later use. */ static void -em_get_wakeup(device_t dev) +em_get_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); u16 eeprom_data = 0, device_id, apme_mask; adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); apme_mask = EM_EEPROM_APME; switch (adapter->hw.mac.type) { + case e1000_82542: + case e1000_82543: + break; + case e1000_82544: + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL2_REG, 1, &eeprom_data); + apme_mask = EM_82544_APME; + break; + case e1000_82546: + case e1000_82546_rev_3: + if (adapter->hw.bus.func == 1) { + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); + break; + } else + e1000_read_nvm(&adapter->hw, + NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); + break; case e1000_82573: case e1000_82583: adapter->has_amt = TRUE; @@ -5274,8 +3297,6 @@ em_get_wakeup(device_t dev) case e1000_ich10lan: case e1000_pchlan: case e1000_pch2lan: - case e1000_pch_lpt: - case e1000_pch_spt: apme_mask = E1000_WUC_APME; adapter->has_amt = TRUE; eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); @@ -5294,6 +3315,25 @@ em_get_wakeup(device_t dev) */ device_id = pci_get_device(dev); switch (device_id) { + case E1000_DEV_ID_82546GB_PCIE: + adapter->wol = 0; + break; + case E1000_DEV_ID_82546EB_FIBER: + case E1000_DEV_ID_82546GB_FIBER: + /* Wake events only supported on port A for dual fiber + * regardless of eeprom setting */ + if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & + E1000_STATUS_FUNC_1) + adapter->wol = 0; + break; + case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: + /* if quad port adapter, disable WoL on all but port A */ + if (global_quad_port_a != 0) + adapter->wol = 0; + /* Reset for multiple quad port adapters */ + if (++global_quad_port_a == 4) + global_quad_port_a = 0; + break; case E1000_DEV_ID_82571EB_FIBER: /* Wake events only supported on port A for dual fiber * regardless of eeprom setting */ @@ -5320,11 +3360,12 @@ em_get_wakeup(device_t dev) * Enable PCI Wake On Lan capability */ static void -em_enable_wakeup(device_t dev) +em_enable_wakeup(if_ctx_t ctx) { - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - u32 pmc, ctrl, ctrl_ext, rctl, wuc; + struct adapter *adapter = iflib_get_softc(ctx); + device_t dev = iflib_get_dev(ctx); + if_t ifp = iflib_get_ifp(ctx); + u32 pmc, ctrl, ctrl_ext, rctl; u16 status; if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0)) @@ -5334,9 +3375,7 @@ em_enable_wakeup(device_t dev) ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); - wuc = E1000_READ_REG(&adapter->hw, E1000_WUC); - wuc |= E1000_WUC_PME_EN; - E1000_WRITE_REG(&adapter->hw, E1000_WUC, wuc); + E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); if ((adapter->hw.mac.type == e1000_ich8lan) || (adapter->hw.mac.type == e1000_pchlan) || @@ -5367,10 +3406,8 @@ em_enable_wakeup(device_t dev) E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); } - if ((adapter->hw.mac.type == e1000_pchlan) || - (adapter->hw.mac.type == e1000_pch2lan) || - (adapter->hw.mac.type == e1000_pch_lpt) || - (adapter->hw.mac.type == e1000_pch_spt)) { + if ((adapter->hw.mac.type == e1000_pchlan) || + (adapter->hw.mac.type == e1000_pch2lan)) { if (em_enable_phy_wakeup(adapter)) return; } else { @@ -5466,11 +3503,10 @@ em_enable_phy_wakeup(struct adapter *adapter) } static void -em_led_func(void *arg, int onoff) +em_if_led_func(if_ctx_t ctx, int onoff) { - struct adapter *adapter = arg; + struct adapter *adapter = iflib_get_softc(ctx); - EM_CORE_LOCK(adapter); if (onoff) { e1000_setup_led(&adapter->hw); e1000_led_on(&adapter->hw); @@ -5478,7 +3514,6 @@ em_led_func(void *arg, int onoff) e1000_led_off(&adapter->hw); e1000_cleanup_led(&adapter->hw); } - EM_CORE_UNLOCK(adapter); } /* @@ -5609,11 +3644,10 @@ em_update_stats_counters(struct adapter *adapter) } static uint64_t -em_get_counter(if_t ifp, ift_counter cnt) +em_if_get_counter(if_ctx_t ctx, ift_counter cnt) { - struct adapter *adapter; - - adapter = if_getsoftc(ifp); + struct adapter *adapter = iflib_get_softc(ctx); + struct ifnet *ifp = iflib_get_ifp(ctx); switch (cnt) { case IFCOUNTER_COLLISIONS: @@ -5649,11 +3683,10 @@ em_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) static void em_add_hw_stats(struct adapter *adapter) { - device_t dev = adapter->dev; - - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - + device_t dev = iflib_get_dev(adapter->ctx); + struct em_tx_queue *tx_que = adapter->tx_queues; + struct em_rx_queue *rx_que = adapter->rx_queues; + struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid *tree = device_get_sysctl_tree(dev); struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); @@ -5700,7 +3733,8 @@ em_add_hw_stats(struct adapter *adapter) CTLFLAG_RD, &adapter->hw.fc.low_water, 0, "Flow Control Low Watermark"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, tx_que++) { + struct tx_ring *txr = &tx_que->txr; snprintf(namebuf, QUEUE_NAME_LEN, "queue_tx_%d", i); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "TX Queue Name"); @@ -5722,8 +3756,11 @@ em_add_hw_stats(struct adapter *adapter) SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); + } - snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", i); + for (int j = 0; j < adapter->rx_num_queues; j++, rx_que++) { + struct rx_ring *rxr = &rx_que->rxr; + snprintf(namebuf, QUEUE_NAME_LEN, "queue_rx_%d", j); queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, CTLFLAG_RD, NULL, "RX Queue Name"); queue_list = SYSCTL_CHILDREN(queue_node); @@ -6010,7 +4047,6 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) adapter = info->adapter; - EM_CORE_LOCK(adapter); regval = E1000_READ_OFFSET(&adapter->hw, info->offset); regval = (regval & ~0xffff) | (ticks & 0xffff); /* Handle a few special cases. */ @@ -6027,7 +4063,6 @@ em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) break; } E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); - EM_CORE_UNLOCK(adapter); return (0); } @@ -6112,10 +4147,9 @@ em_sysctl_eee(SYSCTL_HANDLER_ARGS) error = sysctl_handle_int(oidp, &value, 0, req); if (error || req->newptr == NULL) return (error); - EM_CORE_LOCK(adapter); adapter->hw.dev_spec.ich8lan.eee_disable = (value != 0); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); + em_if_init(adapter->ctx); + return (0); } @@ -6148,8 +4182,8 @@ static void em_print_debug_info(struct adapter *adapter) { device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; + struct tx_ring *txr = &adapter->tx_queues->txr; + struct rx_ring *rxr = &adapter->rx_queues->rxr; if (if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) printf("Interface is RUNNING "); @@ -6161,38 +4195,33 @@ em_print_debug_info(struct adapter *adapter) else printf("and ACTIVE\n"); - for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { + for (int i = 0; i < adapter->tx_num_queues; i++, txr++) { device_printf(dev, "TX Queue %d ------\n", i); device_printf(dev, "hw tdh = %d, hw tdt = %d\n", E1000_READ_REG(&adapter->hw, E1000_TDH(i)), E1000_READ_REG(&adapter->hw, E1000_TDT(i))); - device_printf(dev, "Tx Queue Status = %d\n", txr->busy); - device_printf(dev, "TX descriptors avail = %d\n", - txr->tx_avail); - device_printf(dev, "Tx Descriptors avail failure = %ld\n", - txr->no_desc_avail); - device_printf(dev, "RX Queue %d ------\n", i); + + } + for (int j=0; j < adapter->rx_num_queues; j++, rxr++) { + device_printf(dev, "RX Queue %d ------\n", j); device_printf(dev, "hw rdh = %d, hw rdt = %d\n", - E1000_READ_REG(&adapter->hw, E1000_RDH(i)), - E1000_READ_REG(&adapter->hw, E1000_RDT(i))); - device_printf(dev, "RX discarded packets = %ld\n", - rxr->rx_discarded); - device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check); - device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh); + E1000_READ_REG(&adapter->hw, E1000_RDH(j)), + E1000_READ_REG(&adapter->hw, E1000_RDT(j))); } } -#ifdef EM_MULTIQUEUE + /* * 82574 only: * Write a new value to the EEPROM increasing the number of MSIX * vectors from 3 to 5, for proper multiqueue support. */ static void -em_enable_vectors_82574(struct adapter *adapter) +em_enable_vectors_82574(if_ctx_t ctx) { + struct adapter *adapter = iflib_get_softc(ctx); struct e1000_hw *hw = &adapter->hw; - device_t dev = adapter->dev; + device_t dev = iflib_get_dev(ctx); u16 edata; e1000_read_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); @@ -6207,7 +4236,7 @@ em_enable_vectors_82574(struct adapter *adapter) device_printf(dev, "Writing to eeprom: done\n"); } } -#endif + #ifdef DDB DB_COMMAND(em_reset_dev, em_ddb_reset_dev) @@ -6223,9 +4252,7 @@ DB_COMMAND(em_reset_dev, em_ddb_reset_dev) dev = devclass_get_device(dc, index); if (device_get_driver(dev) == &em_driver) { struct adapter *adapter = device_get_softc(dev); - EM_CORE_LOCK(adapter); - em_init_locked(adapter); - EM_CORE_UNLOCK(adapter); + em_if_init(adapter->ctx); } } } diff --git a/sys/dev/e1000/if_em.h b/sys/dev/e1000/if_em.h index 2a2bf2ccb7ab..635d172bf458 100644 --- a/sys/dev/e1000/if_em.h +++ b/sys/dev/e1000/if_em.h @@ -1,36 +1,93 @@ -/****************************************************************************** +/*- + * Copyright (c) 2016 Matt Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ /*$FreeBSD$*/ +#include "opt_em.h" +#include "opt_ddb.h" +#include "opt_inet.h" +#include "opt_inet6.h" + +#ifdef HAVE_KERNEL_OPTION_HEADERS +#include "opt_device_polling.h" +#endif + +#include +#include +#ifdef DDB +#include +#include +#endif +#if __FreeBSD_version >= 800000 +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "e1000_api.h" +#include "e1000_82571.h" +#include "ifdi_if.h" #ifndef _EM_H_DEFINED_ @@ -51,13 +108,10 @@ * desscriptors should meet the following condition. * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 */ -#define EM_MIN_TXD 80 +#define EM_MIN_TXD 128 #define EM_MAX_TXD 4096 -#ifdef EM_MULTIQUEUE -#define EM_DEFAULT_TXD 4096 -#else -#define EM_DEFAULT_TXD 1024 -#endif +#define EM_DEFAULT_TXD 1024 +#define EM_DEFAULT_MULTI_TXD 4096 /* * EM_RXD - Maximum number of receive Descriptors @@ -72,13 +126,10 @@ * desscriptors should meet the following condition. * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 */ -#define EM_MIN_RXD 80 +#define EM_MIN_RXD 128 #define EM_MAX_RXD 4096 -#ifdef EM_MULTIQUEUE -#define EM_DEFAULT_RXD 4096 -#else -#define EM_DEFAULT_RXD 1024 -#endif +#define EM_DEFAULT_RXD 1024 +#define EM_DEFAULT_MULTI_RXD 4096 /* * EM_TIDV - Transmit Interrupt Delay Value @@ -148,17 +199,6 @@ #define EM_RADV 64 #endif -/* - * This parameter controls the max duration of transmit watchdog. - */ -#define EM_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. - */ -#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8) - /* * This parameter controls whether or not autonegotation is enabled. * 0 - Disable autonegotiation @@ -221,6 +261,18 @@ #define PCICFG_DESC_RING_STATUS 0xe4 #define FLUSH_DESC_REQUIRED 0x100 + +#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ + ((hw->mac.type <= e1000_82576) ? 16 : 8)) +#define IGB_RX_HTHRESH 8 +#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ + (adapter->intr_type == IFLIB_INTR_MSIX)) ? 1 : 4) + +#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) +#define IGB_TX_HTHRESH 1 +#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ + (adapter->intr_type == IFLIB_INTR_MSIX) ? 1 : 16) + /* * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will @@ -242,6 +294,7 @@ #define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) #define EM_BAR_TYPE_MASK 0x00000001 #define EM_BAR_TYPE_MMEM 0x00000000 +#define EM_BAR_TYPE_IO 0x00000001 #define EM_BAR_TYPE_FLASH 0x0014 #define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) #define EM_BAR_MEM_TYPE_MASK 0x00000006 @@ -277,7 +330,11 @@ #define EM_MSIX_LINK 0x01000000 /* For 82574 use */ #define ETH_ZLEN 60 #define ETH_ADDR_LEN 6 -#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ +#define EM_CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ +#define IGB_CSUM_OFFLOAD 0x0E0F /* Offload bits in mbuf flag */ + +#define IGB_PKTTYPE_MASK 0x0000FFF0 +#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ /* * 82574 has a nonstandard address for EIAC @@ -295,19 +352,6 @@ #define EM_NVM_MSIX_N_MASK (0x7 << EM_NVM_MSIX_N_SHIFT) #define EM_NVM_MSIX_N_SHIFT 7 -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct em_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - struct adapter; struct em_int_delay_info { @@ -321,35 +365,31 @@ struct em_int_delay_info { */ struct tx_ring { struct adapter *adapter; - struct mtx tx_mtx; - char mtx_name[16]; + struct em_tx_queue *que; u32 me; - u32 msix; - u32 ims; int busy; - struct em_dma_alloc txdma; struct e1000_tx_desc *tx_base; - struct task tx_task; - struct taskqueue *tq; - u32 next_avail_desc; - u32 next_to_clean; + uint64_t tx_paddr; struct em_txbuffer *tx_buffers; - volatile u16 tx_avail; u32 tx_tso; /* last tx was tso */ - u16 last_hw_offload; - u8 last_hw_ipcso; - u8 last_hw_ipcss; - u8 last_hw_tucso; - u8 last_hw_tucss; -#if __FreeBSD_version >= 800000 - struct buf_ring *br; -#endif + /* Interrupt resources */ - bus_dma_tag_t txtag; void *tag; struct resource *res; unsigned long tx_irq; unsigned long no_desc_avail; + + /* Saved csum offloading context information */ + int csum_flags; + int csum_lhlen; + int csum_iphlen; + + int csum_thlen; + int csum_mss; + int csum_pktlen; + + uint32_t csum_txd_upper; + uint32_t csum_txd_lower; /* last field */ }; /* @@ -357,26 +397,15 @@ struct tx_ring { */ struct rx_ring { struct adapter *adapter; + struct em_rx_queue *que; u32 me; - u32 msix; - u32 ims; - struct mtx rx_mtx; - char mtx_name[16]; u32 payload; - struct task rx_task; - struct taskqueue *tq; union e1000_rx_desc_extended *rx_base; - struct em_dma_alloc rxdma; - u32 next_to_refresh; - u32 next_to_check; - struct em_rxbuffer *rx_buffers; - struct mbuf *fmp; - struct mbuf *lmp; + uint64_t rx_paddr; /* Interrupt resources */ void *tag; struct resource *res; - bus_dma_tag_t rxtag; bool discard; /* Soft stats */ @@ -386,62 +415,68 @@ struct rx_ring { unsigned long rx_bytes; }; +struct em_tx_queue { + struct adapter *adapter; + u32 msix; + u32 eims; /* This queue's EIMS bit */ + u32 me; + struct tx_ring txr; +}; + +struct em_rx_queue { + struct adapter *adapter; + u32 me; + u32 msix; + u32 eims; + struct rx_ring rxr; + u64 irqs; + struct if_irq que_irq; +}; /* Our adapter structure */ struct adapter { - if_t ifp; + struct ifnet *ifp; struct e1000_hw hw; + if_softc_ctx_t shared; + if_ctx_t ctx; +#define tx_num_queues shared->isc_ntxqsets +#define rx_num_queues shared->isc_nrxqsets +#define intr_type shared->isc_intr /* FreeBSD operating-system-specific structures. */ struct e1000_osdep osdep; - device_t dev; + struct device *dev; struct cdev *led_dev; + struct em_tx_queue *tx_queues; + struct em_rx_queue *rx_queues; + struct if_irq irq; + struct resource *memory; struct resource *flash; - struct resource *msix_mem; + struct resource *ioport; + int io_rid; struct resource *res; void *tag; u32 linkvec; u32 ivars; - struct ifmedia media; - struct callout timer; + struct ifmedia *media; int msix; int if_flags; - int max_frame_size; int min_frame_size; - struct mtx core_mtx; int em_insert_vlan_header; u32 ims; bool in_detach; /* Task for FAST handling */ - struct task link_task; - struct task que_task; - struct taskqueue *tq; /* private task queue */ + struct grouptask link_task; - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - - u16 num_vlans; - u8 num_queues; - - /* - * Transmit rings: - * Allocated at run time, an array of rings. - */ - struct tx_ring *tx_rings; - int num_tx_desc; + u16 num_vlans; u32 txd_cmd; - /* - * Receive rings: - * Allocated at run time, an array of rings. - */ - struct rx_ring *rx_rings; - int num_rx_desc; + u32 tx_process_limit; u32 rx_process_limit; u32 rx_mbuf_sz; @@ -467,7 +502,12 @@ struct adapter { u16 link_speed; u16 link_duplex; u32 smartspeed; + u32 dmac; + int link_mask; + u64 que_mask; + + struct em_int_delay_info tx_int_delay; struct em_int_delay_info tx_abs_int_delay; struct em_int_delay_info rx_int_delay; @@ -502,33 +542,9 @@ typedef struct _em_vendor_info_t { } em_vendor_info_t; struct em_txbuffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ + int eop; }; -struct em_rxbuffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ - bus_addr_t paddr; -}; - - -/* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -e1000_rx_unrefreshed(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - - if (rxr->next_to_check > rxr->next_to_refresh) - return (rxr->next_to_check - rxr->next_to_refresh - 1); - else - return ((adapter->num_rx_desc + rxr->next_to_check) - - rxr->next_to_refresh - 1); -} #define EM_CORE_LOCK_INIT(_sc, _name) \ mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF) diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c deleted file mode 100644 index 3c9644d9b03b..000000000000 --- a/sys/dev/e1000/if_igb.c +++ /dev/null @@ -1,6450 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#include "opt_inet.h" -#include "opt_inet6.h" -#include "opt_rss.h" - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include "opt_device_polling.h" -#include "opt_altq.h" -#endif - -#include "if_igb.h" - -/********************************************************************* - * Driver version: - *********************************************************************/ -char igb_driver_version[] = "2.5.3-k"; - - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into e1000_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static igb_vendor_info_t igb_vendor_info_array[] = -{ - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_FLASHLESS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES_FLASHLESS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS, 0, 0, 0}, - {IGB_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII, 0, 0, 0}, - /* required last entry */ - {0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *igb_strings[] = { - "Intel(R) PRO/1000 Network Connection" -}; - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int igb_probe(device_t); -static int igb_attach(device_t); -static int igb_detach(device_t); -static int igb_shutdown(device_t); -static int igb_suspend(device_t); -static int igb_resume(device_t); -#ifndef IGB_LEGACY_TX -static int igb_mq_start(struct ifnet *, struct mbuf *); -static int igb_mq_start_locked(struct ifnet *, struct tx_ring *); -static void igb_qflush(struct ifnet *); -static void igb_deferred_mq_start(void *, int); -#else -static void igb_start(struct ifnet *); -static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); -#endif -static int igb_ioctl(struct ifnet *, u_long, caddr_t); -static uint64_t igb_get_counter(if_t, ift_counter); -static void igb_init(void *); -static void igb_init_locked(struct adapter *); -static void igb_stop(void *); -static void igb_media_status(struct ifnet *, struct ifmediareq *); -static int igb_media_change(struct ifnet *); -static void igb_identify_hardware(struct adapter *); -static int igb_allocate_pci_resources(struct adapter *); -static int igb_allocate_msix(struct adapter *); -static int igb_allocate_legacy(struct adapter *); -static int igb_setup_msix(struct adapter *); -static void igb_free_pci_resources(struct adapter *); -static void igb_local_timer(void *); -static void igb_reset(struct adapter *); -static int igb_setup_interface(device_t, struct adapter *); -static int igb_allocate_queues(struct adapter *); -static void igb_configure_queues(struct adapter *); - -static int igb_allocate_transmit_buffers(struct tx_ring *); -static void igb_setup_transmit_structures(struct adapter *); -static void igb_setup_transmit_ring(struct tx_ring *); -static void igb_initialize_transmit_units(struct adapter *); -static void igb_free_transmit_structures(struct adapter *); -static void igb_free_transmit_buffers(struct tx_ring *); - -static int igb_allocate_receive_buffers(struct rx_ring *); -static int igb_setup_receive_structures(struct adapter *); -static int igb_setup_receive_ring(struct rx_ring *); -static void igb_initialize_receive_units(struct adapter *); -static void igb_free_receive_structures(struct adapter *); -static void igb_free_receive_buffers(struct rx_ring *); -static void igb_free_receive_ring(struct rx_ring *); - -static void igb_enable_intr(struct adapter *); -static void igb_disable_intr(struct adapter *); -static void igb_update_stats_counters(struct adapter *); -static bool igb_txeof(struct tx_ring *); - -static __inline void igb_rx_discard(struct rx_ring *, int); -static __inline void igb_rx_input(struct rx_ring *, - struct ifnet *, struct mbuf *, u32); - -static bool igb_rxeof(struct igb_queue *, int, int *); -static void igb_rx_checksum(u32, struct mbuf *, u32); -static int igb_tx_ctx_setup(struct tx_ring *, - struct mbuf *, u32 *, u32 *); -static int igb_tso_setup(struct tx_ring *, - struct mbuf *, u32 *, u32 *); -static void igb_set_promisc(struct adapter *); -static void igb_disable_promisc(struct adapter *); -static void igb_set_multi(struct adapter *); -static void igb_update_link_status(struct adapter *); -static void igb_refresh_mbufs(struct rx_ring *, int); - -static void igb_register_vlan(void *, struct ifnet *, u16); -static void igb_unregister_vlan(void *, struct ifnet *, u16); -static void igb_setup_vlan_hw_support(struct adapter *); - -static int igb_xmit(struct tx_ring *, struct mbuf **); -static int igb_dma_malloc(struct adapter *, bus_size_t, - struct igb_dma_alloc *, int); -static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); -static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); -static void igb_print_nvm_info(struct adapter *); -static int igb_is_valid_ether_addr(u8 *); -static void igb_add_hw_stats(struct adapter *); - -static void igb_vf_init_stats(struct adapter *); -static void igb_update_vf_stats_counters(struct adapter *); - -/* Management and WOL Support */ -static void igb_init_manageability(struct adapter *); -static void igb_release_manageability(struct adapter *); -static void igb_get_hw_control(struct adapter *); -static void igb_release_hw_control(struct adapter *); -static void igb_enable_wakeup(device_t); -static void igb_led_func(void *, int); - -static int igb_irq_fast(void *); -static void igb_msix_que(void *); -static void igb_msix_link(void *); -static void igb_handle_que(void *context, int pending); -static void igb_handle_link(void *context, int pending); -static void igb_handle_link_locked(struct adapter *); - -static void igb_set_sysctl_value(struct adapter *, const char *, - const char *, int *, int); -static int igb_set_flowcntl(SYSCTL_HANDLER_ARGS); -static int igb_sysctl_dmac(SYSCTL_HANDLER_ARGS); -static int igb_sysctl_eee(SYSCTL_HANDLER_ARGS); - -#ifdef DEVICE_POLLING -static poll_handler_t igb_poll; -#endif /* POLLING */ - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t igb_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, igb_probe), - DEVMETHOD(device_attach, igb_attach), - DEVMETHOD(device_detach, igb_detach), - DEVMETHOD(device_shutdown, igb_shutdown), - DEVMETHOD(device_suspend, igb_suspend), - DEVMETHOD(device_resume, igb_resume), - DEVMETHOD_END -}; - -static driver_t igb_driver = { - "igb", igb_methods, sizeof(struct adapter), -}; - -static devclass_t igb_devclass; -DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); -MODULE_DEPEND(igb, pci, 1, 1, 1); -MODULE_DEPEND(igb, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(igb, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Tunable default values. - *********************************************************************/ - -static SYSCTL_NODE(_hw, OID_AUTO, igb, CTLFLAG_RD, 0, "IGB driver parameters"); - -/* Descriptor defaults */ -static int igb_rxd = IGB_DEFAULT_RXD; -static int igb_txd = IGB_DEFAULT_TXD; -SYSCTL_INT(_hw_igb, OID_AUTO, rxd, CTLFLAG_RDTUN, &igb_rxd, 0, - "Number of receive descriptors per queue"); -SYSCTL_INT(_hw_igb, OID_AUTO, txd, CTLFLAG_RDTUN, &igb_txd, 0, - "Number of transmit descriptors per queue"); - -/* -** AIM: Adaptive Interrupt Moderation -** which means that the interrupt rate -** is varied over time based on the -** traffic for that interrupt vector -*/ -static int igb_enable_aim = TRUE; -SYSCTL_INT(_hw_igb, OID_AUTO, enable_aim, CTLFLAG_RWTUN, &igb_enable_aim, 0, - "Enable adaptive interrupt moderation"); - -/* - * MSIX should be the default for best performance, - * but this allows it to be forced off for testing. - */ -static int igb_enable_msix = 1; -SYSCTL_INT(_hw_igb, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &igb_enable_msix, 0, - "Enable MSI-X interrupts"); - -/* -** Tuneable Interrupt rate -*/ -static int igb_max_interrupt_rate = 8000; -SYSCTL_INT(_hw_igb, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, - &igb_max_interrupt_rate, 0, "Maximum interrupts per second"); - -#ifndef IGB_LEGACY_TX -/* -** Tuneable number of buffers in the buf-ring (drbr_xxx) -*/ -static int igb_buf_ring_size = IGB_BR_SIZE; -SYSCTL_INT(_hw_igb, OID_AUTO, buf_ring_size, CTLFLAG_RDTUN, - &igb_buf_ring_size, 0, "Size of the bufring"); -#endif - -/* -** Header split causes the packet header to -** be dma'd to a separate mbuf from the payload. -** this can have memory alignment benefits. But -** another plus is that small packets often fit -** into the header and thus use no cluster. Its -** a very workload dependent type feature. -*/ -static int igb_header_split = FALSE; -SYSCTL_INT(_hw_igb, OID_AUTO, header_split, CTLFLAG_RDTUN, &igb_header_split, 0, - "Enable receive mbuf header split"); - -/* -** This will autoconfigure based on the -** number of CPUs and max supported -** MSIX messages if left at 0. -*/ -static int igb_num_queues = 0; -SYSCTL_INT(_hw_igb, OID_AUTO, num_queues, CTLFLAG_RDTUN, &igb_num_queues, 0, - "Number of queues to configure, 0 indicates autoconfigure"); - -/* -** Global variable to store last used CPU when binding queues -** to CPUs in igb_allocate_msix. Starts at CPU_FIRST and increments when a -** queue is bound to a cpu. -*/ -static int igb_last_bind_cpu = -1; - -/* How many packets rxeof tries to clean at a time */ -static int igb_rx_process_limit = 100; -SYSCTL_INT(_hw_igb, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, - &igb_rx_process_limit, 0, - "Maximum number of received packets to process at a time, -1 means unlimited"); - -/* How many packets txeof tries to clean at a time */ -static int igb_tx_process_limit = -1; -SYSCTL_INT(_hw_igb, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, - &igb_tx_process_limit, 0, - "Maximum number of sent packets to process at a time, -1 means unlimited"); - -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include -#endif /* DEV_NETMAP */ -/********************************************************************* - * Device identification routine - * - * igb_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -igb_probe(device_t dev) -{ - char adapter_name[256]; - uint16_t pci_vendor_id = 0; - uint16_t pci_device_id = 0; - uint16_t pci_subvendor_id = 0; - uint16_t pci_subdevice_id = 0; - igb_vendor_info_t *ent; - - INIT_DEBUGOUT("igb_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != IGB_INTEL_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = igb_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == 0)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == 0))) { - sprintf(adapter_name, "%s, Version - %s", - igb_strings[ent->index], - igb_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -igb_attach(device_t dev) -{ - struct adapter *adapter; - int error = 0; - u16 eeprom_data; - - INIT_DEBUGOUT("igb_attach: begin"); - - if (resource_disabled("igb", device_get_unit(dev))) { - device_printf(dev, "Disabled by device hint\n"); - return (ENXIO); - } - - adapter = device_get_softc(dev); - adapter->dev = adapter->osdep.dev = dev; - IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); - - /* SYSCTLs */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, - igb_sysctl_nvm_info, "I", "NVM Information"); - - igb_set_sysctl_value(adapter, "enable_aim", - "Interrupt Moderation", &adapter->enable_aim, - igb_enable_aim); - - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "fc", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_set_flowcntl, "I", "Flow Control"); - - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); - - /* Determine hardware and mac info */ - igb_identify_hardware(adapter); - - /* Setup PCI resources */ - if (igb_allocate_pci_resources(adapter)) { - device_printf(dev, "Allocation of PCI resources failed\n"); - error = ENXIO; - goto err_pci; - } - - /* Do Shared Code initialization */ - if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { - device_printf(dev, "Setup of Shared code failed\n"); - error = ENXIO; - goto err_pci; - } - - e1000_get_bus_info(&adapter->hw); - - /* Sysctls for limiting the amount of work done in the taskqueues */ - igb_set_sysctl_value(adapter, "rx_processing_limit", - "max number of rx packets to process", - &adapter->rx_process_limit, igb_rx_process_limit); - - igb_set_sysctl_value(adapter, "tx_processing_limit", - "max number of tx packets to process", - &adapter->tx_process_limit, igb_tx_process_limit); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || - (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - IGB_DEFAULT_TXD, igb_txd); - adapter->num_tx_desc = IGB_DEFAULT_TXD; - } else - adapter->num_tx_desc = igb_txd; - if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || - (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - IGB_DEFAULT_RXD, igb_rxd); - adapter->num_rx_desc = IGB_DEFAULT_RXD; - } else - adapter->num_rx_desc = igb_rxd; - - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_wait_to_complete = FALSE; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - - /* Copper options */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - adapter->hw.phy.mdix = AUTO_ALL_MODES; - adapter->hw.phy.disable_polarity_correction = FALSE; - adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; - } - - /* - * Set the frame limits assuming - * standard ethernet sized frames. - */ - adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; - - /* - ** Allocate and Setup Queues - */ - if (igb_allocate_queues(adapter)) { - error = ENOMEM; - goto err_pci; - } - - /* Allocate the appropriate stats memory */ - if (adapter->vf_ifp) { - adapter->stats = - (struct e1000_vf_stats *)malloc(sizeof \ - (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO); - igb_vf_init_stats(adapter); - } else - adapter->stats = - (struct e1000_hw_stats *)malloc(sizeof \ - (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->stats == NULL) { - device_printf(dev, "Can not allocate stats memory\n"); - error = ENOMEM; - goto err_late; - } - - /* Allocate multicast array memory. */ - adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); - if (adapter->mta == NULL) { - device_printf(dev, "Can not allocate multicast setup array\n"); - error = ENOMEM; - goto err_late; - } - - /* Some adapter-specific advanced features */ - if (adapter->hw.mac.type >= e1000_i350) { - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "dmac", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_sysctl_dmac, "I", "DMA Coalesce"); - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "eee_disabled", CTLTYPE_INT|CTLFLAG_RW, - adapter, 0, igb_sysctl_eee, "I", - "Disable Energy Efficient Ethernet"); - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); - else - e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); - } - } - - /* - ** Start from a known state, this is - ** important in reading the nvm and - ** mac from that. - */ - e1000_reset_hw(&adapter->hw); - - /* Make sure we have a good EEPROM before we read from it */ - if (((adapter->hw.mac.type != e1000_i210) && - (adapter->hw.mac.type != e1000_i211)) && - (e1000_validate_nvm_checksum(&adapter->hw) < 0)) { - /* - ** Some PCI-E parts fail the first check due to - ** the link being in sleep state, call it again, - ** if it fails a second time its a real issue. - */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - device_printf(dev, - "The EEPROM Checksum Is Not Valid\n"); - error = EIO; - goto err_late; - } - } - - /* - ** Copy the permanent MAC address out of the EEPROM - */ - if (e1000_read_mac_addr(&adapter->hw) < 0) { - device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); - error = EIO; - goto err_late; - } - - /* Check its sanity */ - if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { - if (adapter->vf_ifp) { - u8 addr[ETHER_ADDR_LEN]; - arc4rand(&addr, sizeof(addr), 0); - addr[0] &= 0xFE; - addr[0] |= 0x02; - bcopy(addr, adapter->hw.mac.addr, sizeof(addr)); - } else { - device_printf(dev, "Invalid MAC address\n"); - error = EIO; - goto err_late; - } - } - - /* Setup OS specific network interface */ - if (igb_setup_interface(dev, adapter) != 0) - goto err_late; - - /* Now get a good starting state */ - igb_reset(adapter); - - /* Initialize statistics */ - igb_update_stats_counters(adapter); - - adapter->hw.mac.get_link_status = 1; - igb_update_link_status(adapter); - - /* Indicate SOL/IDER usage */ - if (e1000_check_reset_block(&adapter->hw)) - device_printf(dev, - "PHY reset is blocked due to SOL/IDER session.\n"); - - /* Determine if we have to control management hardware */ - adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); - - /* - * Setup Wake-on-Lan - */ - /* APME bit in EEPROM is mapped to WUC.APME */ - eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; - if (eeprom_data) - adapter->wol = E1000_WUFC_MAG; - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - - igb_add_hw_stats(adapter); - - /* Tell the stack that the interface is not active */ - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - adapter->ifp->if_drv_flags |= IFF_DRV_OACTIVE; - - adapter->led_dev = led_create(igb_led_func, adapter, - device_get_nameunit(dev)); - - /* - ** Configure Interrupts - */ - if ((adapter->msix > 1) && (igb_enable_msix)) - error = igb_allocate_msix(adapter); - else /* MSI or Legacy */ - error = igb_allocate_legacy(adapter); - if (error) - goto err_late; - -#ifdef DEV_NETMAP - igb_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - INIT_DEBUGOUT("igb_attach: end"); - - return (0); - -err_late: - if (igb_detach(dev) == 0) /* igb_detach() already did the cleanup */ - return(error); - igb_free_transmit_structures(adapter); - igb_free_receive_structures(adapter); - igb_release_hw_control(adapter); -err_pci: - igb_free_pci_resources(adapter); - if (adapter->ifp != NULL) - if_free(adapter->ifp); - free(adapter->mta, M_DEVBUF); - IGB_CORE_LOCK_DESTROY(adapter); - - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -igb_detach(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - struct ifnet *ifp = adapter->ifp; - - INIT_DEBUGOUT("igb_detach: begin"); - - /* Make sure VLANS are not using driver */ - if (adapter->ifp->if_vlantrunk != NULL) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - - ether_ifdetach(adapter->ifp); - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - -#ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - IGB_CORE_LOCK(adapter); - adapter->in_detach = 1; - igb_stop(adapter); - IGB_CORE_UNLOCK(adapter); - - e1000_phy_hw_reset(&adapter->hw); - - /* Give control back to firmware */ - igb_release_manageability(adapter); - igb_release_hw_control(adapter); - - if (adapter->wol) { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - igb_enable_wakeup(dev); - } - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - callout_drain(&adapter->timer); - -#ifdef DEV_NETMAP - netmap_detach(adapter->ifp); -#endif /* DEV_NETMAP */ - igb_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - igb_free_transmit_structures(adapter); - igb_free_receive_structures(adapter); - if (adapter->mta != NULL) - free(adapter->mta, M_DEVBUF); - - IGB_CORE_LOCK_DESTROY(adapter); - - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -igb_shutdown(device_t dev) -{ - return igb_suspend(dev); -} - -/* - * Suspend/resume device methods. - */ -static int -igb_suspend(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - - IGB_CORE_LOCK(adapter); - - igb_stop(adapter); - - igb_release_manageability(adapter); - igb_release_hw_control(adapter); - - if (adapter->wol) { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - igb_enable_wakeup(dev); - } - - IGB_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); -} - -static int -igb_resume(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; - - IGB_CORE_LOCK(adapter); - igb_init_locked(adapter); - igb_init_manageability(adapter); - - if ((ifp->if_flags & IFF_UP) && - (ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - } - IGB_CORE_UNLOCK(adapter); - - return bus_generic_resume(dev); -} - - -#ifdef IGB_LEGACY_TX - -/********************************************************************* - * Transmit entry point - * - * igb_start is called by the stack to initiate a transmit. - * The driver will remain in this routine as long as there are - * packets to transmit and transmit resources are available. - * In case resources are not available stack is notified and - * the packet is requeued. - **********************************************************************/ - -static void -igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct mbuf *m_head; - - IGB_TX_LOCK_ASSERT(txr); - - if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - if (!adapter->link_active) - return; - - /* Call cleanup if number of TX descriptors low */ - if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) - igb_txeof(txr); - - while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { - if (txr->tx_avail <= IGB_MAX_SCATTER) { - txr->queue_status |= IGB_QUEUE_DEPLETED; - break; - } - IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (igb_xmit(txr, &m_head)) { - if (m_head != NULL) - IFQ_DRV_PREPEND(&ifp->if_snd, m_head); - if (txr->tx_avail <= IGB_MAX_SCATTER) - txr->queue_status |= IGB_QUEUE_DEPLETED; - break; - } - - /* Send a copy of the frame to the BPF listener */ - ETHER_BPF_MTAP(ifp, m_head); - - /* Set watchdog on */ - txr->watchdog_time = ticks; - txr->queue_status |= IGB_QUEUE_WORKING; - } -} - -/* - * Legacy TX driver routine, called from the - * stack, always uses tx[0], and spins for it. - * Should not be used with multiqueue tx - */ -static void -igb_start(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IGB_TX_LOCK(txr); - igb_start_locked(txr, ifp); - IGB_TX_UNLOCK(txr); - } - return; -} - -#else /* ~IGB_LEGACY_TX */ - -/* -** Multiqueue Transmit Entry: -** quick turnaround to the stack -** -*/ -static int -igb_mq_start(struct ifnet *ifp, struct mbuf *m) -{ - struct adapter *adapter = ifp->if_softc; - struct igb_queue *que; - struct tx_ring *txr; - int i, err = 0; -#ifdef RSS - uint32_t bucket_id; -#endif - - /* Which queue to use */ - /* - * When doing RSS, map it to the same outbound queue - * as the incoming flow would be mapped to. - * - * If everything is setup correctly, it should be the - * same bucket that the current CPU we're on is. - */ - if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { -#ifdef RSS - if (rss_hash2bucket(m->m_pkthdr.flowid, - M_HASHTYPE_GET(m), &bucket_id) == 0) { - /* XXX TODO: spit out something if bucket_id > num_queues? */ - i = bucket_id % adapter->num_queues; - } else { -#endif - i = m->m_pkthdr.flowid % adapter->num_queues; -#ifdef RSS - } -#endif - } else { - i = curcpu % adapter->num_queues; - } - txr = &adapter->tx_rings[i]; - que = &adapter->queues[i]; - - err = drbr_enqueue(ifp, txr->br, m); - if (err) - return (err); - if (IGB_TX_TRYLOCK(txr)) { - igb_mq_start_locked(ifp, txr); - IGB_TX_UNLOCK(txr); - } else - taskqueue_enqueue(que->tq, &txr->txq_task); - - return (0); -} - -static int -igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct mbuf *next; - int err = 0, enq = 0; - - IGB_TX_LOCK_ASSERT(txr); - - if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) || - adapter->link_active == 0) - return (ENETDOWN); - - /* Process the queue */ - while ((next = drbr_peek(ifp, txr->br)) != NULL) { - if ((err = igb_xmit(txr, &next)) != 0) { - if (next == NULL) { - /* It was freed, move forward */ - drbr_advance(ifp, txr->br); - } else { - /* - * Still have one left, it may not be - * the same since the transmit function - * may have changed it. - */ - drbr_putback(ifp, txr->br, next); - } - break; - } - drbr_advance(ifp, txr->br); - enq++; - if (next->m_flags & M_MCAST && adapter->vf_ifp) - if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); - ETHER_BPF_MTAP(ifp, next); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - } - if (enq > 0) { - /* Set the watchdog */ - txr->queue_status |= IGB_QUEUE_WORKING; - txr->watchdog_time = ticks; - } - if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) - igb_txeof(txr); - if (txr->tx_avail <= IGB_MAX_SCATTER) - txr->queue_status |= IGB_QUEUE_DEPLETED; - return (err); -} - -/* - * Called from a taskqueue to drain queued transmit packets. - */ -static void -igb_deferred_mq_start(void *arg, int pending) -{ - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - struct ifnet *ifp = adapter->ifp; - - IGB_TX_LOCK(txr); - if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); - IGB_TX_UNLOCK(txr); -} - -/* -** Flush all ring buffers -*/ -static void -igb_qflush(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct tx_ring *txr = adapter->tx_rings; - struct mbuf *m; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) - m_freem(m); - IGB_TX_UNLOCK(txr); - } - if_qflush(ifp); -} -#endif /* ~IGB_LEGACY_TX */ - -/********************************************************************* - * Ioctl entry point - * - * igb_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = ifp->if_softc; - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) - igb_init(adapter); -#ifdef INET - if (!(ifp->if_flags & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - IGB_CORE_LOCK(adapter); - max_frame_size = 9234; - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - IGB_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - ifp->if_mtu = ifr->ifr_mtu; - adapter->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - IGB_CORE_LOCK(adapter); - if (ifp->if_flags & IFF_UP) { - if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { - if ((ifp->if_flags ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - igb_disable_promisc(adapter); - igb_set_promisc(adapter); - } - } else - igb_init_locked(adapter); - } else - if (ifp->if_drv_flags & IFF_DRV_RUNNING) - igb_stop(adapter); - adapter->if_flags = ifp->if_flags; - IGB_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - IGB_CORE_LOCK(adapter); - igb_disable_intr(adapter); - igb_set_multi(adapter); -#ifdef DEVICE_POLLING - if (!(ifp->if_capenable & IFCAP_POLLING)) -#endif - igb_enable_intr(adapter); - IGB_CORE_UNLOCK(adapter); - } - break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - IGB_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - IGB_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - IGB_CORE_UNLOCK(adapter); - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); - break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ ifp->if_capenable; -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(igb_poll, ifp); - if (error) - return (error); - IGB_CORE_LOCK(adapter); - igb_disable_intr(adapter); - ifp->if_capenable |= IFCAP_POLLING; - IGB_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - IGB_CORE_LOCK(adapter); - igb_enable_intr(adapter); - ifp->if_capenable &= ~IFCAP_POLLING; - IGB_CORE_UNLOCK(adapter); - } - } -#endif -#if __FreeBSD_version >= 1000000 - /* HW cannot turn these on/off separately */ - if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) { - ifp->if_capenable ^= IFCAP_RXCSUM; - ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; - reinit = 1; - } - if (mask & IFCAP_TXCSUM) { - ifp->if_capenable ^= IFCAP_TXCSUM; - reinit = 1; - } - if (mask & IFCAP_TXCSUM_IPV6) { - ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; - reinit = 1; - } -#else - if (mask & IFCAP_HWCSUM) { - ifp->if_capenable ^= IFCAP_HWCSUM; - reinit = 1; - } -#endif - if (mask & IFCAP_TSO4) { - ifp->if_capenable ^= IFCAP_TSO4; - reinit = 1; - } - if (mask & IFCAP_TSO6) { - ifp->if_capenable ^= IFCAP_TSO6; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWFILTER) { - ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTSO) { - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; - reinit = 1; - } - if (mask & IFCAP_LRO) { - ifp->if_capenable ^= IFCAP_LRO; - reinit = 1; - } - if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) - igb_init(adapter); - VLAN_CAPABILITIES(ifp); - break; - } - - default: - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -igb_init_locked(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - device_t dev = adapter->dev; - - INIT_DEBUGOUT("igb_init: begin"); - - IGB_CORE_LOCK_ASSERT(adapter); - - igb_disable_intr(adapter); - callout_stop(&adapter->timer); - - /* Get the latest mac address, User can use a LAA */ - bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); - - /* Put the address into the Receive Address Array */ - e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - - igb_reset(adapter); - igb_update_link_status(adapter); - - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - - /* Set hardware offload abilities */ - ifp->if_hwassist = 0; - if (ifp->if_capenable & IFCAP_TXCSUM) { -#if __FreeBSD_version >= 1000000 - ifp->if_hwassist |= (CSUM_IP_TCP | CSUM_IP_UDP); - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_IP_SCTP; -#else - ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); -#if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_SCTP; -#endif -#endif - } - -#if __FreeBSD_version >= 1000000 - if (ifp->if_capenable & IFCAP_TXCSUM_IPV6) { - ifp->if_hwassist |= (CSUM_IP6_TCP | CSUM_IP6_UDP); - if (adapter->hw.mac.type != e1000_82575) - ifp->if_hwassist |= CSUM_IP6_SCTP; - } -#endif - if (ifp->if_capenable & IFCAP_TSO) - ifp->if_hwassist |= CSUM_TSO; - - /* Clear bad data from Rx FIFOs */ - e1000_rx_fifo_flush_82575(&adapter->hw); - - /* Configure for OS presence */ - igb_init_manageability(adapter); - - /* Prepare transmit descriptors and buffers */ - igb_setup_transmit_structures(adapter); - igb_initialize_transmit_units(adapter); - - /* Setup Multicast table */ - igb_set_multi(adapter); - - /* - ** Figure out the desired mbuf pool - ** for doing jumbo/packetsplit - */ - if (adapter->max_frame_size <= 2048) - adapter->rx_mbuf_sz = MCLBYTES; - else if (adapter->max_frame_size <= 4096) - adapter->rx_mbuf_sz = MJUMPAGESIZE; - else - adapter->rx_mbuf_sz = MJUM9BYTES; - - /* Prepare receive descriptors and buffers */ - if (igb_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - return; - } - igb_initialize_receive_units(adapter); - - /* Enable VLAN support */ - if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) - igb_setup_vlan_hw_support(adapter); - - /* Don't lose promiscuous settings */ - igb_set_promisc(adapter); - - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - - callout_reset(&adapter->timer, hz, igb_local_timer, adapter); - e1000_clear_hw_cntrs_base_generic(&adapter->hw); - - if (adapter->msix > 1) /* Set up queue routing */ - igb_configure_queues(adapter); - - /* this clears any pending interrupts */ - E1000_READ_REG(&adapter->hw, E1000_ICR); -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (ifp->if_capenable & IFCAP_POLLING) - igb_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - { - igb_enable_intr(adapter); - E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); - } - - /* Set Energy Efficient Ethernet */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - if (adapter->hw.mac.type == e1000_i354) - e1000_set_eee_i354(&adapter->hw, TRUE, TRUE); - else - e1000_set_eee_i350(&adapter->hw, TRUE, TRUE); - } -} - -static void -igb_init(void *arg) -{ - struct adapter *adapter = arg; - - IGB_CORE_LOCK(adapter); - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); -} - - -static void -igb_handle_que(void *context, int pending) -{ - struct igb_queue *que = context; - struct adapter *adapter = que->adapter; - struct tx_ring *txr = que->txr; - struct ifnet *ifp = adapter->ifp; - - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - bool more; - - more = igb_rxeof(que, adapter->rx_process_limit, NULL); - - IGB_TX_LOCK(txr); - igb_txeof(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - /* Do we need another? */ - if (more) { - taskqueue_enqueue(que->tq, &que->que_task); - return; - } - } - -#ifdef DEVICE_POLLING - if (ifp->if_capenable & IFCAP_POLLING) - return; -#endif - /* Reenable this interrupt */ - if (que->eims) - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); - else - igb_enable_intr(adapter); -} - -/* Deal with link in a sleepable context */ -static void -igb_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - - IGB_CORE_LOCK(adapter); - igb_handle_link_locked(adapter); - IGB_CORE_UNLOCK(adapter); -} - -static void -igb_handle_link_locked(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct ifnet *ifp = adapter->ifp; - - IGB_CORE_LOCK_ASSERT(adapter); - adapter->hw.mac.get_link_status = 1; - igb_update_link_status(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && adapter->link_active) { - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - } -} - -/********************************************************************* - * - * MSI/Legacy Deferred - * Interrupt Service routine - * - *********************************************************************/ -static int -igb_irq_fast(void *arg) -{ - struct adapter *adapter = arg; - struct igb_queue *que = adapter->queues; - u32 reg_icr; - - - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - - /* Hot eject? */ - if (reg_icr == 0xffffffff) - return FILTER_STRAY; - - /* Definitely not our interrupt. */ - if (reg_icr == 0x0) - return FILTER_STRAY; - - if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) - return FILTER_STRAY; - - /* - * Mask interrupts until the taskqueue is finished running. This is - * cheap, just assume that it is needed. This also works around the - * MSI message reordering errata on certain systems. - */ - igb_disable_intr(adapter); - taskqueue_enqueue(que->tq, &que->que_task); - - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) - taskqueue_enqueue(que->tq, &adapter->link_task); - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - return FILTER_HANDLED; -} - -#ifdef DEVICE_POLLING -#if __FreeBSD_version >= 800000 -#define POLL_RETURN_COUNT(a) (a) -static int -#else -#define POLL_RETURN_COUNT(a) -static void -#endif -igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = ifp->if_softc; - struct igb_queue *que; - struct tx_ring *txr; - u32 reg_icr, rx_done = 0; - u32 loop = IGB_MAX_LOOP; - bool more; - - IGB_CORE_LOCK(adapter); - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { - IGB_CORE_UNLOCK(adapter); - return POLL_RETURN_COUNT(rx_done); - } - - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) - igb_handle_link_locked(adapter); - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - } - IGB_CORE_UNLOCK(adapter); - - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - txr = que->txr; - - igb_rxeof(que, count, &rx_done); - - IGB_TX_LOCK(txr); - do { - more = igb_txeof(txr); - } while (loop-- && more); -#ifndef IGB_LEGACY_TX - if (!drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - } - - return POLL_RETURN_COUNT(rx_done); -} -#endif /* DEVICE_POLLING */ - -/********************************************************************* - * - * MSIX Que Interrupt Service routine - * - **********************************************************************/ -static void -igb_msix_que(void *arg) -{ - struct igb_queue *que = arg; - struct adapter *adapter = que->adapter; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = que->txr; - struct rx_ring *rxr = que->rxr; - u32 newitr = 0; - bool more_rx; - - /* Ignore spurious interrupts */ - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - return; - - E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); - ++que->irqs; - - IGB_TX_LOCK(txr); - igb_txeof(txr); -#ifndef IGB_LEGACY_TX - /* Process the stack queue only if not depleted */ - if (((txr->queue_status & IGB_QUEUE_DEPLETED) == 0) && - !drbr_empty(ifp, txr->br)) - igb_mq_start_locked(ifp, txr); -#else - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - igb_start_locked(txr, ifp); -#endif - IGB_TX_UNLOCK(txr); - - more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL); - - if (adapter->enable_aim == FALSE) - goto no_calc; - /* - ** Do Adaptive Interrupt Moderation: - ** - Write out last calculated setting - ** - Calculate based on average size over - ** the last interval. - */ - if (que->eitr_setting) - E1000_WRITE_REG(&adapter->hw, - E1000_EITR(que->msix), que->eitr_setting); - - que->eitr_setting = 0; - - /* Idle, do nothing */ - if ((txr->bytes == 0) && (rxr->bytes == 0)) - goto no_calc; - - /* Used half Default if sub-gig */ - if (adapter->link_speed != 1000) - newitr = IGB_DEFAULT_ITR / 2; - else { - if ((txr->bytes) && (txr->packets)) - newitr = txr->bytes/txr->packets; - if ((rxr->bytes) && (rxr->packets)) - newitr = max(newitr, - (rxr->bytes / rxr->packets)); - newitr += 24; /* account for hardware frame, crc */ - /* set an upper boundary */ - newitr = min(newitr, 3000); - /* Be nice to the mid range */ - if ((newitr > 300) && (newitr < 1200)) - newitr = (newitr / 3); - else - newitr = (newitr / 2); - } - newitr &= 0x7FFC; /* Mask invalid bits */ - if (adapter->hw.mac.type == e1000_82575) - newitr |= newitr << 16; - else - newitr |= E1000_EITR_CNT_IGNR; - - /* save for next interrupt */ - que->eitr_setting = newitr; - - /* Reset state */ - txr->bytes = 0; - txr->packets = 0; - rxr->bytes = 0; - rxr->packets = 0; - -no_calc: - /* Schedule a clean task if needed*/ - if (more_rx) - taskqueue_enqueue(que->tq, &que->que_task); - else - /* Reenable this interrupt */ - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); - return; -} - - -/********************************************************************* - * - * MSIX Link Interrupt Service routine - * - **********************************************************************/ - -static void -igb_msix_link(void *arg) -{ - struct adapter *adapter = arg; - u32 icr; - - ++adapter->link_irq; - icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (!(icr & E1000_ICR_LSC)) - goto spurious; - igb_handle_link(adapter, 0); - -spurious: - /* Rearm */ - E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); - return; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) -{ - struct adapter *adapter = ifp->if_softc; - - INIT_DEBUGOUT("igb_media_status: begin"); - - IGB_CORE_LOCK(adapter); - igb_update_link_status(adapter); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!adapter->link_active) { - IGB_CORE_UNLOCK(adapter); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - switch (adapter->link_speed) { - case 10: - ifmr->ifm_active |= IFM_10_T; - break; - case 100: - /* - ** Support for 100Mb SFP - these are Fiber - ** but the media type appears as serdes - */ - if (adapter->hw.phy.media_type == - e1000_media_type_internal_serdes) - ifmr->ifm_active |= IFM_100_FX; - else - ifmr->ifm_active |= IFM_100_TX; - break; - case 1000: - ifmr->ifm_active |= IFM_1000_T; - break; - case 2500: - ifmr->ifm_active |= IFM_2500_SX; - break; - } - - if (adapter->link_duplex == FULL_DUPLEX) - ifmr->ifm_active |= IFM_FDX; - else - ifmr->ifm_active |= IFM_HDX; - - IGB_CORE_UNLOCK(adapter); -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -igb_media_change(struct ifnet *ifp) -{ - struct adapter *adapter = ifp->if_softc; - struct ifmedia *ifm = &adapter->media; - - INIT_DEBUGOUT("igb_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - IGB_CORE_LOCK(adapter); - switch (IFM_SUBTYPE(ifm->ifm_media)) { - case IFM_AUTO: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - break; - case IFM_1000_LX: - case IFM_1000_SX: - case IFM_1000_T: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; - break; - case IFM_100_TX: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; - break; - case IFM_10_T: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; - break; - default: - device_printf(adapter->dev, "Unsupported media type\n"); - } - - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - - return (0); -} - - -/********************************************************************* - * - * This routine maps the mbufs to Advanced TX descriptors. - * - **********************************************************************/ -static int -igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) -{ - struct adapter *adapter = txr->adapter; - u32 olinfo_status = 0, cmd_type_len; - int i, j, error, nsegs; - int first; - bool remap = TRUE; - struct mbuf *m_head; - bus_dma_segment_t segs[IGB_MAX_SCATTER]; - bus_dmamap_t map; - struct igb_tx_buf *txbuf; - union e1000_adv_tx_desc *txd = NULL; - - m_head = *m_headp; - - /* Basic descriptor defines */ - cmd_type_len = (E1000_ADVTXD_DTYP_DATA | - E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); - - if (m_head->m_flags & M_VLANTAG) - cmd_type_len |= E1000_ADVTXD_DCMD_VLE; - - /* - * Important to capture the first descriptor - * used because it will contain the index of - * the one we tell the hardware to report back - */ - first = txr->next_avail_desc; - txbuf = &txr->tx_buffers[first]; - map = txbuf->map; - - /* - * Map the packet for DMA. - */ -retry: - error = bus_dmamap_load_mbuf_sg(txr->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (__predict_false(error)) { - struct mbuf *m; - - switch (error) { - case EFBIG: - /* Try it again? - one try */ - if (remap == TRUE) { - remap = FALSE; - m = m_collapse(*m_headp, M_NOWAIT, - IGB_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - goto retry; - } else - return (error); - default: - txr->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } - - /* Make certain there are enough descriptors */ - if (txr->tx_avail < (nsegs + 2)) { - txr->no_desc_avail++; - bus_dmamap_unload(txr->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* - ** Set up the appropriate offload context - ** this will consume the first descriptor - */ - error = igb_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); - if (__predict_false(error)) { - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - olinfo_status |= txr->me << 4; - - i = txr->next_avail_desc; - for (j = 0; j < nsegs; j++) { - bus_size_t seglen; - bus_addr_t segaddr; - - txbuf = &txr->tx_buffers[i]; - txd = &txr->tx_base[i]; - seglen = segs[j].ds_len; - segaddr = htole64(segs[j].ds_addr); - - txd->read.buffer_addr = segaddr; - txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | - cmd_type_len | seglen); - txd->read.olinfo_status = htole32(olinfo_status); - - if (++i == txr->num_desc) - i = 0; - } - - txd->read.cmd_type_len |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - txr->tx_avail -= nsegs; - txr->next_avail_desc = i; - - txbuf->m_head = m_head; - /* - ** Here we swap the map so the last descriptor, - ** which gets the completion interrupt has the - ** real map, and the first descriptor gets the - ** unused map from this descriptor. - */ - txr->tx_buffers[first].map = txbuf->map; - txbuf->map = map; - bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); - - /* Set the EOP descriptor that will be marked done */ - txbuf = &txr->tx_buffers[first]; - txbuf->eop = txd; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - /* - * Advance the Transmit Descriptor Tail (Tdt), this tells the - * hardware that this frame is available to transmit. - */ - ++txr->total_packets; - E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); - - return (0); -} -static void -igb_set_promisc(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u32 reg; - - if (adapter->vf_ifp) { - e1000_promisc_set_vf(hw, e1000_promisc_enabled); - return; - } - - reg = E1000_READ_REG(hw, E1000_RCTL); - if (ifp->if_flags & IFF_PROMISC) { - reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } else if (ifp->if_flags & IFF_ALLMULTI) { - reg |= E1000_RCTL_MPE; - reg &= ~E1000_RCTL_UPE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } -} - -static void -igb_disable_promisc(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; - u32 reg; - int mcnt = 0; - - if (adapter->vf_ifp) { - e1000_promisc_set_vf(hw, e1000_promisc_disabled); - return; - } - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= (~E1000_RCTL_UPE); - if (ifp->if_flags & IFF_ALLMULTI) - mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else { - struct ifmultiaddr *ifma; -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - } - /* Don't disable if in MAX groups */ - if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) - reg &= (~E1000_RCTL_MPE); - E1000_WRITE_REG(hw, E1000_RCTL, reg); -} - - -/********************************************************************* - * Multicast Update - * - * This routine is called whenever multicast address list is updated. - * - **********************************************************************/ - -static void -igb_set_multi(struct adapter *adapter) -{ - struct ifnet *ifp = adapter->ifp; - struct ifmultiaddr *ifma; - u32 reg_rctl = 0; - u8 *mta; - - int mcnt = 0; - - IOCTL_DEBUGOUT("igb_set_multi: begin"); - - mta = adapter->mta; - bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES); - -#if __FreeBSD_version < 800000 - IF_ADDR_LOCK(ifp); -#else - if_maddr_rlock(ifp); -#endif - TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { - if (ifma->ifma_addr->sa_family != AF_LINK) - continue; - - if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) - break; - - bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), - &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); - mcnt++; - } -#if __FreeBSD_version < 800000 - IF_ADDR_UNLOCK(ifp); -#else - if_maddr_runlock(ifp); -#endif - - if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else - e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); -} - - -/********************************************************************* - * Timer routine: - * This routine checks for link status, - * updates statistics, and does the watchdog. - * - **********************************************************************/ - -static void -igb_local_timer(void *arg) -{ - struct adapter *adapter = arg; - device_t dev = adapter->dev; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - struct igb_queue *que = adapter->queues; - int hung = 0, busy = 0; - - - IGB_CORE_LOCK_ASSERT(adapter); - - igb_update_link_status(adapter); - igb_update_stats_counters(adapter); - - /* - ** Check the TX queues status - ** - central locked handling of OACTIVE - ** - watchdog only if all queues show hung - */ - for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { - if ((txr->queue_status & IGB_QUEUE_HUNG) && - (adapter->pause_frames == 0)) - ++hung; - if (txr->queue_status & IGB_QUEUE_DEPLETED) - ++busy; - if ((txr->queue_status & IGB_QUEUE_IDLE) == 0) - taskqueue_enqueue(que->tq, &que->que_task); - } - if (hung == adapter->num_queues) - goto timeout; - if (busy == adapter->num_queues) - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - else if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) && - (busy < adapter->num_queues)) - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - - adapter->pause_frames = 0; - callout_reset(&adapter->timer, hz, igb_local_timer, adapter); -#ifndef DEVICE_POLLING - /* Schedule all queue interrupts - deadlock protection */ - E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->que_mask); -#endif - return; - -timeout: - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, - E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), - E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); - device_printf(dev,"TX(%d) desc avail = %d," - "Next TX to Clean = %d\n", - txr->me, txr->tx_avail, txr->next_to_clean); - adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - adapter->watchdog_events++; - igb_init_locked(adapter); -} - -static void -igb_update_link_status(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_fc_info *fc = &hw->fc; - struct ifnet *ifp = adapter->ifp; - device_t dev = adapter->dev; - struct tx_ring *txr = adapter->tx_rings; - u32 link_check, thstat, ctrl; - char *flowctl = NULL; - - link_check = thstat = ctrl = 0; - - /* Get the cached link value or read for real */ - switch (hw->phy.media_type) { - case e1000_media_type_copper: - if (hw->mac.get_link_status) { - /* Do the work to read phy */ - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - } else - link_check = TRUE; - break; - case e1000_media_type_fiber: - e1000_check_for_link(hw); - link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); - break; - case e1000_media_type_internal_serdes: - e1000_check_for_link(hw); - link_check = adapter->hw.mac.serdes_has_link; - break; - /* VF device is type_unknown */ - case e1000_media_type_unknown: - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - /* Fall thru */ - default: - break; - } - - /* Check for thermal downshift or shutdown */ - if (hw->mac.type == e1000_i350) { - thstat = E1000_READ_REG(hw, E1000_THSTAT); - ctrl = E1000_READ_REG(hw, E1000_CTRL_EXT); - } - - /* Get the flow control for display */ - switch (fc->current_mode) { - case e1000_fc_rx_pause: - flowctl = "RX"; - break; - case e1000_fc_tx_pause: - flowctl = "TX"; - break; - case e1000_fc_full: - flowctl = "Full"; - break; - case e1000_fc_none: - default: - flowctl = "None"; - break; - } - - /* Now we check if a transition has happened */ - if (link_check && (adapter->link_active == 0)) { - e1000_get_speed_and_duplex(&adapter->hw, - &adapter->link_speed, &adapter->link_duplex); - if (bootverbose) - device_printf(dev, "Link is up %d Mbps %s," - " Flow Control: %s\n", - adapter->link_speed, - ((adapter->link_duplex == FULL_DUPLEX) ? - "Full Duplex" : "Half Duplex"), flowctl); - adapter->link_active = 1; - ifp->if_baudrate = adapter->link_speed * 1000000; - if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && - (thstat & E1000_THSTAT_LINK_THROTTLE)) - device_printf(dev, "Link: thermal downshift\n"); - /* Delay Link Up for Phy update */ - if (((hw->mac.type == e1000_i210) || - (hw->mac.type == e1000_i211)) && - (hw->phy.id == I210_I_PHY_ID)) - msec_delay(I210_LINK_DELAY); - /* Reset if the media type changed. */ - if (hw->dev_spec._82575.media_changed) { - hw->dev_spec._82575.media_changed = false; - adapter->flags |= IGB_MEDIA_RESET; - igb_reset(adapter); - } - /* This can sleep */ - if_link_state_change(ifp, LINK_STATE_UP); - } else if (!link_check && (adapter->link_active == 1)) { - ifp->if_baudrate = adapter->link_speed = 0; - adapter->link_duplex = 0; - if (bootverbose) - device_printf(dev, "Link is Down\n"); - if ((ctrl & E1000_CTRL_EXT_LINK_MODE_GMII) && - (thstat & E1000_THSTAT_PWR_DOWN)) - device_printf(dev, "Link: thermal shutdown\n"); - adapter->link_active = 0; - /* This can sleep */ - if_link_state_change(ifp, LINK_STATE_DOWN); - /* Reset queue state */ - for (int i = 0; i < adapter->num_queues; i++, txr++) - txr->queue_status = IGB_QUEUE_IDLE; - } -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - **********************************************************************/ - -static void -igb_stop(void *arg) -{ - struct adapter *adapter = arg; - struct ifnet *ifp = adapter->ifp; - struct tx_ring *txr = adapter->tx_rings; - - IGB_CORE_LOCK_ASSERT(adapter); - - INIT_DEBUGOUT("igb_stop: begin"); - - igb_disable_intr(adapter); - - callout_stop(&adapter->timer); - - /* Tell the stack that the interface is no longer active */ - ifp->if_drv_flags &= ~IFF_DRV_RUNNING; - ifp->if_drv_flags |= IFF_DRV_OACTIVE; - - /* Disarm watchdog timer. */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - txr->queue_status = IGB_QUEUE_IDLE; - IGB_TX_UNLOCK(txr); - } - - e1000_reset_hw(&adapter->hw); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); - - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); -} - - -/********************************************************************* - * - * Determine hardware revision. - * - **********************************************************************/ -static void -igb_identify_hardware(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); - adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - - /* Save off the information about this board */ - adapter->hw.vendor_id = pci_get_vendor(dev); - adapter->hw.device_id = pci_get_device(dev); - adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); - adapter->hw.subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - adapter->hw.subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - /* Set MAC type early for PCI setup */ - e1000_set_mac_type(&adapter->hw); - - /* Are we a VF device? */ - if ((adapter->hw.mac.type == e1000_vfadapt) || - (adapter->hw.mac.type == e1000_vfadapt_i350)) - adapter->vf_ifp = 1; - else - adapter->vf_ifp = 0; -} - -static int -igb_allocate_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int rid; - - rid = PCIR_BAR(0); - adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - if (adapter->pci_mem == NULL) { - device_printf(dev, "Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->pci_mem); - adapter->osdep.mem_bus_space_handle = - rman_get_bushandle(adapter->pci_mem); - adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - - adapter->num_queues = 1; /* Defaults for Legacy or MSI */ - - /* This will setup either MSI/X or MSI */ - adapter->msix = igb_setup_msix(adapter); - adapter->hw.back = &adapter->osdep; - - return (0); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -static int -igb_allocate_legacy(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = adapter->queues; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr = adapter->tx_rings; -#endif - int error, rid = 0; - - /* Turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - /* MSI RID is 1 */ - if (adapter->msix == 1) - rid = 1; - - /* We allocate a single interrupt resource */ - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); - } - -#ifndef IGB_LEGACY_TX - TASK_INIT(&txr->txq_task, 0, igb_deferred_mq_start, txr); -#endif - - /* - * Try allocating a fast interrupt and the associated deferred - * processing contexts. - */ - TASK_INIT(&que->que_task, 0, igb_handle_que, que); - /* Make tasklet for deferred link handling */ - TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); - que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq", - device_get_nameunit(adapter->dev)); - if ((error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, - adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(que->tq); - que->tq = NULL; - return (error); - } - - return (0); -} - - -/********************************************************************* - * - * Setup the MSIX Queue Interrupt handlers: - * - **********************************************************************/ -static int -igb_allocate_msix(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = adapter->queues; - int error, rid, vector = 0; - int cpu_id = 0; -#ifdef RSS - cpuset_t cpu_mask; -#endif - - /* Be sure to start with all interrupts disabled */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); - E1000_WRITE_FLUSH(&adapter->hw); - -#ifdef RSS - /* - * If we're doing RSS, the number of queues needs to - * match the number of RSS buckets that are configured. - * - * + If there's more queues than RSS buckets, we'll end - * up with queues that get no traffic. - * - * + If there's more RSS buckets than queues, we'll end - * up having multiple RSS buckets map to the same queue, - * so there'll be some contention. - */ - if (adapter->num_queues != rss_getnumbuckets()) { - device_printf(dev, - "%s: number of queues (%d) != number of RSS buckets (%d)" - "; performance will be impacted.\n", - __func__, - adapter->num_queues, - rss_getnumbuckets()); - } -#endif - - for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { - rid = vector +1; - que->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (que->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "MSIX Queue Interrupt\n"); - return (ENXIO); - } - error = bus_setup_intr(dev, que->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - igb_msix_que, que, &que->tag); - if (error) { - que->res = NULL; - device_printf(dev, "Failed to register Queue handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, que->res, que->tag, "que %d", i); -#endif - que->msix = vector; - if (adapter->hw.mac.type == e1000_82575) - que->eims = E1000_EICR_TX_QUEUE0 << i; - else - que->eims = 1 << vector; - -#ifdef RSS - /* - * The queue ID is used as the RSS layer bucket ID. - * We look up the queue ID -> RSS CPU ID and select - * that. - */ - cpu_id = rss_getcpu(i % rss_getnumbuckets()); -#else - /* - * Bind the msix vector, and thus the - * rings to the corresponding cpu. - * - * This just happens to match the default RSS round-robin - * bucket -> queue -> CPU allocation. - */ - if (adapter->num_queues > 1) { - if (igb_last_bind_cpu < 0) - igb_last_bind_cpu = CPU_FIRST(); - cpu_id = igb_last_bind_cpu; - } -#endif - - if (adapter->num_queues > 1) { - bus_bind_intr(dev, que->res, cpu_id); -#ifdef RSS - device_printf(dev, - "Bound queue %d to RSS bucket %d\n", - i, cpu_id); -#else - device_printf(dev, - "Bound queue %d to cpu %d\n", - i, cpu_id); -#endif - } - -#ifndef IGB_LEGACY_TX - TASK_INIT(&que->txr->txq_task, 0, igb_deferred_mq_start, - que->txr); -#endif - /* Make tasklet for deferred handling */ - TASK_INIT(&que->que_task, 0, igb_handle_que, que); - que->tq = taskqueue_create("igb_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); - if (adapter->num_queues > 1) { - /* - * Only pin the taskqueue thread to a CPU if - * RSS is in use. - * - * This again just happens to match the default RSS - * round-robin bucket -> queue -> CPU allocation. - */ -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, - "%s que (bucket %d)", - device_get_nameunit(adapter->dev), - cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s que (qid %d)", - device_get_nameunit(adapter->dev), - cpu_id); -#endif - } else { - taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", - device_get_nameunit(adapter->dev)); - } - - /* Finally update the last bound CPU id */ - if (adapter->num_queues > 1) - igb_last_bind_cpu = CPU_NEXT(igb_last_bind_cpu); - } - - /* And Link */ - rid = vector + 1; - adapter->res = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res == NULL) { - device_printf(dev, - "Unable to allocate bus resource: " - "MSIX Link Interrupt\n"); - return (ENXIO); - } - if ((error = bus_setup_intr(dev, adapter->res, - INTR_TYPE_NET | INTR_MPSAFE, NULL, - igb_msix_link, adapter, &adapter->tag)) != 0) { - device_printf(dev, "Failed to register Link handler"); - return (error); - } -#if __FreeBSD_version >= 800504 - bus_describe_intr(dev, adapter->res, adapter->tag, "link"); -#endif - adapter->linkvec = vector; - - return (0); -} - - -static void -igb_configure_queues(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct igb_queue *que; - u32 tmp, ivar = 0, newitr = 0; - - /* First turn on RSS capability */ - if (adapter->hw.mac.type != e1000_82575) - E1000_WRITE_REG(hw, E1000_GPIE, - E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | - E1000_GPIE_PBA | E1000_GPIE_NSICR); - - /* Turn on MSIX */ - switch (adapter->hw.mac.type) { - case e1000_82580: - case e1000_i350: - case e1000_i354: - case e1000_i210: - case e1000_i211: - case e1000_vfadapt: - case e1000_vfadapt_i350: - /* RX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i & 1) { - ivar &= 0xFF00FFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 16; - } else { - ivar &= 0xFFFFFF00; - ivar |= que->msix | E1000_IVAR_VALID; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - } - /* TX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i >> 1; - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i & 1) { - ivar &= 0x00FFFFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 24; - } else { - ivar &= 0xFFFF00FF; - ivar |= (que->msix | E1000_IVAR_VALID) << 8; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - - /* And for the link interrupt */ - ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; - adapter->link_mask = 1 << adapter->linkvec; - E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); - break; - case e1000_82576: - /* RX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i < 8) { - ivar &= 0xFFFFFF00; - ivar |= que->msix | E1000_IVAR_VALID; - } else { - ivar &= 0xFF00FFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 16; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - /* TX entries */ - for (int i = 0; i < adapter->num_queues; i++) { - u32 index = i & 0x7; /* Each IVAR has two entries */ - ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); - que = &adapter->queues[i]; - if (i < 8) { - ivar &= 0xFFFF00FF; - ivar |= (que->msix | E1000_IVAR_VALID) << 8; - } else { - ivar &= 0x00FFFFFF; - ivar |= (que->msix | E1000_IVAR_VALID) << 24; - } - E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); - adapter->que_mask |= que->eims; - } - - /* And for the link interrupt */ - ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; - adapter->link_mask = 1 << adapter->linkvec; - E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); - break; - - case e1000_82575: - /* enable MSI-X support*/ - tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); - tmp |= E1000_CTRL_EXT_PBA_CLR; - /* Auto-Mask interrupts upon ICR read. */ - tmp |= E1000_CTRL_EXT_EIAME; - tmp |= E1000_CTRL_EXT_IRCA; - E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); - - /* Queues */ - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - tmp = E1000_EICR_RX_QUEUE0 << i; - tmp |= E1000_EICR_TX_QUEUE0 << i; - que->eims = tmp; - E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), - i, que->eims); - adapter->que_mask |= que->eims; - } - - /* Link */ - E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), - E1000_EIMS_OTHER); - adapter->link_mask |= E1000_EIMS_OTHER; - default: - break; - } - - /* Set the starting interrupt rate */ - if (igb_max_interrupt_rate > 0) - newitr = (4000000 / igb_max_interrupt_rate) & 0x7FFC; - - if (hw->mac.type == e1000_82575) - newitr |= newitr << 16; - else - newitr |= E1000_EITR_CNT_IGNR; - - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); - } - - return; -} - - -static void -igb_free_pci_resources(struct adapter *adapter) -{ - struct igb_queue *que = adapter->queues; - device_t dev = adapter->dev; - int rid; - - /* - ** There is a slight possibility of a failure mode - ** in attach that will result in entering this function - ** before interrupt resources have been initialized, and - ** in that case we do not want to execute the loops below - ** We can detect this reliably by the state of the adapter - ** res pointer. - */ - if (adapter->res == NULL) - goto mem; - - /* - * First release all the interrupt resources: - */ - for (int i = 0; i < adapter->num_queues; i++, que++) { - rid = que->msix + 1; - if (que->tag != NULL) { - bus_teardown_intr(dev, que->res, que->tag); - que->tag = NULL; - } - if (que->res != NULL) - bus_release_resource(dev, - SYS_RES_IRQ, rid, que->res); - } - - /* Clean the Legacy or Link interrupt last */ - if (adapter->linkvec) /* we are doing MSIX */ - rid = adapter->linkvec + 1; - else - (adapter->msix != 0) ? (rid = 1):(rid = 0); - - que = adapter->queues; - if (adapter->tag != NULL) { - taskqueue_drain(que->tq, &adapter->link_task); - bus_teardown_intr(dev, adapter->res, adapter->tag); - adapter->tag = NULL; - } - if (adapter->res != NULL) - bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); - - for (int i = 0; i < adapter->num_queues; i++, que++) { - if (que->tq != NULL) { -#ifndef IGB_LEGACY_TX - taskqueue_drain(que->tq, &que->txr->txq_task); -#endif - taskqueue_drain(que->tq, &que->que_task); - taskqueue_free(que->tq); - } - } -mem: - if (adapter->msix) - pci_release_msi(dev); - - if (adapter->msix_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - adapter->memrid, adapter->msix_mem); - - if (adapter->pci_mem != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->pci_mem); - -} - -/* - * Setup Either MSI/X or MSI - */ -static int -igb_setup_msix(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int bar, want, queues, msgs, maxqueues; - - /* tuneable override */ - if (igb_enable_msix == 0) - goto msi; - - /* First try MSI/X */ - msgs = pci_msix_count(dev); - if (msgs == 0) - goto msi; - /* - ** Some new devices, as with ixgbe, now may - ** use a different BAR, so we need to keep - ** track of which is used. - */ - adapter->memrid = PCIR_BAR(IGB_MSIX_BAR); - bar = pci_read_config(dev, adapter->memrid, 4); - if (bar == 0) /* use next bar */ - adapter->memrid += 4; - adapter->msix_mem = bus_alloc_resource_any(dev, - SYS_RES_MEMORY, &adapter->memrid, RF_ACTIVE); - if (adapter->msix_mem == NULL) { - /* May not be enabled */ - device_printf(adapter->dev, - "Unable to map MSIX table \n"); - goto msi; - } - - queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; - - /* Override via tuneable */ - if (igb_num_queues != 0) - queues = igb_num_queues; - -#ifdef RSS - /* If we're doing RSS, clamp at the number of RSS buckets */ - if (queues > rss_getnumbuckets()) - queues = rss_getnumbuckets(); -#endif - - - /* Sanity check based on HW */ - switch (adapter->hw.mac.type) { - case e1000_82575: - maxqueues = 4; - break; - case e1000_82576: - case e1000_82580: - case e1000_i350: - case e1000_i354: - maxqueues = 8; - break; - case e1000_i210: - maxqueues = 4; - break; - case e1000_i211: - maxqueues = 2; - break; - default: /* VF interfaces */ - maxqueues = 1; - break; - } - - /* Final clamp on the actual hardware capability */ - if (queues > maxqueues) - queues = maxqueues; - - /* - ** One vector (RX/TX pair) per queue - ** plus an additional for Link interrupt - */ - want = queues + 1; - if (msgs >= want) - msgs = want; - else { - device_printf(adapter->dev, - "MSIX Configuration Problem, " - "%d vectors configured, but %d queues wanted!\n", - msgs, want); - goto msi; - } - if ((pci_alloc_msix(dev, &msgs) == 0) && (msgs == want)) { - device_printf(adapter->dev, - "Using MSIX interrupts with %d vectors\n", msgs); - adapter->num_queues = queues; - return (msgs); - } - /* - ** If MSIX alloc failed or provided us with - ** less than needed, free and fall through to MSI - */ - pci_release_msi(dev); - -msi: - if (adapter->msix_mem != NULL) { - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); - adapter->msix_mem = NULL; - } - msgs = 1; - if (pci_alloc_msi(dev, &msgs) == 0) { - device_printf(adapter->dev," Using an MSI interrupt\n"); - return (msgs); - } - device_printf(adapter->dev," Using a Legacy interrupt\n"); - return (0); -} - -/********************************************************************* - * - * Initialize the DMA Coalescing feature - * - **********************************************************************/ -static void -igb_init_dmac(struct adapter *adapter, u32 pba) -{ - device_t dev = adapter->dev; - struct e1000_hw *hw = &adapter->hw; - u32 dmac, reg = ~E1000_DMACR_DMAC_EN; - u16 hwm; - - if (hw->mac.type == e1000_i211) - return; - - if (hw->mac.type > e1000_82580) { - - if (adapter->dmac == 0) { /* Disabling it */ - E1000_WRITE_REG(hw, E1000_DMACR, reg); - return; - } else - device_printf(dev, "DMA Coalescing enabled\n"); - - /* Set starting threshold */ - E1000_WRITE_REG(hw, E1000_DMCTXTH, 0); - - hwm = 64 * pba - adapter->max_frame_size / 16; - if (hwm < 64 * (pba - 6)) - hwm = 64 * (pba - 6); - reg = E1000_READ_REG(hw, E1000_FCRTC); - reg &= ~E1000_FCRTC_RTH_COAL_MASK; - reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT) - & E1000_FCRTC_RTH_COAL_MASK); - E1000_WRITE_REG(hw, E1000_FCRTC, reg); - - - dmac = pba - adapter->max_frame_size / 512; - if (dmac < pba - 10) - dmac = pba - 10; - reg = E1000_READ_REG(hw, E1000_DMACR); - reg &= ~E1000_DMACR_DMACTHR_MASK; - reg = ((dmac << E1000_DMACR_DMACTHR_SHIFT) - & E1000_DMACR_DMACTHR_MASK); - - /* transition to L0x or L1 if available..*/ - reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK); - - /* Check if status is 2.5Gb backplane connection - * before configuration of watchdog timer, which is - * in msec values in 12.8usec intervals - * watchdog timer= msec values in 32usec intervals - * for non 2.5Gb connection - */ - if (hw->mac.type == e1000_i354) { - int status = E1000_READ_REG(hw, E1000_STATUS); - if ((status & E1000_STATUS_2P5_SKU) && - (!(status & E1000_STATUS_2P5_SKU_OVER))) - reg |= ((adapter->dmac * 5) >> 6); - else - reg |= (adapter->dmac >> 5); - } else { - reg |= (adapter->dmac >> 5); - } - - E1000_WRITE_REG(hw, E1000_DMACR, reg); - - E1000_WRITE_REG(hw, E1000_DMCRTRH, 0); - - /* Set the interval before transition */ - reg = E1000_READ_REG(hw, E1000_DMCTLX); - if (hw->mac.type == e1000_i350) - reg |= IGB_DMCTLX_DCFLUSH_DIS; - /* - ** in 2.5Gb connection, TTLX unit is 0.4 usec - ** which is 0x4*2 = 0xA. But delay is still 4 usec - */ - if (hw->mac.type == e1000_i354) { - int status = E1000_READ_REG(hw, E1000_STATUS); - if ((status & E1000_STATUS_2P5_SKU) && - (!(status & E1000_STATUS_2P5_SKU_OVER))) - reg |= 0xA; - else - reg |= 0x4; - } else { - reg |= 0x4; - } - - E1000_WRITE_REG(hw, E1000_DMCTLX, reg); - - /* free space in tx packet buffer to wake from DMA coal */ - E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_TXPBSIZE - - (2 * adapter->max_frame_size)) >> 6); - - /* make low power state decision controlled by DMA coal */ - reg = E1000_READ_REG(hw, E1000_PCIEMISC); - reg &= ~E1000_PCIEMISC_LX_DECISION; - E1000_WRITE_REG(hw, E1000_PCIEMISC, reg); - - } else if (hw->mac.type == e1000_82580) { - u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC); - E1000_WRITE_REG(hw, E1000_PCIEMISC, - reg & ~E1000_PCIEMISC_LX_DECISION); - E1000_WRITE_REG(hw, E1000_DMACR, 0); - } -} - - -/********************************************************************* - * - * Set up an fresh starting state - * - **********************************************************************/ -static void -igb_reset(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct e1000_hw *hw = &adapter->hw; - struct e1000_fc_info *fc = &hw->fc; - struct ifnet *ifp = adapter->ifp; - u32 pba = 0; - u16 hwm; - - INIT_DEBUGOUT("igb_reset: begin"); - - /* Let the firmware know the OS is in control */ - igb_get_hw_control(adapter); - - /* - * Packet Buffer Allocation (PBA) - * Writing PBA sets the receive portion of the buffer - * the remainder is used for the transmit buffer. - */ - switch (hw->mac.type) { - case e1000_82575: - pba = E1000_PBA_32K; - break; - case e1000_82576: - case e1000_vfadapt: - pba = E1000_READ_REG(hw, E1000_RXPBS); - pba &= E1000_RXPBS_SIZE_MASK_82576; - break; - case e1000_82580: - case e1000_i350: - case e1000_i354: - case e1000_vfadapt_i350: - pba = E1000_READ_REG(hw, E1000_RXPBS); - pba = e1000_rxpbs_adjust_82580(pba); - break; - case e1000_i210: - case e1000_i211: - pba = E1000_PBA_34K; - default: - break; - } - - /* Special needs in case of Jumbo frames */ - if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { - u32 tx_space, min_tx, min_rx; - pba = E1000_READ_REG(hw, E1000_PBA); - tx_space = pba >> 16; - pba &= 0xffff; - min_tx = (adapter->max_frame_size + - sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; - min_tx = roundup2(min_tx, 1024); - min_tx >>= 10; - min_rx = adapter->max_frame_size; - min_rx = roundup2(min_rx, 1024); - min_rx >>= 10; - if (tx_space < min_tx && - ((min_tx - tx_space) < pba)) { - pba = pba - (min_tx - tx_space); - /* - * if short on rx space, rx wins - * and must trump tx adjustment - */ - if (pba < min_rx) - pba = min_rx; - } - E1000_WRITE_REG(hw, E1000_PBA, pba); - } - - INIT_DEBUGOUT1("igb_init: pba=%dK",pba); - - /* - * These parameters control the automatic generation (Tx) and - * response (Rx) to Ethernet PAUSE frames. - * - High water mark should allow for at least two frames to be - * received after sending an XOFF. - * - Low water mark works best when it is very near the high water mark. - * This allows the receiver to restart by sending XON when it has - * drained a bit. - */ - hwm = min(((pba << 10) * 9 / 10), - ((pba << 10) - 2 * adapter->max_frame_size)); - - if (hw->mac.type < e1000_82576) { - fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ - fc->low_water = fc->high_water - 8; - } else { - fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ - fc->low_water = fc->high_water - 16; - } - - fc->pause_time = IGB_FC_PAUSE_TIME; - fc->send_xon = TRUE; - if (adapter->fc) - fc->requested_mode = adapter->fc; - else - fc->requested_mode = e1000_fc_default; - - /* Issue a global reset */ - e1000_reset_hw(hw); - E1000_WRITE_REG(hw, E1000_WUC, 0); - - /* Reset for AutoMediaDetect */ - if (adapter->flags & IGB_MEDIA_RESET) { - e1000_setup_init_funcs(hw, TRUE); - e1000_get_bus_info(hw); - adapter->flags &= ~IGB_MEDIA_RESET; - } - - if (e1000_init_hw(hw) < 0) - device_printf(dev, "Hardware Initialization Failed\n"); - - /* Setup DMA Coalescing */ - igb_init_dmac(adapter, pba); - - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - e1000_get_phy_info(hw); - e1000_check_for_link(hw); - return; -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -igb_setup_interface(device_t dev, struct adapter *adapter) -{ - struct ifnet *ifp; - - INIT_DEBUGOUT("igb_setup_interface: begin"); - - ifp = adapter->ifp = if_alloc(IFT_ETHER); - if (ifp == NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - ifp->if_init = igb_init; - ifp->if_softc = adapter; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = igb_ioctl; - ifp->if_get_counter = igb_get_counter; - - /* TSO parameters */ - ifp->if_hw_tsomax = IP_MAXPACKET; - ifp->if_hw_tsomaxsegcount = IGB_MAX_SCATTER; - ifp->if_hw_tsomaxsegsize = IGB_TSO_SEG_SIZE; - -#ifndef IGB_LEGACY_TX - ifp->if_transmit = igb_mq_start; - ifp->if_qflush = igb_qflush; -#else - ifp->if_start = igb_start; - IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); - ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; - IFQ_SET_READY(&ifp->if_snd); -#endif - - ether_ifattach(ifp, adapter->hw.mac.addr); - - ifp->if_capabilities = ifp->if_capenable = 0; - - ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; -#if __FreeBSD_version >= 1000000 - ifp->if_capabilities |= IFCAP_HWCSUM_IPV6; -#endif - ifp->if_capabilities |= IFCAP_TSO; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; - ifp->if_capenable = ifp->if_capabilities; - - /* Don't enable LRO by default */ - ifp->if_capabilities |= IFCAP_LRO; - -#ifdef DEVICE_POLLING - ifp->if_capabilities |= IFCAP_POLLING; -#endif - - /* - * Tell the upper layer(s) we - * support full VLAN capability. - */ - ifp->if_hdrlen = sizeof(struct ether_vlan_header); - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU; - ifp->if_capenable |= IFCAP_VLAN_HWTAGGING - | IFCAP_VLAN_HWTSO - | IFCAP_VLAN_MTU; - - /* - ** Don't turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the igb driver you can - ** enable this and get full hardware tag filtering. - */ - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&adapter->media, IFM_IMASK, - igb_media_change, igb_media_status); - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); - } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); - if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); - } - } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); - return (0); -} - - -/* - * Manage DMA'able memory. - */ -static void -igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - -static int -igb_dma_malloc(struct adapter *adapter, bus_size_t size, - struct igb_dma_alloc *dma, int mapflags) -{ - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - IGB_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } - - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for the transmit and receive rings, and then - * the descriptors associated with each, called only once at attach. - * - **********************************************************************/ -static int -igb_allocate_queues(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct igb_queue *que = NULL; - struct tx_ring *txr = NULL; - struct rx_ring *rxr = NULL; - int rsize, tsize, error = E1000_SUCCESS; - int txconf = 0, rxconf = 0; - - /* First allocate the top level queue structs */ - if (!(adapter->queues = - (struct igb_queue *) malloc(sizeof(struct igb_queue) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate queue memory\n"); - error = ENOMEM; - goto fail; - } - - /* Next allocate the TX ring struct memory */ - if (!(adapter->tx_rings = - (struct tx_ring *) malloc(sizeof(struct tx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate TX ring memory\n"); - error = ENOMEM; - goto tx_fail; - } - - /* Now allocate the RX */ - if (!(adapter->rx_rings = - (struct rx_ring *) malloc(sizeof(struct rx_ring) * - adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate RX ring memory\n"); - error = ENOMEM; - goto rx_fail; - } - - tsize = roundup2(adapter->num_tx_desc * - sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); - /* - * Now set up the TX queues, txconf is needed to handle the - * possibility that things fail midcourse and we need to - * undo memory gracefully - */ - for (int i = 0; i < adapter->num_queues; i++, txconf++) { - /* Set up some basics */ - txr = &adapter->tx_rings[i]; - txr->adapter = adapter; - txr->me = i; - txr->num_desc = adapter->num_tx_desc; - - /* Initialize the TX lock */ - snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); - - if (igb_dma_malloc(adapter, tsize, - &txr->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate TX Descriptor memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - txr->tx_base = (union e1000_adv_tx_desc *)txr->txdma.dma_vaddr; - bzero((void *)txr->tx_base, tsize); - - /* Now allocate transmit buffers for the ring */ - if (igb_allocate_transmit_buffers(txr)) { - device_printf(dev, - "Critical Failure setting up transmit buffers\n"); - error = ENOMEM; - goto err_tx_desc; - } -#ifndef IGB_LEGACY_TX - /* Allocate a buf ring */ - txr->br = buf_ring_alloc(igb_buf_ring_size, M_DEVBUF, - M_WAITOK, &txr->tx_mtx); -#endif - } - - /* - * Next the RX queues... - */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); - for (int i = 0; i < adapter->num_queues; i++, rxconf++) { - rxr = &adapter->rx_rings[i]; - rxr->adapter = adapter; - rxr->me = i; - - /* Initialize the RX lock */ - snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", - device_get_nameunit(dev), txr->me); - mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); - - if (igb_dma_malloc(adapter, rsize, - &rxr->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, - "Unable to allocate RxDescriptor memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; - bzero((void *)rxr->rx_base, rsize); - - /* Allocate receive buffers for the ring*/ - if (igb_allocate_receive_buffers(rxr)) { - device_printf(dev, - "Critical Failure setting up receive buffers\n"); - error = ENOMEM; - goto err_rx_desc; - } - } - - /* - ** Finally set up the queue holding structs - */ - for (int i = 0; i < adapter->num_queues; i++) { - que = &adapter->queues[i]; - que->adapter = adapter; - que->txr = &adapter->tx_rings[i]; - que->rxr = &adapter->rx_rings[i]; - } - - return (0); - -err_rx_desc: - for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) - igb_dma_free(adapter, &rxr->rxdma); -err_tx_desc: - for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) - igb_dma_free(adapter, &txr->txdma); - free(adapter->rx_rings, M_DEVBUF); -rx_fail: -#ifndef IGB_LEGACY_TX - buf_ring_free(txr->br, M_DEVBUF); -#endif - free(adapter->tx_rings, M_DEVBUF); -tx_fail: - free(adapter->queues, M_DEVBUF); -fail: - return (error); -} - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. This is - * called only once at attach, setup is done every reset. - * - **********************************************************************/ -static int -igb_allocate_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - device_t dev = adapter->dev; - struct igb_tx_buf *txbuf; - int error, i; - - /* - * Setup DMA descriptor areas. - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - IGB_TSO_SIZE, /* maxsize */ - IGB_MAX_SCATTER, /* nsegments */ - PAGE_SIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &txr->txtag))) { - device_printf(dev,"Unable to allocate TX DMA tag\n"); - goto fail; - } - - if (!(txr->tx_buffers = - (struct igb_tx_buf *) malloc(sizeof(struct igb_tx_buf) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer dma maps */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - } - - return 0; -fail: - /* We free all, it handles case where we are in the middle */ - igb_free_transmit_structures(adapter); - return (error); -} - -/********************************************************************* - * - * Initialize a transmit ring. - * - **********************************************************************/ -static void -igb_setup_transmit_ring(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct igb_tx_buf *txbuf; - int i; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(adapter->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - /* Clear the old descriptor contents */ - IGB_TX_LOCK(txr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_TX, txr->me, 0); -#endif /* DEV_NETMAP */ - bzero((void *)txr->tx_base, - (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); - /* Reset indices */ - txr->next_avail_desc = 0; - txr->next_to_clean = 0; - - /* Free any existing tx buffers. */ - txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { - if (txbuf->m_head != NULL) { - bus_dmamap_sync(txr->txtag, txbuf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, txbuf->map); - m_freem(txbuf->m_head); - txbuf->m_head = NULL; - } -#ifdef DEV_NETMAP - if (slot) { - int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); - /* no need to set the address */ - netmap_load_map(na, txr->txtag, txbuf->map, NMB(na, slot + si)); - } -#endif /* DEV_NETMAP */ - /* clear the watch index */ - txbuf->eop = NULL; - } - - /* Set number of descriptors available */ - txr->tx_avail = adapter->num_tx_desc; - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - IGB_TX_UNLOCK(txr); -} - -/********************************************************************* - * - * Initialize all transmit rings. - * - **********************************************************************/ -static void -igb_setup_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) - igb_setup_transmit_ring(txr); - - return; -} - -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -igb_initialize_transmit_units(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - struct e1000_hw *hw = &adapter->hw; - u32 tctl, txdctl; - - INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); - tctl = txdctl = 0; - - /* Setup the Tx Descriptor Rings */ - for (int i = 0; i < adapter->num_queues; i++, txr++) { - u64 bus_addr = txr->txdma.dma_paddr; - - E1000_WRITE_REG(hw, E1000_TDLEN(i), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); - E1000_WRITE_REG(hw, E1000_TDBAH(i), - (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_TDBAL(i), - (uint32_t)bus_addr); - - /* Setup the HW Tx Head and Tail descriptor pointers */ - E1000_WRITE_REG(hw, E1000_TDT(i), 0); - E1000_WRITE_REG(hw, E1000_TDH(i), 0); - - HW_DEBUGOUT2("Base = %x, Length = %x\n", - E1000_READ_REG(hw, E1000_TDBAL(i)), - E1000_READ_REG(hw, E1000_TDLEN(i))); - - txr->queue_status = IGB_QUEUE_IDLE; - - txdctl |= IGB_TX_PTHRESH; - txdctl |= IGB_TX_HTHRESH << 8; - txdctl |= IGB_TX_WTHRESH << 16; - txdctl |= E1000_TXDCTL_QUEUE_ENABLE; - E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); - } - - if (adapter->vf_ifp) - return; - - e1000_config_collision_dist(hw); - - /* Program the Transmit Control Register */ - tctl = E1000_READ_REG(hw, E1000_TCTL); - tctl &= ~E1000_TCTL_CT; - tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | - (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); - - /* This write will effectively turn on the transmit unit. */ - E1000_WRITE_REG(hw, E1000_TCTL, tctl); -} - -/********************************************************************* - * - * Free all transmit rings. - * - **********************************************************************/ -static void -igb_free_transmit_structures(struct adapter *adapter) -{ - struct tx_ring *txr = adapter->tx_rings; - - for (int i = 0; i < adapter->num_queues; i++, txr++) { - IGB_TX_LOCK(txr); - igb_free_transmit_buffers(txr); - igb_dma_free(adapter, &txr->txdma); - IGB_TX_UNLOCK(txr); - IGB_TX_LOCK_DESTROY(txr); - } - free(adapter->tx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free transmit ring related data structures. - * - **********************************************************************/ -static void -igb_free_transmit_buffers(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; - struct igb_tx_buf *tx_buffer; - int i; - - INIT_DEBUGOUT("free_transmit_ring: begin"); - - if (txr->tx_buffers == NULL) - return; - - tx_buffer = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { - if (tx_buffer->m_head != NULL) { - bus_dmamap_sync(txr->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - if (tx_buffer->map != NULL) { - bus_dmamap_destroy(txr->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } else if (tx_buffer->map != NULL) { - bus_dmamap_unload(txr->txtag, - tx_buffer->map); - bus_dmamap_destroy(txr->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } -#ifndef IGB_LEGACY_TX - if (txr->br != NULL) - buf_ring_free(txr->br, M_DEVBUF); -#endif - if (txr->tx_buffers != NULL) { - free(txr->tx_buffers, M_DEVBUF); - txr->tx_buffers = NULL; - } - if (txr->txtag != NULL) { - bus_dma_tag_destroy(txr->txtag); - txr->txtag = NULL; - } - return; -} - -/********************************************************************** - * - * Setup work for hardware segmentation offload (TSO) on - * adapters using advanced tx descriptors - * - **********************************************************************/ -static int -igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, - u32 *cmd_type_len, u32 *olinfo_status) -{ - struct adapter *adapter = txr->adapter; - struct e1000_adv_tx_context_desc *TXD; - u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; - u32 mss_l4len_idx = 0, paylen; - u16 vtag = 0, eh_type; - int ctxd, ehdrlen, ip_hlen, tcp_hlen; - struct ether_vlan_header *eh; -#ifdef INET6 - struct ip6_hdr *ip6; -#endif -#ifdef INET - struct ip *ip; -#endif - struct tcphdr *th; - - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - eh_type = eh->evl_proto; - } else { - ehdrlen = ETHER_HDR_LEN; - eh_type = eh->evl_encap_proto; - } - - switch (ntohs(eh_type)) { -#ifdef INET6 - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - /* XXX-BZ For now we do not pretend to support ext. hdrs. */ - if (ip6->ip6_nxt != IPPROTO_TCP) - return (ENXIO); - ip_hlen = sizeof(struct ip6_hdr); - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); - th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0); - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; - break; -#endif -#ifdef INET - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - if (ip->ip_p != IPPROTO_TCP) - return (ENXIO); - ip->ip_sum = 0; - ip_hlen = ip->ip_hl << 2; - th = (struct tcphdr *)((caddr_t)ip + ip_hlen); - th->th_sum = in_pseudo(ip->ip_src.s_addr, - ip->ip_dst.s_addr, htons(IPPROTO_TCP)); - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; - /* Tell transmit desc to also do IPv4 checksum. */ - *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; - break; -#endif - default: - panic("%s: CSUM_TSO but no supported IP version (0x%04x)", - __func__, ntohs(eh_type)); - break; - } - - ctxd = txr->next_avail_desc; - TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; - - tcp_hlen = th->th_off << 2; - - /* This is used in the transmit desc in encap */ - paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; - - /* VLAN MACLEN IPLEN */ - if (mp->m_flags & M_VLANTAG) { - vtag = htole16(mp->m_pkthdr.ether_vtag); - vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); - } - - vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= ip_hlen; - TXD->vlan_macip_lens = htole32(vlan_macip_lens); - - /* ADV DTYPE TUCMD */ - type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; - TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); - - /* MSS L4LEN IDX */ - mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); - mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - mss_l4len_idx |= txr->me << 4; - TXD->mss_l4len_idx = htole32(mss_l4len_idx); - - TXD->seqnum_seed = htole32(0); - - if (++ctxd == txr->num_desc) - ctxd = 0; - - txr->tx_avail--; - txr->next_avail_desc = ctxd; - *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; - *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; - *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; - ++txr->tso_tx; - return (0); -} - -/********************************************************************* - * - * Advanced Context Descriptor setup for VLAN, CSUM or TSO - * - **********************************************************************/ - -static int -igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, - u32 *cmd_type_len, u32 *olinfo_status) -{ - struct e1000_adv_tx_context_desc *TXD; - struct adapter *adapter = txr->adapter; - struct ether_vlan_header *eh; - struct ip *ip; - struct ip6_hdr *ip6; - u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0; - int ehdrlen, ip_hlen = 0; - u16 etype; - u8 ipproto = 0; - int offload = TRUE; - int ctxd = txr->next_avail_desc; - u16 vtag = 0; - - /* First check if TSO is to be used */ - if (mp->m_pkthdr.csum_flags & CSUM_TSO) - return (igb_tso_setup(txr, mp, cmd_type_len, olinfo_status)); - - if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) - offload = FALSE; - - /* Indicate the whole packet as payload when not doing TSO */ - *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT; - - /* Now ready a context descriptor */ - TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; - - /* - ** In advanced descriptors the vlan tag must - ** be placed into the context descriptor. Hence - ** we need to make one even if not doing offloads. - */ - if (mp->m_flags & M_VLANTAG) { - vtag = htole16(mp->m_pkthdr.ether_vtag); - vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); - } else if (offload == FALSE) /* ... no offload to do */ - return (0); - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - ehdrlen = ETHER_HDR_LEN; - } - - /* Set the ether header length */ - vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; - - switch (etype) { - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - ip_hlen = ip->ip_hl << 2; - ipproto = ip->ip_p; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; - break; - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - ip_hlen = sizeof(struct ip6_hdr); - /* XXX-BZ this will go badly in case of ext hdrs. */ - ipproto = ip6->ip6_nxt; - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; - break; - default: - offload = FALSE; - break; - } - - vlan_macip_lens |= ip_hlen; - type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; - - switch (ipproto) { - case IPPROTO_TCP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_TCP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; - break; - case IPPROTO_UDP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_UDP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; - break; - -#if __FreeBSD_version >= 800000 - case IPPROTO_SCTP: -#if __FreeBSD_version >= 1000000 - if (mp->m_pkthdr.csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) -#else - if (mp->m_pkthdr.csum_flags & CSUM_SCTP) -#endif - type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; - break; -#endif - default: - offload = FALSE; - break; - } - - if (offload) /* For the TX descriptor setup */ - *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; - - /* 82575 needs the queue index added */ - if (adapter->hw.mac.type == e1000_82575) - mss_l4len_idx = txr->me << 4; - - /* Now copy bits into descriptor */ - TXD->vlan_macip_lens = htole32(vlan_macip_lens); - TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); - TXD->seqnum_seed = htole32(0); - TXD->mss_l4len_idx = htole32(mss_l4len_idx); - - /* We've consumed the first desc, adjust counters */ - if (++ctxd == txr->num_desc) - ctxd = 0; - txr->next_avail_desc = ctxd; - --txr->tx_avail; - - return (0); -} - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - * TRUE return means there's work in the ring to clean, FALSE its empty. - **********************************************************************/ -static bool -igb_txeof(struct tx_ring *txr) -{ - struct adapter *adapter = txr->adapter; -#ifdef DEV_NETMAP - struct ifnet *ifp = adapter->ifp; -#endif /* DEV_NETMAP */ - u32 work, processed = 0; - int limit = adapter->tx_process_limit; - struct igb_tx_buf *buf; - union e1000_adv_tx_desc *txd; - - mtx_assert(&txr->tx_mtx, MA_OWNED); - -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, txr->me)) - return (FALSE); -#endif /* DEV_NETMAP */ - - if (txr->tx_avail == txr->num_desc) { - txr->queue_status = IGB_QUEUE_IDLE; - return FALSE; - } - - /* Get work starting point */ - work = txr->next_to_clean; - buf = &txr->tx_buffers[work]; - txd = &txr->tx_base[work]; - work -= txr->num_desc; /* The distance to ring end */ - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - do { - union e1000_adv_tx_desc *eop = buf->eop; - if (eop == NULL) /* No work */ - break; - - if ((eop->wb.status & E1000_TXD_STAT_DD) == 0) - break; /* I/O not complete */ - - if (buf->m_head) { - txr->bytes += - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(txr->txtag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } - buf->eop = NULL; - ++txr->tx_avail; - - /* We clean the range if multi segment */ - while (txd != eop) { - ++txd; - ++buf; - ++work; - /* wrap the ring? */ - if (__predict_false(!work)) { - work -= txr->num_desc; - buf = txr->tx_buffers; - txd = txr->tx_base; - } - if (buf->m_head) { - txr->bytes += - buf->m_head->m_pkthdr.len; - bus_dmamap_sync(txr->txtag, - buf->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txr->txtag, - buf->map); - m_freem(buf->m_head); - buf->m_head = NULL; - } - ++txr->tx_avail; - buf->eop = NULL; - - } - ++txr->packets; - ++processed; - txr->watchdog_time = ticks; - - /* Try the next packet */ - ++txd; - ++buf; - ++work; - /* reset with a wrap */ - if (__predict_false(!work)) { - work -= txr->num_desc; - buf = txr->tx_buffers; - txd = txr->tx_base; - } - prefetch(txd); - } while (__predict_true(--limit)); - - bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - work += txr->num_desc; - txr->next_to_clean = work; - - /* - ** Watchdog calculation, we know there's - ** work outstanding or the first return - ** would have been taken, so none processed - ** for too long indicates a hang. - */ - if ((!processed) && ((ticks - txr->watchdog_time) > IGB_WATCHDOG)) - txr->queue_status |= IGB_QUEUE_HUNG; - - if (txr->tx_avail >= IGB_QUEUE_THRESHOLD) - txr->queue_status &= ~IGB_QUEUE_DEPLETED; - - if (txr->tx_avail == txr->num_desc) { - txr->queue_status = IGB_QUEUE_IDLE; - return (FALSE); - } - - return (TRUE); -} - -/********************************************************************* - * - * Refresh mbuf buffers for RX descriptor rings - * - now keeps its own state so discards due to resource - * exhaustion are unnecessary, if an mbuf cannot be obtained - * it just returns, keeping its placeholder, thus it can simply - * be recalled to try again. - * - **********************************************************************/ -static void -igb_refresh_mbufs(struct rx_ring *rxr, int limit) -{ - struct adapter *adapter = rxr->adapter; - bus_dma_segment_t hseg[1]; - bus_dma_segment_t pseg[1]; - struct igb_rx_buf *rxbuf; - struct mbuf *mh, *mp; - int i, j, nsegs, error; - bool refreshed = FALSE; - - i = j = rxr->next_to_refresh; - /* - ** Get one descriptor beyond - ** our work mark to control - ** the loop. - */ - if (++j == adapter->num_rx_desc) - j = 0; - - while (j != limit) { - rxbuf = &rxr->rx_buffers[i]; - /* No hdr mbuf used with header split off */ - if (rxr->hdr_split == FALSE) - goto no_split; - if (rxbuf->m_head == NULL) { - mh = m_gethdr(M_NOWAIT, MT_DATA); - if (mh == NULL) - goto update; - } else - mh = rxbuf->m_head; - - mh->m_pkthdr.len = mh->m_len = MHLEN; - mh->m_len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: hdr dmamap load" - " failure - %d\n", error); - m_free(mh); - rxbuf->m_head = NULL; - goto update; - } - rxbuf->m_head = mh; - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.hdr_addr = - htole64(hseg[0].ds_addr); -no_split: - if (rxbuf->m_pack == NULL) { - mp = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (mp == NULL) - goto update; - } else - mp = rxbuf->m_pack; - - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - printf("Refresh mbufs: payload dmamap load" - " failure - %d\n", error); - m_free(mp); - rxbuf->m_pack = NULL; - goto update; - } - rxbuf->m_pack = mp; - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_PREREAD); - rxr->rx_base[i].read.pkt_addr = - htole64(pseg[0].ds_addr); - refreshed = TRUE; /* I feel wefreshed :) */ - - i = j; /* our next is precalculated */ - rxr->next_to_refresh = i; - if (++j == adapter->num_rx_desc) - j = 0; - } -update: - if (refreshed) /* update tail */ - E1000_WRITE_REG(&adapter->hw, - E1000_RDT(rxr->me), rxr->next_to_refresh); - return; -} - - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -igb_allocate_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - device_t dev = adapter->dev; - struct igb_rx_buf *rxbuf; - int i, bsize, error; - - bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; - if (!(rxr->rx_buffers = - (struct igb_rx_buf *) malloc(bsize, - M_DEVBUF, M_NOWAIT | M_ZERO))) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MSIZE, /* maxsize */ - 1, /* nsegments */ - MSIZE, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->htag))) { - device_printf(dev, "Unable to create RX DMA tag\n"); - goto fail; - } - - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MJUM9BYTES, /* maxsize */ - 1, /* nsegments */ - MJUM9BYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockfuncarg */ - &rxr->ptag))) { - device_printf(dev, "Unable to create RX payload DMA tag\n"); - goto fail; - } - - for (i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - error = bus_dmamap_create(rxr->htag, 0, &rxbuf->hmap); - if (error) { - device_printf(dev, - "Unable to create RX head DMA maps\n"); - goto fail; - } - error = bus_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); - if (error) { - device_printf(dev, - "Unable to create RX packet DMA maps\n"); - goto fail; - } - } - - return (0); - -fail: - /* Frees all, but can handle partial completion */ - igb_free_receive_structures(adapter); - return (error); -} - - -static void -igb_free_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct igb_rx_buf *rxbuf; - - - for (int i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - } -} - - -/********************************************************************* - * - * Initialize a receive ring and its buffers. - * - **********************************************************************/ -static int -igb_setup_receive_ring(struct rx_ring *rxr) -{ - struct adapter *adapter; - struct ifnet *ifp; - device_t dev; - struct igb_rx_buf *rxbuf; - bus_dma_segment_t pseg[1], hseg[1]; - struct lro_ctrl *lro = &rxr->lro; - int rsize, nsegs, error = 0; -#ifdef DEV_NETMAP - struct netmap_adapter *na = NA(rxr->adapter->ifp); - struct netmap_slot *slot; -#endif /* DEV_NETMAP */ - - adapter = rxr->adapter; - dev = adapter->dev; - ifp = adapter->ifp; - - /* Clear the ring contents */ - IGB_RX_LOCK(rxr); -#ifdef DEV_NETMAP - slot = netmap_reset(na, NR_RX, rxr->me, 0); -#endif /* DEV_NETMAP */ - rsize = roundup2(adapter->num_rx_desc * - sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); - bzero((void *)rxr->rx_base, rsize); - - /* - ** Free current RX buffer structures and their mbufs - */ - igb_free_receive_ring(rxr); - - /* Configure for header split? */ - if (igb_header_split) - rxr->hdr_split = TRUE; - - /* Now replenish the ring mbufs */ - for (int j = 0; j < adapter->num_rx_desc; ++j) { - struct mbuf *mh, *mp; - - rxbuf = &rxr->rx_buffers[j]; -#ifdef DEV_NETMAP - if (slot) { - /* slot sj is mapped to the j-th NIC-ring entry */ - int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + sj, &paddr); - netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); - /* Update descriptor */ - rxr->rx_base[j].read.pkt_addr = htole64(paddr); - continue; - } -#endif /* DEV_NETMAP */ - if (rxr->hdr_split == FALSE) - goto skip_head; - - /* First the header */ - rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); - if (rxbuf->m_head == NULL) { - error = ENOBUFS; - goto fail; - } - m_adj(rxbuf->m_head, ETHER_ALIGN); - mh = rxbuf->m_head; - mh->m_len = mh->m_pkthdr.len = MHLEN; - mh->m_flags |= M_PKTHDR; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->htag, - rxbuf->hmap, rxbuf->m_head, hseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) /* Nothing elegant to do here */ - goto fail; - bus_dmamap_sync(rxr->htag, - rxbuf->hmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); - -skip_head: - /* Now the payload cluster */ - rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); - if (rxbuf->m_pack == NULL) { - error = ENOBUFS; - goto fail; - } - mp = rxbuf->m_pack; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; - /* Get the memory mapping */ - error = bus_dmamap_load_mbuf_sg(rxr->ptag, - rxbuf->pmap, mp, pseg, - &nsegs, BUS_DMA_NOWAIT); - if (error != 0) - goto fail; - bus_dmamap_sync(rxr->ptag, - rxbuf->pmap, BUS_DMASYNC_PREREAD); - /* Update descriptor */ - rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); - } - - /* Setup our descriptor indices */ - rxr->next_to_check = 0; - rxr->next_to_refresh = adapter->num_rx_desc - 1; - rxr->lro_enabled = FALSE; - rxr->rx_split_packets = 0; - rxr->rx_bytes = 0; - - rxr->fmp = NULL; - rxr->lmp = NULL; - - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* - ** Now set up the LRO interface, we - ** also only do head split when LRO - ** is enabled, since so often they - ** are undesirable in similar setups. - */ - if (ifp->if_capenable & IFCAP_LRO) { - error = tcp_lro_init(lro); - if (error) { - device_printf(dev, "LRO Initialization failed!\n"); - goto fail; - } - INIT_DEBUGOUT("RX LRO Initialized\n"); - rxr->lro_enabled = TRUE; - lro->ifp = adapter->ifp; - } - - IGB_RX_UNLOCK(rxr); - return (0); - -fail: - igb_free_receive_ring(rxr); - IGB_RX_UNLOCK(rxr); - return (error); -} - - -/********************************************************************* - * - * Initialize all receive rings. - * - **********************************************************************/ -static int -igb_setup_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - int i; - - for (i = 0; i < adapter->num_queues; i++, rxr++) - if (igb_setup_receive_ring(rxr)) - goto fail; - - return (0); -fail: - /* - * Free RX buffers allocated so far, we will only handle - * the rings that completed, the failing case will have - * cleaned up for itself. 'i' is the endpoint. - */ - for (int j = 0; j < i; ++j) { - rxr = &adapter->rx_rings[j]; - IGB_RX_LOCK(rxr); - igb_free_receive_ring(rxr); - IGB_RX_UNLOCK(rxr); - } - - return (ENOBUFS); -} - -/* - * Initialise the RSS mapping for NICs that support multiple transmit/ - * receive rings. - */ -static void -igb_initialise_rss_mapping(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - int i; - int queue_id; - u32 reta; - u32 rss_key[10], mrqc, shift = 0; - - /* XXX? */ - if (adapter->hw.mac.type == e1000_82575) - shift = 6; - - /* - * The redirection table controls which destination - * queue each bucket redirects traffic to. - * Each DWORD represents four queues, with the LSB - * being the first queue in the DWORD. - * - * This just allocates buckets to queues using round-robin - * allocation. - * - * NOTE: It Just Happens to line up with the default - * RSS allocation method. - */ - - /* Warning FM follows */ - reta = 0; - for (i = 0; i < 128; i++) { -#ifdef RSS - queue_id = rss_get_indirection_to_bucket(i); - /* - * If we have more queues than buckets, we'll - * end up mapping buckets to a subset of the - * queues. - * - * If we have more buckets than queues, we'll - * end up instead assigning multiple buckets - * to queues. - * - * Both are suboptimal, but we need to handle - * the case so we don't go out of bounds - * indexing arrays and such. - */ - queue_id = queue_id % adapter->num_queues; -#else - queue_id = (i % adapter->num_queues); -#endif - /* Adjust if required */ - queue_id = queue_id << shift; - - /* - * The low 8 bits are for hash value (n+0); - * The next 8 bits are for hash value (n+1), etc. - */ - reta = reta >> 8; - reta = reta | ( ((uint32_t) queue_id) << 24); - if ((i & 3) == 3) { - E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta); - reta = 0; - } - } - - /* Now fill in hash table */ - - /* - * MRQC: Multiple Receive Queues Command - * Set queuing to RSS control, number depends on the device. - */ - mrqc = E1000_MRQC_ENABLE_RSS_8Q; - -#ifdef RSS - /* XXX ew typecasting */ - rss_getkey((uint8_t *) &rss_key); -#else - arc4rand(&rss_key, sizeof(rss_key), 0); -#endif - for (i = 0; i < 10; i++) - E1000_WRITE_REG_ARRAY(hw, - E1000_RSSRK(0), i, rss_key[i]); - - /* - * Configure the RSS fields to hash upon. - */ - mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | - E1000_MRQC_RSS_FIELD_IPV4_TCP); - mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | - E1000_MRQC_RSS_FIELD_IPV6_TCP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | - E1000_MRQC_RSS_FIELD_IPV6_UDP); - mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | - E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); - - E1000_WRITE_REG(hw, E1000_MRQC, mrqc); -} - -/********************************************************************* - * - * Enable receive unit. - * - **********************************************************************/ -static void -igb_initialize_receive_units(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - struct ifnet *ifp = adapter->ifp; - struct e1000_hw *hw = &adapter->hw; - u32 rctl, rxcsum, psize, srrctl = 0; - - INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); - - /* - * Make sure receives are disabled while setting - * up the descriptor ring - */ - rctl = E1000_READ_REG(hw, E1000_RCTL); - E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - - /* - ** Set up for header split - */ - if (igb_header_split) { - /* Use a standard mbuf for the header */ - srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; - srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; - } else - srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; - - /* - ** Set up for jumbo frames - */ - if (ifp->if_mtu > ETHERMTU) { - rctl |= E1000_RCTL_LPE; - if (adapter->rx_mbuf_sz == MJUMPAGESIZE) { - srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; - } else if (adapter->rx_mbuf_sz > MJUMPAGESIZE) { - srrctl |= 8192 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX; - } - /* Set maximum packet len */ - psize = adapter->max_frame_size; - /* are we on a vlan? */ - if (adapter->ifp->if_vlantrunk != NULL) - psize += VLAN_TAG_SIZE; - E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); - } else { - rctl &= ~E1000_RCTL_LPE; - srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; - rctl |= E1000_RCTL_SZ_2048; - } - - /* - * If TX flow control is disabled and there's >1 queue defined, - * enable DROP. - * - * This drops frames rather than hanging the RX MAC for all queues. - */ - if ((adapter->num_queues > 1) && - (adapter->fc == e1000_fc_none || - adapter->fc == e1000_fc_rx_pause)) { - srrctl |= E1000_SRRCTL_DROP_EN; - } - - /* Setup the Base and Length of the Rx Descriptor Rings */ - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - u64 bus_addr = rxr->rxdma.dma_paddr; - u32 rxdctl; - - E1000_WRITE_REG(hw, E1000_RDLEN(i), - adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(hw, E1000_RDBAH(i), - (uint32_t)(bus_addr >> 32)); - E1000_WRITE_REG(hw, E1000_RDBAL(i), - (uint32_t)bus_addr); - E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); - /* Enable this Queue */ - rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); - rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; - rxdctl &= 0xFFF00000; - rxdctl |= IGB_RX_PTHRESH; - rxdctl |= IGB_RX_HTHRESH << 8; - rxdctl |= IGB_RX_WTHRESH << 16; - E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); - } - - /* - ** Setup for RX MultiQueue - */ - rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); - if (adapter->num_queues >1) { - - /* rss setup */ - igb_initialise_rss_mapping(adapter); - - /* - ** NOTE: Receive Full-Packet Checksum Offload - ** is mutually exclusive with Multiqueue. However - ** this is not the same as TCP/IP checksums which - ** still work. - */ - rxcsum |= E1000_RXCSUM_PCSD; -#if __FreeBSD_version >= 800000 - /* For SCTP Offload */ - if ((hw->mac.type != e1000_82575) && - (ifp->if_capenable & IFCAP_RXCSUM)) - rxcsum |= E1000_RXCSUM_CRCOFL; -#endif - } else { - /* Non RSS setup */ - if (ifp->if_capenable & IFCAP_RXCSUM) { - rxcsum |= E1000_RXCSUM_IPPCSE; -#if __FreeBSD_version >= 800000 - if (adapter->hw.mac.type != e1000_82575) - rxcsum |= E1000_RXCSUM_CRCOFL; -#endif - } else - rxcsum &= ~E1000_RXCSUM_TUOFL; - } - E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); - - /* Setup the Receive Control Register */ - rctl &= ~(3 << E1000_RCTL_MO_SHIFT); - rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | - E1000_RCTL_RDMTS_HALF | - (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - /* Strip CRC bytes. */ - rctl |= E1000_RCTL_SECRC; - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - /* Don't store bad packets */ - rctl &= ~E1000_RCTL_SBP; - - /* Enable Receives */ - E1000_WRITE_REG(hw, E1000_RCTL, rctl); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers - * - needs to be after enable - */ - for (int i = 0; i < adapter->num_queues; i++) { - rxr = &adapter->rx_rings[i]; - E1000_WRITE_REG(hw, E1000_RDH(i), rxr->next_to_check); -#ifdef DEV_NETMAP - /* - * an init() while a netmap client is active must - * preserve the rx buffers passed to userspace. - * In this driver it means we adjust RDT to - * something different from next_to_refresh - * (which is not used in netmap mode). - */ - if (ifp->if_capenable & IFCAP_NETMAP) { - struct netmap_adapter *na = NA(adapter->ifp); - struct netmap_kring *kring = &na->rx_rings[i]; - int t = rxr->next_to_refresh - nm_kr_rxspace(kring); - - if (t >= adapter->num_rx_desc) - t -= adapter->num_rx_desc; - else if (t < 0) - t += adapter->num_rx_desc; - E1000_WRITE_REG(hw, E1000_RDT(i), t); - } else -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(hw, E1000_RDT(i), rxr->next_to_refresh); - } - return; -} - -/********************************************************************* - * - * Free receive rings. - * - **********************************************************************/ -static void -igb_free_receive_structures(struct adapter *adapter) -{ - struct rx_ring *rxr = adapter->rx_rings; - - for (int i = 0; i < adapter->num_queues; i++, rxr++) { - struct lro_ctrl *lro = &rxr->lro; - igb_free_receive_buffers(rxr); - tcp_lro_free(lro); - igb_dma_free(adapter, &rxr->rxdma); - } - - free(adapter->rx_rings, M_DEVBUF); -} - -/********************************************************************* - * - * Free receive ring data structures. - * - **********************************************************************/ -static void -igb_free_receive_buffers(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - struct igb_rx_buf *rxbuf; - int i; - - INIT_DEBUGOUT("free_receive_structures: begin"); - - /* Cleanup any existing buffers */ - if (rxr->rx_buffers != NULL) { - for (i = 0; i < adapter->num_rx_desc; i++) { - rxbuf = &rxr->rx_buffers[i]; - if (rxbuf->m_head != NULL) { - bus_dmamap_sync(rxr->htag, rxbuf->hmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - rxbuf->m_head->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_head); - } - if (rxbuf->m_pack != NULL) { - bus_dmamap_sync(rxr->ptag, rxbuf->pmap, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - rxbuf->m_pack->m_flags |= M_PKTHDR; - m_freem(rxbuf->m_pack); - } - rxbuf->m_head = NULL; - rxbuf->m_pack = NULL; - if (rxbuf->hmap != NULL) { - bus_dmamap_destroy(rxr->htag, rxbuf->hmap); - rxbuf->hmap = NULL; - } - if (rxbuf->pmap != NULL) { - bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); - rxbuf->pmap = NULL; - } - } - if (rxr->rx_buffers != NULL) { - free(rxr->rx_buffers, M_DEVBUF); - rxr->rx_buffers = NULL; - } - } - - if (rxr->htag != NULL) { - bus_dma_tag_destroy(rxr->htag); - rxr->htag = NULL; - } - if (rxr->ptag != NULL) { - bus_dma_tag_destroy(rxr->ptag); - rxr->ptag = NULL; - } -} - -static __inline void -igb_rx_discard(struct rx_ring *rxr, int i) -{ - struct igb_rx_buf *rbuf; - - rbuf = &rxr->rx_buffers[i]; - - /* Partially received? Free the chain */ - if (rxr->fmp != NULL) { - rxr->fmp->m_flags |= M_PKTHDR; - m_freem(rxr->fmp); - rxr->fmp = NULL; - rxr->lmp = NULL; - } - - /* - ** With advanced descriptors the writeback - ** clobbers the buffer addrs, so its easier - ** to just free the existing mbufs and take - ** the normal refresh path to get new buffers - ** and mapping. - */ - if (rbuf->m_head) { - m_free(rbuf->m_head); - rbuf->m_head = NULL; - bus_dmamap_unload(rxr->htag, rbuf->hmap); - } - - if (rbuf->m_pack) { - m_free(rbuf->m_pack); - rbuf->m_pack = NULL; - bus_dmamap_unload(rxr->ptag, rbuf->pmap); - } - - return; -} - -static __inline void -igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) -{ - - /* - * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet - * should be computed by hardware. Also it should not have VLAN tag in - * ethernet header. - */ - if (rxr->lro_enabled && - (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && - (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == - (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && - (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { - /* - * Send to the stack if: - ** - LRO not enabled, or - ** - no LRO resources, or - ** - lro enqueue fails - */ - if (rxr->lro.lro_cnt != 0) - if (tcp_lro_rx(&rxr->lro, m, 0) == 0) - return; - } - IGB_RX_UNLOCK(rxr); - (*ifp->if_input)(ifp, m); - IGB_RX_LOCK(rxr); -} - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * Return TRUE if more to clean, FALSE otherwise - *********************************************************************/ -static bool -igb_rxeof(struct igb_queue *que, int count, int *done) -{ - struct adapter *adapter = que->adapter; - struct rx_ring *rxr = que->rxr; - struct ifnet *ifp = adapter->ifp; - struct lro_ctrl *lro = &rxr->lro; - int i, processed = 0, rxdone = 0; - u32 ptype, staterr = 0; - union e1000_adv_rx_desc *cur; - - IGB_RX_LOCK(rxr); - /* Sync the ring. */ - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, rxr->me, &processed)) { - IGB_RX_UNLOCK(rxr); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - /* Main clean loop */ - for (i = rxr->next_to_check; count != 0;) { - struct mbuf *sendmp, *mh, *mp; - struct igb_rx_buf *rxbuf; - u16 hlen, plen, hdr, vtag, pkt_info; - bool eop = FALSE; - - cur = &rxr->rx_base[i]; - staterr = le32toh(cur->wb.upper.status_error); - if ((staterr & E1000_RXD_STAT_DD) == 0) - break; - if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) - break; - count--; - sendmp = mh = mp = NULL; - cur->wb.upper.status_error = 0; - rxbuf = &rxr->rx_buffers[i]; - plen = le16toh(cur->wb.upper.length); - ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; - if (((adapter->hw.mac.type == e1000_i350) || - (adapter->hw.mac.type == e1000_i354)) && - (staterr & E1000_RXDEXT_STATERR_LB)) - vtag = be16toh(cur->wb.upper.vlan); - else - vtag = le16toh(cur->wb.upper.vlan); - hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); - pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); - eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); - - /* - * Free the frame (all segments) if we're at EOP and - * it's an error. - * - * The datasheet states that EOP + status is only valid for - * the final segment in a multi-segment frame. - */ - if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { - adapter->dropped_pkts++; - ++rxr->rx_discarded; - igb_rx_discard(rxr, i); - goto next_desc; - } - - /* - ** The way the hardware is configured to - ** split, it will ONLY use the header buffer - ** when header split is enabled, otherwise we - ** get normal behavior, ie, both header and - ** payload are DMA'd into the payload buffer. - ** - ** The fmp test is to catch the case where a - ** packet spans multiple descriptors, in that - ** case only the first header is valid. - */ - if (rxr->hdr_split && rxr->fmp == NULL) { - bus_dmamap_unload(rxr->htag, rxbuf->hmap); - hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> - E1000_RXDADV_HDRBUFLEN_SHIFT; - if (hlen > IGB_HDR_BUF) - hlen = IGB_HDR_BUF; - mh = rxr->rx_buffers[i].m_head; - mh->m_len = hlen; - /* clear buf pointer for refresh */ - rxbuf->m_head = NULL; - /* - ** Get the payload length, this - ** could be zero if its a small - ** packet. - */ - if (plen > 0) { - mp = rxr->rx_buffers[i].m_pack; - mp->m_len = plen; - mh->m_next = mp; - /* clear buf pointer */ - rxbuf->m_pack = NULL; - rxr->rx_split_packets++; - } - } else { - /* - ** Either no header split, or a - ** secondary piece of a fragmented - ** split packet. - */ - mh = rxr->rx_buffers[i].m_pack; - mh->m_len = plen; - /* clear buf info for refresh */ - rxbuf->m_pack = NULL; - } - bus_dmamap_unload(rxr->ptag, rxbuf->pmap); - - ++processed; /* So we know when to refresh */ - - /* Initial frame - setup */ - if (rxr->fmp == NULL) { - mh->m_pkthdr.len = mh->m_len; - /* Save the head of the chain */ - rxr->fmp = mh; - rxr->lmp = mh; - if (mp != NULL) { - /* Add payload if split */ - mh->m_pkthdr.len += mp->m_len; - rxr->lmp = mh->m_next; - } - } else { - /* Chain mbuf's together */ - rxr->lmp->m_next = mh; - rxr->lmp = rxr->lmp->m_next; - rxr->fmp->m_pkthdr.len += mh->m_len; - } - - if (eop) { - rxr->fmp->m_pkthdr.rcvif = ifp; - rxr->rx_packets++; - /* capture data for AIM */ - rxr->packets++; - rxr->bytes += rxr->fmp->m_pkthdr.len; - rxr->rx_bytes += rxr->fmp->m_pkthdr.len; - - if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) - igb_rx_checksum(staterr, rxr->fmp, ptype); - - if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && - (staterr & E1000_RXD_STAT_VP) != 0) { - rxr->fmp->m_pkthdr.ether_vtag = vtag; - rxr->fmp->m_flags |= M_VLANTAG; - } - - /* - * In case of multiqueue, we have RXCSUM.PCSD bit set - * and never cleared. This means we have RSS hash - * available to be used. - */ - if (adapter->num_queues > 1) { - rxr->fmp->m_pkthdr.flowid = - le32toh(cur->wb.lower.hi_dword.rss); - switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { - case E1000_RXDADV_RSSTYPE_IPV4_TCP: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV4); - break; - case E1000_RXDADV_RSSTYPE_IPV4: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV4); - break; - case E1000_RXDADV_RSSTYPE_IPV6_TCP: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV6); - break; - case E1000_RXDADV_RSSTYPE_IPV6_EX: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV6_EX); - break; - case E1000_RXDADV_RSSTYPE_IPV6: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_IPV6); - break; - case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_RSS_TCP_IPV6_EX); - break; - default: - /* XXX fallthrough */ - M_HASHTYPE_SET(rxr->fmp, - M_HASHTYPE_OPAQUE_HASH); - } - } else { -#ifndef IGB_LEGACY_TX - rxr->fmp->m_pkthdr.flowid = que->msix; - M_HASHTYPE_SET(rxr->fmp, M_HASHTYPE_OPAQUE); -#endif - } - sendmp = rxr->fmp; - /* Make sure to set M_PKTHDR. */ - sendmp->m_flags |= M_PKTHDR; - rxr->fmp = NULL; - rxr->lmp = NULL; - } - -next_desc: - bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - /* - ** Send to the stack or LRO - */ - if (sendmp != NULL) { - rxr->next_to_check = i; - igb_rx_input(rxr, ifp, sendmp, ptype); - i = rxr->next_to_check; - rxdone++; - } - - /* Every 8 descriptors we go to refresh mbufs */ - if (processed == 8) { - igb_refresh_mbufs(rxr, i); - processed = 0; - } - } - - /* Catch any remainders */ - if (igb_rx_unrefreshed(rxr)) - igb_refresh_mbufs(rxr, i); - - rxr->next_to_check = i; - - /* - * Flush any outstanding LRO work - */ - tcp_lro_flush_all(lro); - - if (done != NULL) - *done += rxdone; - - IGB_RX_UNLOCK(rxr); - return ((staterr & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) -{ - u16 status = (u16)staterr; - u8 errors = (u8) (staterr >> 24); - int sctp; - - /* Ignore Checksum bit is set */ - if (status & E1000_RXD_STAT_IXSM) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && - (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) - sctp = 1; - else - sctp = 0; - if (status & E1000_RXD_STAT_IPCS) { - /* Did it pass? */ - if (!(errors & E1000_RXD_ERR_IPE)) { - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; - } else - mp->m_pkthdr.csum_flags = 0; - } - - if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { - u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); -#if __FreeBSD_version >= 800000 - if (sctp) /* reassign */ - type = CSUM_SCTP_VALID; -#endif - /* Did it pass? */ - if (!(errors & E1000_RXD_ERR_TCPE)) { - mp->m_pkthdr.csum_flags |= type; - if (sctp == 0) - mp->m_pkthdr.csum_data = htons(0xffff); - } - } - return; -} - -/* - * This routine is run via an vlan - * config EVENT - */ -static void -igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct adapter *adapter = ifp->if_softc; - u32 index, bit; - - if (ifp->if_softc != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IGB_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] |= (1 << bit); - ++adapter->num_vlans; - /* Change hw filter setting */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) - igb_setup_vlan_hw_support(adapter); - IGB_CORE_UNLOCK(adapter); -} - -/* - * This routine is run via an vlan - * unconfig EVENT - */ -static void -igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) -{ - struct adapter *adapter = ifp->if_softc; - u32 index, bit; - - if (ifp->if_softc != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - IGB_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] &= ~(1 << bit); - --adapter->num_vlans; - /* Change hw filter setting */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) - igb_setup_vlan_hw_support(adapter); - IGB_CORE_UNLOCK(adapter); -} - -static void -igb_setup_vlan_hw_support(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct ifnet *ifp = adapter->ifp; - u32 reg; - - if (adapter->vf_ifp) { - e1000_rlpml_set_vf(hw, - adapter->max_frame_size + VLAN_TAG_SIZE); - return; - } - - reg = E1000_READ_REG(hw, E1000_CTRL); - reg |= E1000_CTRL_VME; - E1000_WRITE_REG(hw, E1000_CTRL, reg); - - /* Enable the Filter Table */ - if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= ~E1000_RCTL_CFIEN; - reg |= E1000_RCTL_VFE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); - } - - /* Update the frame size */ - E1000_WRITE_REG(&adapter->hw, E1000_RLPML, - adapter->max_frame_size + VLAN_TAG_SIZE); - - /* Don't bother with table if no vlans */ - if ((adapter->num_vlans == 0) || - ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) - return; - /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ - for (int i = 0; i < IGB_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) { - if (adapter->vf_ifp) - e1000_vfta_set_vf(hw, - adapter->shadow_vfta[i], TRUE); - else - e1000_write_vfta(hw, - i, adapter->shadow_vfta[i]); - } -} - -static void -igb_enable_intr(struct adapter *adapter) -{ - /* With RSS set up what to auto clear */ - if (adapter->msix_mem) { - u32 mask = (adapter->que_mask | adapter->link_mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIAC, mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIAM, mask); - E1000_WRITE_REG(&adapter->hw, E1000_EIMS, mask); - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - E1000_IMS_LSC); - } else { - E1000_WRITE_REG(&adapter->hw, E1000_IMS, - IMS_ENABLE_MASK); - } - E1000_WRITE_FLUSH(&adapter->hw); - - return; -} - -static void -igb_disable_intr(struct adapter *adapter) -{ - if (adapter->msix_mem) { - E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); - E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); - } - E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); - E1000_WRITE_FLUSH(&adapter->hw); - return; -} - -/* - * Bit of a misnomer, what this really means is - * to enable OS management of the system... aka - * to disable special hardware management features - */ -static void -igb_init_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* disable hardware interception of ARP */ - manc &= ~(E1000_MANC_ARP_EN); - - /* enable receiving management packets to the host */ - manc |= E1000_MANC_EN_MNG2HOST; - manc2h |= 1 << 5; /* Mng Port 623 */ - manc2h |= 1 << 6; /* Mng Port 664 */ - E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * Give control back to hardware management - * controller if there is one. - */ -static void -igb_release_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* re-enable hardware interception of ARP */ - manc |= E1000_MANC_ARP_EN; - manc &= ~E1000_MANC_EN_MNG2HOST; - - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that - * the driver is loaded. - * - */ -static void -igb_get_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (adapter->vf_ifp) - return; - - /* Let firmware know the driver has taken over */ - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); -} - -/* - * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that the - * driver is no longer loaded. - * - */ -static void -igb_release_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (adapter->vf_ifp) - return; - - /* Let firmware taken over control of h/w */ - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); -} - -static int -igb_is_valid_ether_addr(uint8_t *addr) -{ - char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; - - if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { - return (FALSE); - } - - return (TRUE); -} - - -/* - * Enable PCI Wake On Lan capability - */ -static void -igb_enable_wakeup(device_t dev) -{ - u16 cap, status; - u8 id; - - /* First find the capabilities pointer*/ - cap = pci_read_config(dev, PCIR_CAP_PTR, 2); - /* Read the PM Capabilities */ - id = pci_read_config(dev, cap, 1); - if (id != PCIY_PMG) /* Something wrong */ - return; - /* OK, we have the power capabilities, so - now get the status register */ - cap += PCIR_POWER_STATUS; - status = pci_read_config(dev, cap, 2); - status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, cap, status, 2); - return; -} - -static void -igb_led_func(void *arg, int onoff) -{ - struct adapter *adapter = arg; - - IGB_CORE_LOCK(adapter); - if (onoff) { - e1000_setup_led(&adapter->hw); - e1000_led_on(&adapter->hw); - } else { - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); - } - IGB_CORE_UNLOCK(adapter); -} - -static uint64_t -igb_get_vf_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - struct e1000_vf_stats *stats; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr; - uint64_t rv; -#endif - - adapter = if_getsoftc(ifp); - stats = (struct e1000_vf_stats *)adapter->stats; - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (stats->gprc); - case IFCOUNTER_OPACKETS: - return (stats->gptc); - case IFCOUNTER_IBYTES: - return (stats->gorc); - case IFCOUNTER_OBYTES: - return (stats->gotc); - case IFCOUNTER_IMCASTS: - return (stats->mprc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts); - case IFCOUNTER_OERRORS: - return (adapter->watchdog_events); -#ifndef IGB_LEGACY_TX - case IFCOUNTER_OQDROPS: - rv = 0; - txr = adapter->tx_rings; - for (int i = 0; i < adapter->num_queues; i++, txr++) - rv += txr->br->br_drops; - return (rv); -#endif - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -static uint64_t -igb_get_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - struct e1000_hw_stats *stats; -#ifndef IGB_LEGACY_TX - struct tx_ring *txr; - uint64_t rv; -#endif - - adapter = if_getsoftc(ifp); - if (adapter->vf_ifp) - return (igb_get_vf_counter(ifp, cnt)); - - stats = (struct e1000_hw_stats *)adapter->stats; - - switch (cnt) { - case IFCOUNTER_IPACKETS: - return (stats->gprc); - case IFCOUNTER_OPACKETS: - return (stats->gptc); - case IFCOUNTER_IBYTES: - return (stats->gorc); - case IFCOUNTER_OBYTES: - return (stats->gotc); - case IFCOUNTER_IMCASTS: - return (stats->mprc); - case IFCOUNTER_OMCASTS: - return (stats->mptc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts + stats->rxerrc + - stats->crcerrs + stats->algnerrc + - stats->ruc + stats->roc + stats->cexterr); - case IFCOUNTER_OERRORS: - return (stats->ecol + stats->latecol + - adapter->watchdog_events); - case IFCOUNTER_COLLISIONS: - return (stats->colc); - case IFCOUNTER_IQDROPS: - return (stats->mpc); -#ifndef IGB_LEGACY_TX - case IFCOUNTER_OQDROPS: - rv = 0; - txr = adapter->tx_rings; - for (int i = 0; i < adapter->num_queues; i++, txr++) - rv += txr->br->br_drops; - return (rv); -#endif - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -/********************************************************************** - * - * Update the board statistics counters. - * - **********************************************************************/ -static void -igb_update_stats_counters(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_hw_stats *stats; - - /* - ** The virtual function adapter has only a - ** small controlled set of stats, do only - ** those and return. - */ - if (adapter->vf_ifp) { - igb_update_vf_stats_counters(adapter); - return; - } - - stats = (struct e1000_hw_stats *)adapter->stats; - - if (adapter->hw.phy.media_type == e1000_media_type_copper || - (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { - stats->symerrs += - E1000_READ_REG(hw,E1000_SYMERRS); - stats->sec += E1000_READ_REG(hw, E1000_SEC); - } - - stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); - stats->mpc += E1000_READ_REG(hw, E1000_MPC); - stats->scc += E1000_READ_REG(hw, E1000_SCC); - stats->ecol += E1000_READ_REG(hw, E1000_ECOL); - - stats->mcc += E1000_READ_REG(hw, E1000_MCC); - stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); - stats->colc += E1000_READ_REG(hw, E1000_COLC); - stats->dc += E1000_READ_REG(hw, E1000_DC); - stats->rlec += E1000_READ_REG(hw, E1000_RLEC); - stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); - stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); - /* - ** For watchdog management we need to know if we have been - ** paused during the last interval, so capture that here. - */ - adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); - stats->xoffrxc += adapter->pause_frames; - stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); - stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); - stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); - stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); - stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); - stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); - stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); - stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); - stats->gprc += E1000_READ_REG(hw, E1000_GPRC); - stats->bprc += E1000_READ_REG(hw, E1000_BPRC); - stats->mprc += E1000_READ_REG(hw, E1000_MPRC); - stats->gptc += E1000_READ_REG(hw, E1000_GPTC); - - /* For the 64-bit byte counters the low dword must be read first. */ - /* Both registers clear on the read of the high dword */ - - stats->gorc += E1000_READ_REG(hw, E1000_GORCL) + - ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32); - stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) + - ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32); - - stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); - stats->ruc += E1000_READ_REG(hw, E1000_RUC); - stats->rfc += E1000_READ_REG(hw, E1000_RFC); - stats->roc += E1000_READ_REG(hw, E1000_ROC); - stats->rjc += E1000_READ_REG(hw, E1000_RJC); - - stats->mgprc += E1000_READ_REG(hw, E1000_MGTPRC); - stats->mgpdc += E1000_READ_REG(hw, E1000_MGTPDC); - stats->mgptc += E1000_READ_REG(hw, E1000_MGTPTC); - - stats->tor += E1000_READ_REG(hw, E1000_TORL) + - ((u64)E1000_READ_REG(hw, E1000_TORH) << 32); - stats->tot += E1000_READ_REG(hw, E1000_TOTL) + - ((u64)E1000_READ_REG(hw, E1000_TOTH) << 32); - - stats->tpr += E1000_READ_REG(hw, E1000_TPR); - stats->tpt += E1000_READ_REG(hw, E1000_TPT); - stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); - stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); - stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); - stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); - stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); - stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); - stats->mptc += E1000_READ_REG(hw, E1000_MPTC); - stats->bptc += E1000_READ_REG(hw, E1000_BPTC); - - /* Interrupt Counts */ - - stats->iac += E1000_READ_REG(hw, E1000_IAC); - stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); - stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); - stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); - stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); - stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); - stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); - stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); - stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); - - /* Host to Card Statistics */ - - stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); - stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); - stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); - stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); - stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); - stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); - stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); - stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) + - ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32)); - stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) + - ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32)); - stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); - stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); - stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); - - stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); - stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); - stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); - stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); - stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); - stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); - - /* Driver specific counters */ - adapter->device_control = E1000_READ_REG(hw, E1000_CTRL); - adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL); - adapter->int_mask = E1000_READ_REG(hw, E1000_IMS); - adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS); - adapter->packet_buf_alloc_tx = - ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16); - adapter->packet_buf_alloc_rx = - (E1000_READ_REG(hw, E1000_PBA) & 0xffff); -} - - -/********************************************************************** - * - * Initialize the VF board statistics counters. - * - **********************************************************************/ -static void -igb_vf_init_stats(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_vf_stats *stats; - - stats = (struct e1000_vf_stats *)adapter->stats; - if (stats == NULL) - return; - stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC); - stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC); - stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC); - stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC); - stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC); -} - -/********************************************************************** - * - * Update the VF board statistics counters. - * - **********************************************************************/ -static void -igb_update_vf_stats_counters(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - struct e1000_vf_stats *stats; - - if (adapter->link_speed == 0) - return; - - stats = (struct e1000_vf_stats *)adapter->stats; - - UPDATE_VF_REG(E1000_VFGPRC, - stats->last_gprc, stats->gprc); - UPDATE_VF_REG(E1000_VFGORC, - stats->last_gorc, stats->gorc); - UPDATE_VF_REG(E1000_VFGPTC, - stats->last_gptc, stats->gptc); - UPDATE_VF_REG(E1000_VFGOTC, - stats->last_gotc, stats->gotc); - UPDATE_VF_REG(E1000_VFMPRC, - stats->last_mprc, stats->mprc); -} - -/* Export a single 32-bit register via a read-only sysctl. */ -static int -igb_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - u_int val; - - adapter = oidp->oid_arg1; - val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); - return (sysctl_handle_int(oidp, &val, 0, req)); -} - -/* -** Tuneable interrupt rate handler -*/ -static int -igb_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS) -{ - struct igb_queue *que = ((struct igb_queue *)oidp->oid_arg1); - int error; - u32 reg, usec, rate; - - reg = E1000_READ_REG(&que->adapter->hw, E1000_EITR(que->msix)); - usec = ((reg & 0x7FFC) >> 2); - if (usec > 0) - rate = 1000000 / usec; - else - rate = 0; - error = sysctl_handle_int(oidp, &rate, 0, req); - if (error || !req->newptr) - return error; - return 0; -} - -/* - * Add sysctl variables, one per statistic, to the system. - */ -static void -igb_add_hw_stats(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - struct tx_ring *txr = adapter->tx_rings; - struct rx_ring *rxr = adapter->rx_rings; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct e1000_hw_stats *stats = adapter->stats; - - struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node; - struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list; - -#define QUEUE_NAME_LEN 32 - char namebuf[QUEUE_NAME_LEN]; - - /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", - CTLFLAG_RD, &adapter->dropped_pkts, - "Driver dropped packets"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", - CTLFLAG_RD, &adapter->link_irq, - "Link MSIX IRQ Handled"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", - CTLFLAG_RD, &adapter->mbuf_defrag_failed, - "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", - CTLFLAG_RD, &adapter->rx_overruns, - "RX overruns"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", - CTLFLAG_RD, &adapter->watchdog_events, - "Watchdog timeouts"); - - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", - CTLFLAG_RD, &adapter->device_control, - "Device Control Register"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", - CTLFLAG_RD, &adapter->rx_control, - "Receiver Control Register"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", - CTLFLAG_RD, &adapter->int_mask, - "Interrupt Mask"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", - CTLFLAG_RD, &adapter->eint_mask, - "Extended Interrupt Mask"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", - CTLFLAG_RD, &adapter->packet_buf_alloc_tx, - "Transmit Buffer Packet Allocation"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", - CTLFLAG_RD, &adapter->packet_buf_alloc_rx, - "Receive Buffer Packet Allocation"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", - CTLFLAG_RD, &adapter->hw.fc.high_water, 0, - "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", - CTLFLAG_RD, &adapter->hw.fc.low_water, 0, - "Flow Control Low Watermark"); - - for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { - struct lro_ctrl *lro = &rxr->lro; - - snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); - queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, - CTLFLAG_RD, NULL, "Queue Name"); - queue_list = SYSCTL_CHILDREN(queue_node); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate", - CTLTYPE_UINT | CTLFLAG_RD, &adapter->queues[i], - sizeof(&adapter->queues[i]), - igb_sysctl_interrupt_rate_handler, - "IU", "Interrupt Rate"); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(txr->me), - igb_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(txr->me), - igb_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", - CTLFLAG_RD, &txr->no_desc_avail, - "Queue Descriptors Unavailable"); - SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets", - CTLFLAG_RD, &txr->total_packets, - "Queue Packets Transmitted"); - - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(rxr->me), - igb_sysctl_reg_handler, "IU", - "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(rxr->me), - igb_sysctl_reg_handler, "IU", - "Receive Descriptor Tail"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", - CTLFLAG_RD, &rxr->rx_packets, - "Queue Packets Received"); - SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", - CTLFLAG_RD, &rxr->rx_bytes, - "Queue Bytes Received"); - SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_queued", - CTLFLAG_RD, &lro->lro_queued, 0, - "LRO Queued"); - SYSCTL_ADD_U64(ctx, queue_list, OID_AUTO, "lro_flushed", - CTLFLAG_RD, &lro->lro_flushed, 0, - "LRO Flushed"); - } - - /* MAC stats get their own sub node */ - - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", - CTLFLAG_RD, NULL, "MAC Statistics"); - stat_list = SYSCTL_CHILDREN(stat_node); - - /* - ** VF adapter has a very limited set of stats - ** since its not managing the metal, so to speak. - */ - if (adapter->vf_ifp) { - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &stats->gprc, - "Good Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &stats->gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &stats->gorc, - "Good Octets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &stats->gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &stats->mprc, - "Multicast Packets Received"); - return; - } - - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", - CTLFLAG_RD, &stats->ecol, - "Excessive collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", - CTLFLAG_RD, &stats->scc, - "Single collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", - CTLFLAG_RD, &stats->mcc, - "Multiple collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", - CTLFLAG_RD, &stats->latecol, - "Late collisions"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", - CTLFLAG_RD, &stats->colc, - "Collision Count"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", - CTLFLAG_RD, &stats->symerrs, - "Symbol Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", - CTLFLAG_RD, &stats->sec, - "Sequence Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", - CTLFLAG_RD, &stats->dc, - "Defer Count"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", - CTLFLAG_RD, &stats->mpc, - "Missed Packets"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_length_errors", - CTLFLAG_RD, &stats->rlec, - "Receive Length Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", - CTLFLAG_RD, &stats->rnbc, - "Receive No Buffers"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", - CTLFLAG_RD, &stats->ruc, - "Receive Undersize"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", - CTLFLAG_RD, &stats->rfc, - "Fragmented Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", - CTLFLAG_RD, &stats->roc, - "Oversized Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", - CTLFLAG_RD, &stats->rjc, - "Recevied Jabber"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", - CTLFLAG_RD, &stats->rxerrc, - "Receive Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", - CTLFLAG_RD, &stats->crcerrs, - "CRC errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", - CTLFLAG_RD, &stats->algnerrc, - "Alignment Errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_no_crs", - CTLFLAG_RD, &stats->tncrs, - "Transmit with No CRS"); - /* On 82575 these are collision counts */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", - CTLFLAG_RD, &stats->cexterr, - "Collision/Carrier extension errors"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", - CTLFLAG_RD, &stats->xonrxc, - "XON Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", - CTLFLAG_RD, &stats->xontxc, - "XON Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", - CTLFLAG_RD, &stats->xoffrxc, - "XOFF Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", - CTLFLAG_RD, &stats->xofftxc, - "XOFF Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "unsupported_fc_recvd", - CTLFLAG_RD, &stats->fcruc, - "Unsupported Flow Control Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_recvd", - CTLFLAG_RD, &stats->mgprc, - "Management Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_drop", - CTLFLAG_RD, &stats->mgpdc, - "Management Packets Dropped"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mgmt_pkts_txd", - CTLFLAG_RD, &stats->mgptc, - "Management Packets Transmitted"); - /* Packet Reception Stats */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", - CTLFLAG_RD, &stats->tpr, - "Total Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &stats->gprc, - "Good Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", - CTLFLAG_RD, &stats->bprc, - "Broadcast Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &stats->mprc, - "Multicast Packets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", - CTLFLAG_RD, &stats->prc64, - "64 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", - CTLFLAG_RD, &stats->prc127, - "65-127 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", - CTLFLAG_RD, &stats->prc255, - "128-255 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", - CTLFLAG_RD, &stats->prc511, - "256-511 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", - CTLFLAG_RD, &stats->prc1023, - "512-1023 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", - CTLFLAG_RD, &stats->prc1522, - "1023-1522 byte frames received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &stats->gorc, - "Good Octets Received"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_recvd", - CTLFLAG_RD, &stats->tor, - "Total Octets Received"); - - /* Packet Transmission Stats */ - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &stats->gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_octets_txd", - CTLFLAG_RD, &stats->tot, - "Total Octets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", - CTLFLAG_RD, &stats->tpt, - "Total Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &stats->gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", - CTLFLAG_RD, &stats->bptc, - "Broadcast Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", - CTLFLAG_RD, &stats->mptc, - "Multicast Packets Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", - CTLFLAG_RD, &stats->ptc64, - "64 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", - CTLFLAG_RD, &stats->ptc127, - "65-127 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", - CTLFLAG_RD, &stats->ptc255, - "128-255 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", - CTLFLAG_RD, &stats->ptc511, - "256-511 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", - CTLFLAG_RD, &stats->ptc1023, - "512-1023 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", - CTLFLAG_RD, &stats->ptc1522, - "1024-1522 byte frames transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", - CTLFLAG_RD, &stats->tsctc, - "TSO Contexts Transmitted"); - SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", - CTLFLAG_RD, &stats->tsctfc, - "TSO Contexts Failed"); - - - /* Interrupt Stats */ - - int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", - CTLFLAG_RD, NULL, "Interrupt Statistics"); - int_list = SYSCTL_CHILDREN(int_node); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", - CTLFLAG_RD, &stats->iac, - "Interrupt Assertion Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", - CTLFLAG_RD, &stats->icrxptc, - "Interrupt Cause Rx Pkt Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", - CTLFLAG_RD, &stats->icrxatc, - "Interrupt Cause Rx Abs Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", - CTLFLAG_RD, &stats->ictxptc, - "Interrupt Cause Tx Pkt Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", - CTLFLAG_RD, &stats->ictxatc, - "Interrupt Cause Tx Abs Timer Expire Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", - CTLFLAG_RD, &stats->ictxqec, - "Interrupt Cause Tx Queue Empty Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", - CTLFLAG_RD, &stats->ictxqmtc, - "Interrupt Cause Tx Queue Min Thresh Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", - CTLFLAG_RD, &stats->icrxdmtc, - "Interrupt Cause Rx Desc Min Thresh Count"); - - SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", - CTLFLAG_RD, &stats->icrxoc, - "Interrupt Cause Receiver Overrun Count"); - - /* Host to Card Stats */ - - host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", - CTLFLAG_RD, NULL, - "Host to Card Statistics"); - - host_list = SYSCTL_CHILDREN(host_node); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt", - CTLFLAG_RD, &stats->cbtmpc, - "Circuit Breaker Tx Packet Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard", - CTLFLAG_RD, &stats->htdpmc, - "Host Transmit Discarded Packets"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt", - CTLFLAG_RD, &stats->rpthc, - "Rx Packets To Host"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts", - CTLFLAG_RD, &stats->cbrmpc, - "Circuit Breaker Rx Packet Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop", - CTLFLAG_RD, &stats->cbrdpc, - "Circuit Breaker Rx Dropped Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt", - CTLFLAG_RD, &stats->hgptc, - "Host Good Packets Tx Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop", - CTLFLAG_RD, &stats->htcbdpc, - "Host Tx Circuit Breaker Dropped Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes", - CTLFLAG_RD, &stats->hgorc, - "Host Good Octets Received Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes", - CTLFLAG_RD, &stats->hgotc, - "Host Good Octets Transmit Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors", - CTLFLAG_RD, &stats->lenerrs, - "Length Errors"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt", - CTLFLAG_RD, &stats->scvpc, - "SerDes/SGMII Code Violation Pkt Count"); - - SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed", - CTLFLAG_RD, &stats->hrmpc, - "Header Redirection Missed Packet Count"); -} - - -/********************************************************************** - * - * This routine provides a way to dump out the adapter eeprom, - * often a useful debug/service tool. This only dumps the first - * 32 words, stuff that matters is in that extent. - * - **********************************************************************/ -static int -igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - int error; - int result; - - result = -1; - error = sysctl_handle_int(oidp, &result, 0, req); - - if (error || !req->newptr) - return (error); - - /* - * This value will cause a hex dump of the - * first 32 16-bit words of the EEPROM to - * the screen. - */ - if (result == 1) { - adapter = (struct adapter *)arg1; - igb_print_nvm_info(adapter); - } - - return (error); -} - -static void -igb_print_nvm_info(struct adapter *adapter) -{ - u16 eeprom_data; - int i, j, row = 0; - - /* Its a bit crude, but it gets the job done */ - printf("\nInterface EEPROM Dump:\n"); - printf("Offset\n0x0000 "); - for (i = 0, j = 0; i < 32; i++, j++) { - if (j == 8) { /* Make the offset block */ - j = 0; ++row; - printf("\n0x00%x0 ",row); - } - e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); - printf("%04x ", eeprom_data); - } - printf("\n"); -} - -static void -igb_set_sysctl_value(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - -/* -** Set flow control using sysctl: -** Flow control values: -** 0 - off -** 1 - rx pause -** 2 - tx pause -** 3 - full -*/ -static int -igb_set_flowcntl(SYSCTL_HANDLER_ARGS) -{ - int error; - static int input = 3; /* default is full */ - struct adapter *adapter = (struct adapter *) arg1; - - error = sysctl_handle_int(oidp, &input, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - - switch (input) { - case e1000_fc_rx_pause: - case e1000_fc_tx_pause: - case e1000_fc_full: - case e1000_fc_none: - adapter->hw.fc.requested_mode = input; - adapter->fc = input; - break; - default: - /* Do nothing */ - return (error); - } - - adapter->hw.fc.current_mode = adapter->hw.fc.requested_mode; - e1000_force_mac_fc(&adapter->hw); - /* XXX TODO: update DROP_EN on each RX queue if appropriate */ - return (error); -} - -/* -** Manage DMA Coalesce: -** Control values: -** 0/1 - off/on -** Legal timer values are: -** 250,500,1000-10000 in thousands -*/ -static int -igb_sysctl_dmac(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter = (struct adapter *) arg1; - int error; - - error = sysctl_handle_int(oidp, &adapter->dmac, 0, req); - - if ((error) || (req->newptr == NULL)) - return (error); - - switch (adapter->dmac) { - case 0: - /* Disabling */ - break; - case 1: /* Just enable and use default */ - adapter->dmac = 1000; - break; - case 250: - case 500: - case 1000: - case 2000: - case 3000: - case 4000: - case 5000: - case 6000: - case 7000: - case 8000: - case 9000: - case 10000: - /* Legal values - allow */ - break; - default: - /* Do nothing, illegal value */ - adapter->dmac = 0; - return (EINVAL); - } - /* Reinit the interface */ - igb_init(adapter); - return (error); -} - -/* -** Manage Energy Efficient Ethernet: -** Control values: -** 0/1 - enabled/disabled -*/ -static int -igb_sysctl_eee(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter = (struct adapter *) arg1; - int error, value; - - value = adapter->hw.dev_spec._82575.eee_disable; - error = sysctl_handle_int(oidp, &value, 0, req); - if (error || req->newptr == NULL) - return (error); - IGB_CORE_LOCK(adapter); - adapter->hw.dev_spec._82575.eee_disable = (value != 0); - igb_init_locked(adapter); - IGB_CORE_UNLOCK(adapter); - return (0); -} diff --git a/sys/dev/e1000/if_igb.h b/sys/dev/e1000/if_igb.h deleted file mode 100644 index e2f2219cc94c..000000000000 --- a/sys/dev/e1000/if_igb.h +++ /dev/null @@ -1,634 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -#ifndef _IF_IGB_H_ -#define _IF_IGB_H_ - -#ifdef ALTQ -#define IGB_LEGACY_TX -#endif - -#include -#include -#ifndef IGB_LEGACY_TX -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#ifdef RSS -#include -#include -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "e1000_api.h" -#include "e1000_82575.h" - -/* Tunables */ -/* - * IGB_TXD: Maximum number of Transmit Descriptors - * - * This value is the number of transmit descriptors allocated by the driver. - * Increasing this value allows the driver to queue more transmits. Each - * descriptor is 16 bytes. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define IGB_MIN_TXD 256 -#define IGB_DEFAULT_TXD 1024 -#define IGB_MAX_TXD 4096 - -/* - * IGB_RXD: Maximum number of Receive Descriptors - * - * This value is the number of receive descriptors allocated by the driver. - * Increasing this value allows the driver to buffer more incoming packets. - * Each descriptor is 16 bytes. A receive buffer is also allocated for each - * descriptor. The maximum MTU size is 16110. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define IGB_MIN_RXD 256 -#define IGB_DEFAULT_RXD 1024 -#define IGB_MAX_RXD 4096 - -/* - * IGB_TIDV - Transmit Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value delays the generation of transmit interrupts in units of - * 1.024 microseconds. Transmit interrupt reduction can improve CPU - * efficiency if properly tuned for specific network traffic. If the - * system is reporting dropped transmits, this value may be set too high - * causing the driver to run out of available transmit descriptors. - */ -#define IGB_TIDV 64 - -/* - * IGB_TADV - Transmit Absolute Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * transmit interrupt is generated. Useful only if IGB_TIDV is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is sent on the wire within the set amount of time. Proper tuning, - * along with IGB_TIDV, may improve traffic throughput in specific - * network conditions. - */ -#define IGB_TADV 64 - -/* - * IGB_RDTR - Receive Interrupt Delay Timer (Packet Timer) - * Valid Range: 0-65535 (0=off) - * Default Value: 0 - * This value delays the generation of receive interrupts in units of 1.024 - * microseconds. Receive interrupt reduction can improve CPU efficiency if - * properly tuned for specific network traffic. Increasing this value adds - * extra latency to frame reception and can end up decreasing the throughput - * of TCP traffic. If the system is reporting dropped receives, this value - * may be set too high, causing the driver to run out of available receive - * descriptors. - * - * CAUTION: When setting IGB_RDTR to a value other than 0, adapters - * may hang (stop transmitting) under certain network conditions. - * If this occurs a WATCHDOG message is logged in the system - * event log. In addition, the controller is automatically reset, - * restoring the network connection. To eliminate the potential - * for the hang ensure that IGB_RDTR is set to 0. - */ -#define IGB_RDTR 0 - -/* - * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * receive interrupt is generated. Useful only if IGB_RDTR is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is received within the set amount of time. Proper tuning, - * along with IGB_RDTR, may improve traffic throughput in specific network - * conditions. - */ -#define IGB_RADV 64 - -/* - * This parameter controls the duration of transmit watchdog timer. - */ -#define IGB_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. Cleaning earlier seems a win. - */ -#define IGB_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 2) - -/* - * This parameter controls whether or not autonegotation is enabled. - * 0 - Disable autonegotiation - * 1 - Enable autonegotiation - */ -#define DO_AUTO_NEG 1 - -/* - * This parameter control whether or not the driver will wait for - * autonegotiation to complete. - * 1 - Wait for autonegotiation to complete - * 0 - Don't wait for autonegotiation to complete - */ -#define WAIT_FOR_AUTO_NEG_DEFAULT 0 - -/* Tunables -- End */ - -#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ - ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ - ADVERTISE_1000_FULL) - -#define AUTO_ALL_MODES 0 - -/* PHY master/slave setting */ -#define IGB_MASTER_SLAVE e1000_ms_hw_default - -/* Support AutoMediaDetect for Marvell M88 PHY in i354 */ -#define IGB_MEDIA_RESET (1 << 0) - -/* - * Micellaneous constants - */ -#define IGB_INTEL_VENDOR_ID 0x8086 - -#define IGB_JUMBO_PBA 0x00000028 -#define IGB_DEFAULT_PBA 0x00000030 -#define IGB_SMARTSPEED_DOWNSHIFT 3 -#define IGB_SMARTSPEED_MAX 15 -#define IGB_MAX_LOOP 10 - -#define IGB_RX_PTHRESH ((hw->mac.type == e1000_i354) ? 12 : \ - ((hw->mac.type <= e1000_82576) ? 16 : 8)) -#define IGB_RX_HTHRESH 8 -#define IGB_RX_WTHRESH ((hw->mac.type == e1000_82576 && \ - adapter->msix_mem) ? 1 : 4) - -#define IGB_TX_PTHRESH ((hw->mac.type == e1000_i354) ? 20 : 8) -#define IGB_TX_HTHRESH 1 -#define IGB_TX_WTHRESH ((hw->mac.type != e1000_82575 && \ - adapter->msix_mem) ? 1 : 16) - -#define MAX_NUM_MULTICAST_ADDRESSES 128 -#define PCI_ANY_ID (~0U) -#define ETHER_ALIGN 2 -#define IGB_TX_BUFFER_SIZE ((uint32_t) 1514) -#define IGB_FC_PAUSE_TIME 0x0680 -#define IGB_EEPROM_APME 0x400; -/* Queue minimum free for use */ -#define IGB_QUEUE_THRESHOLD (adapter->num_tx_desc / 8) - -/* - * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be - * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will - * also optimize cache line size effect. H/W supports up to cache line size 128. - */ -#define IGB_DBA_ALIGN 128 - -#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */ - -/* PCI Config defines */ -#define IGB_MSIX_BAR 3 - -/* Defines for printing debug information */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") -#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) -#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) -#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") -#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) -#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) -#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") -#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) -#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) - -#define IGB_MAX_SCATTER 40 -#define IGB_VFTA_SIZE 128 -#define IGB_BR_SIZE 4096 /* ring buf size */ -#define IGB_TSO_SIZE (65535 + sizeof(struct ether_vlan_header)) -#define IGB_TSO_SEG_SIZE 4096 /* Max dma segment size */ -#define IGB_TXPBSIZE 20408 -#define IGB_HDR_BUF 128 -#define IGB_PKTTYPE_MASK 0x0000FFF0 -#define IGB_DMCTLX_DCFLUSH_DIS 0x80000000 /* Disable DMA Coalesce Flush */ -#define ETH_ZLEN 60 -#define ETH_ADDR_LEN 6 - -/* Offload bits in mbuf flag */ -#if __FreeBSD_version >= 1000000 -#define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_IP_TCP|CSUM_IP_UDP|CSUM_IP_SCTP) -#define CSUM_OFFLOAD_IPV6 (CSUM_IP6_TCP|CSUM_IP6_UDP|CSUM_IP6_SCTP) -#define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6) -#elif __FreeBSD_version >= 800000 -#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) -#else -#define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) -#endif - -/* Define the starting Interrupt rate per Queue */ -#define IGB_INTS_PER_SEC 8000 -#define IGB_DEFAULT_ITR ((1000000/IGB_INTS_PER_SEC) << 2) - -#define IGB_LINK_ITR 2000 -#define I210_LINK_DELAY 1000 - -/* Precision Time Sync (IEEE 1588) defines */ -#define ETHERTYPE_IEEE1588 0x88F7 -#define PICOSECS_PER_TICK 20833 -#define TSYNC_PORT 319 /* UDP port for the protocol */ - -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct igb_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - - -/* -** Driver queue struct: this is the interrupt container -** for the associated tx and rx ring. -*/ -struct igb_queue { - struct adapter *adapter; - u32 msix; /* This queue's MSIX vector */ - u32 eims; /* This queue's EIMS bit */ - u32 eitr_setting; - struct resource *res; - void *tag; - struct tx_ring *txr; - struct rx_ring *rxr; - struct task que_task; - struct taskqueue *tq; - u64 irqs; -}; - -/* - * The transmit ring, one per queue - */ -struct tx_ring { - struct adapter *adapter; - struct mtx tx_mtx; - u32 me; - int watchdog_time; - union e1000_adv_tx_desc *tx_base; - struct igb_tx_buf *tx_buffers; - struct igb_dma_alloc txdma; - volatile u16 tx_avail; - u16 next_avail_desc; - u16 next_to_clean; - u16 num_desc; - enum { - IGB_QUEUE_IDLE = 1, - IGB_QUEUE_WORKING = 2, - IGB_QUEUE_HUNG = 4, - IGB_QUEUE_DEPLETED = 8, - } queue_status; - u32 txd_cmd; - bus_dma_tag_t txtag; - char mtx_name[16]; -#ifndef IGB_LEGACY_TX - struct buf_ring *br; - struct task txq_task; -#endif - u32 bytes; /* used for AIM */ - u32 packets; - /* Soft Stats */ - unsigned long tso_tx; - unsigned long no_tx_map_avail; - unsigned long no_tx_dma_setup; - u64 no_desc_avail; - u64 total_packets; -}; - -/* - * Receive ring: one per queue - */ -struct rx_ring { - struct adapter *adapter; - u32 me; - struct igb_dma_alloc rxdma; - union e1000_adv_rx_desc *rx_base; - struct lro_ctrl lro; - bool lro_enabled; - bool hdr_split; - struct mtx rx_mtx; - char mtx_name[16]; - u32 next_to_refresh; - u32 next_to_check; - struct igb_rx_buf *rx_buffers; - bus_dma_tag_t htag; /* dma tag for rx head */ - bus_dma_tag_t ptag; /* dma tag for rx packet */ - /* - * First/last mbuf pointers, for - * collecting multisegment RX packets. - */ - struct mbuf *fmp; - struct mbuf *lmp; - - u32 bytes; - u32 packets; - int rdt; - int rdh; - - /* Soft stats */ - u64 rx_split_packets; - u64 rx_discarded; - u64 rx_packets; - u64 rx_bytes; -}; - -struct adapter { - struct ifnet *ifp; - struct e1000_hw hw; - - struct e1000_osdep osdep; - device_t dev; - struct cdev *led_dev; - - struct resource *pci_mem; - struct resource *msix_mem; - int memrid; - - /* - * Interrupt resources: this set is - * either used for legacy, or for Link - * when doing MSIX - */ - void *tag; - struct resource *res; - - struct ifmedia media; - struct callout timer; - int msix; - int if_flags; - int pause_frames; - - struct mtx core_mtx; - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - - u16 num_vlans; - u16 num_queues; - - /* - ** Shadow VFTA table, this is needed because - ** the real vlan filter table gets cleared during - ** a soft reset and the driver needs to be able - ** to repopulate it. - */ - u32 shadow_vfta[IGB_VFTA_SIZE]; - - /* Info about the interface */ - u32 optics; - u32 fc; /* local flow ctrl setting */ - int advertise; /* link speeds */ - bool link_active; - u16 max_frame_size; - u16 num_segs; - u16 link_speed; - bool link_up; - u32 linkvec; - u16 link_duplex; - u32 dmac; - int link_mask; - - /* Flags */ - u32 flags; - - /* Mbuf cluster size */ - u32 rx_mbuf_sz; - - /* Support for pluggable optics */ - bool sfp_probe; - struct task link_task; /* Link tasklet */ - struct task mod_task; /* SFP tasklet */ - struct task msf_task; /* Multispeed Fiber */ - struct taskqueue *tq; - - /* - ** Queues: - ** This is the irq holder, it has - ** and RX/TX pair or rings associated - ** with it. - */ - struct igb_queue *queues; - - /* - * Transmit rings: - * Allocated at run time, an array of rings. - */ - struct tx_ring *tx_rings; - u32 num_tx_desc; - - /* - * Receive rings: - * Allocated at run time, an array of rings. - */ - struct rx_ring *rx_rings; - u64 que_mask; - u32 num_rx_desc; - - /* Multicast array memory */ - u8 *mta; - - /* Misc stats maintained by the driver */ - unsigned long device_control; - unsigned long dropped_pkts; - unsigned long eint_mask; - unsigned long int_mask; - unsigned long link_irq; - unsigned long mbuf_defrag_failed; - unsigned long no_tx_dma_setup; - unsigned long packet_buf_alloc_rx; - unsigned long packet_buf_alloc_tx; - unsigned long rx_control; - unsigned long rx_overruns; - unsigned long watchdog_events; - - /* Used in pf and vf */ - void *stats; - - int enable_aim; - int has_manage; - int wol; - int rx_process_limit; - int tx_process_limit; - u16 vf_ifp; /* a VF interface */ - bool in_detach; /* Used only in igb_ioctl */ - -}; - -/* ****************************************************************************** - * vendor_info_array - * - * This array contains the list of Subvendor/Subdevice IDs on which the driver - * should load. - * - * ******************************************************************************/ -typedef struct _igb_vendor_info_t { - unsigned int vendor_id; - unsigned int device_id; - unsigned int subvendor_id; - unsigned int subdevice_id; - unsigned int index; -} igb_vendor_info_t; - -struct igb_tx_buf { - union e1000_adv_tx_desc *eop; - struct mbuf *m_head; - bus_dmamap_t map; -}; - -struct igb_rx_buf { - struct mbuf *m_head; - struct mbuf *m_pack; - bus_dmamap_t hmap; /* bus_dma map for header */ - bus_dmamap_t pmap; /* bus_dma map for packet */ -}; - -/* -** Find the number of unrefreshed RX descriptors -*/ -static inline u16 -igb_rx_unrefreshed(struct rx_ring *rxr) -{ - struct adapter *adapter = rxr->adapter; - - if (rxr->next_to_check > rxr->next_to_refresh) - return (rxr->next_to_check - rxr->next_to_refresh - 1); - else - return ((adapter->num_rx_desc + rxr->next_to_check) - - rxr->next_to_refresh - 1); -} - -#define IGB_CORE_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->core_mtx, _name, "IGB Core Lock", MTX_DEF) -#define IGB_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx) -#define IGB_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) -#define IGB_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) -#define IGB_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED) - -#define IGB_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx) -#define IGB_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) -#define IGB_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) -#define IGB_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) -#define IGB_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) - -#define IGB_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) -#define IGB_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) -#define IGB_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx) -#define IGB_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_mtx, MA_OWNED) - -#define UPDATE_VF_REG(reg, last, cur) \ -{ \ - u32 new = E1000_READ_REG(hw, reg); \ - if (new < last) \ - cur += 0x100000000LL; \ - last = new; \ - cur &= 0xFFFFFFFF00000000LL; \ - cur |= new; \ -} - -#if __FreeBSD_version >= 800000 && __FreeBSD_version < 800504 -static __inline int -drbr_needs_enqueue(struct ifnet *ifp, struct buf_ring *br) -{ -#ifdef ALTQ - if (ALTQ_IS_ENABLED(&ifp->if_snd)) - return (1); -#endif - return (!buf_ring_empty(br)); -} -#endif - -#endif /* _IF_IGB_H_ */ - - diff --git a/sys/dev/e1000/if_lem.c b/sys/dev/e1000/if_lem.c deleted file mode 100644 index 537ccbe5ff11..000000000000 --- a/sys/dev/e1000/if_lem.c +++ /dev/null @@ -1,4730 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - -/* - * Uncomment the following extensions for better performance in a VM, - * especially if you have support in the hypervisor. - * See http://info.iet.unipi.it/~luigi/netmap/ - */ -// #define BATCH_DISPATCH -// #define NIC_SEND_COMBINING - -#include "opt_inet.h" -#include "opt_inet6.h" - -#ifdef HAVE_KERNEL_OPTION_HEADERS -#include "opt_device_polling.h" -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "e1000_api.h" -#include "if_lem.h" - -/********************************************************************* - * Legacy Em Driver version: - *********************************************************************/ -char lem_driver_version[] = "1.1.0"; - -/********************************************************************* - * PCI Device ID Table - * - * Used by probe to select devices to load on - * Last field stores an index into e1000_strings - * Last entry must be all 0s - * - * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } - *********************************************************************/ - -static em_vendor_info_t lem_vendor_info_array[] = -{ - /* Intel(R) PRO/1000 Network Connection */ - { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, - PCI_ANY_ID, PCI_ANY_ID, 0}, - - { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, - { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, - /* required last entry */ - { 0, 0, 0, 0, 0} -}; - -/********************************************************************* - * Table of branding strings for all supported NICs. - *********************************************************************/ - -static char *lem_strings[] = { - "Intel(R) PRO/1000 Legacy Network Connection" -}; - -/********************************************************************* - * Function prototypes - *********************************************************************/ -static int lem_probe(device_t); -static int lem_attach(device_t); -static int lem_detach(device_t); -static int lem_shutdown(device_t); -static int lem_suspend(device_t); -static int lem_resume(device_t); -static void lem_start(if_t); -static void lem_start_locked(if_t ifp); -static int lem_ioctl(if_t, u_long, caddr_t); -static uint64_t lem_get_counter(if_t, ift_counter); -static void lem_init(void *); -static void lem_init_locked(struct adapter *); -static void lem_stop(void *); -static void lem_media_status(if_t, struct ifmediareq *); -static int lem_media_change(if_t); -static void lem_identify_hardware(struct adapter *); -static int lem_allocate_pci_resources(struct adapter *); -static int lem_allocate_irq(struct adapter *adapter); -static void lem_free_pci_resources(struct adapter *); -static void lem_local_timer(void *); -static int lem_hardware_init(struct adapter *); -static int lem_setup_interface(device_t, struct adapter *); -static void lem_setup_transmit_structures(struct adapter *); -static void lem_initialize_transmit_unit(struct adapter *); -static int lem_setup_receive_structures(struct adapter *); -static void lem_initialize_receive_unit(struct adapter *); -static void lem_enable_intr(struct adapter *); -static void lem_disable_intr(struct adapter *); -static void lem_free_transmit_structures(struct adapter *); -static void lem_free_receive_structures(struct adapter *); -static void lem_update_stats_counters(struct adapter *); -static void lem_add_hw_stats(struct adapter *adapter); -static void lem_txeof(struct adapter *); -static void lem_tx_purge(struct adapter *); -static int lem_allocate_receive_structures(struct adapter *); -static int lem_allocate_transmit_structures(struct adapter *); -static bool lem_rxeof(struct adapter *, int, int *); -#ifndef __NO_STRICT_ALIGNMENT -static int lem_fixup_rx(struct adapter *); -#endif -static void lem_receive_checksum(struct adapter *, struct e1000_rx_desc *, - struct mbuf *); -static void lem_transmit_checksum_setup(struct adapter *, struct mbuf *, - u32 *, u32 *); -static void lem_set_promisc(struct adapter *); -static void lem_disable_promisc(struct adapter *); -static void lem_set_multi(struct adapter *); -static void lem_update_link_status(struct adapter *); -static int lem_get_buf(struct adapter *, int); -static void lem_register_vlan(void *, if_t, u16); -static void lem_unregister_vlan(void *, if_t, u16); -static void lem_setup_vlan_hw_support(struct adapter *); -static int lem_xmit(struct adapter *, struct mbuf **); -static void lem_smartspeed(struct adapter *); -static int lem_82547_fifo_workaround(struct adapter *, int); -static void lem_82547_update_fifo_head(struct adapter *, int); -static int lem_82547_tx_fifo_reset(struct adapter *); -static void lem_82547_move_tail(void *); -static int lem_dma_malloc(struct adapter *, bus_size_t, - struct em_dma_alloc *, int); -static void lem_dma_free(struct adapter *, struct em_dma_alloc *); -static int lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); -static void lem_print_nvm_info(struct adapter *); -static int lem_is_valid_ether_addr(u8 *); -static u32 lem_fill_descriptors (bus_addr_t address, u32 length, - PDESC_ARRAY desc_array); -static int lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS); -static void lem_add_int_delay_sysctl(struct adapter *, const char *, - const char *, struct em_int_delay_info *, int, int); -static void lem_set_flow_cntrl(struct adapter *, const char *, - const char *, int *, int); -/* Management and WOL Support */ -static void lem_init_manageability(struct adapter *); -static void lem_release_manageability(struct adapter *); -static void lem_get_hw_control(struct adapter *); -static void lem_release_hw_control(struct adapter *); -static void lem_get_wakeup(device_t); -static void lem_enable_wakeup(device_t); -static int lem_enable_phy_wakeup(struct adapter *); -static void lem_led_func(void *, int); - -static void lem_intr(void *); -static int lem_irq_fast(void *); -static void lem_handle_rxtx(void *context, int pending); -static void lem_handle_link(void *context, int pending); -static void lem_add_rx_process_limit(struct adapter *, const char *, - const char *, int *, int); - -#ifdef DEVICE_POLLING -static poll_handler_t lem_poll; -#endif /* POLLING */ - -/********************************************************************* - * FreeBSD Device Interface Entry Points - *********************************************************************/ - -static device_method_t lem_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, lem_probe), - DEVMETHOD(device_attach, lem_attach), - DEVMETHOD(device_detach, lem_detach), - DEVMETHOD(device_shutdown, lem_shutdown), - DEVMETHOD(device_suspend, lem_suspend), - DEVMETHOD(device_resume, lem_resume), - DEVMETHOD_END -}; - -static driver_t lem_driver = { - "em", lem_methods, sizeof(struct adapter), -}; - -extern devclass_t em_devclass; -DRIVER_MODULE(lem, pci, lem_driver, em_devclass, 0, 0); -MODULE_DEPEND(lem, pci, 1, 1, 1); -MODULE_DEPEND(lem, ether, 1, 1, 1); -#ifdef DEV_NETMAP -MODULE_DEPEND(lem, netmap, 1, 1, 1); -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Tunable default values. - *********************************************************************/ - -#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) -#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) - -#define MAX_INTS_PER_SEC 8000 -#define DEFAULT_ITR (1000000000/(MAX_INTS_PER_SEC * 256)) - -static int lem_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); -static int lem_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); -static int lem_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); -static int lem_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); -/* - * increase lem_rxd and lem_txd to at least 2048 in netmap mode - * for better performance. - */ -static int lem_rxd = EM_DEFAULT_RXD; -static int lem_txd = EM_DEFAULT_TXD; -static int lem_smart_pwr_down = FALSE; - -/* Controls whether promiscuous also shows bad packets */ -static int lem_debug_sbp = FALSE; - -TUNABLE_INT("hw.em.tx_int_delay", &lem_tx_int_delay_dflt); -TUNABLE_INT("hw.em.rx_int_delay", &lem_rx_int_delay_dflt); -TUNABLE_INT("hw.em.tx_abs_int_delay", &lem_tx_abs_int_delay_dflt); -TUNABLE_INT("hw.em.rx_abs_int_delay", &lem_rx_abs_int_delay_dflt); -TUNABLE_INT("hw.em.rxd", &lem_rxd); -TUNABLE_INT("hw.em.txd", &lem_txd); -TUNABLE_INT("hw.em.smart_pwr_down", &lem_smart_pwr_down); -TUNABLE_INT("hw.em.sbp", &lem_debug_sbp); - -/* Interrupt style - default to fast */ -static int lem_use_legacy_irq = 0; -TUNABLE_INT("hw.em.use_legacy_irq", &lem_use_legacy_irq); - -/* How many packets rxeof tries to clean at a time */ -static int lem_rx_process_limit = 100; -TUNABLE_INT("hw.em.rx_process_limit", &lem_rx_process_limit); - -/* Flow control setting - default to FULL */ -static int lem_fc_setting = e1000_fc_full; -TUNABLE_INT("hw.em.fc_setting", &lem_fc_setting); - -/* Global used in WOL setup with multiport cards */ -static int global_quad_port_a = 0; - -#ifdef DEV_NETMAP /* see ixgbe.c for details */ -#include -#endif /* DEV_NETMAP */ - -/********************************************************************* - * Device identification routine - * - * em_probe determines if the driver should be loaded on - * adapter based on PCI vendor/device id of the adapter. - * - * return BUS_PROBE_DEFAULT on success, positive on failure - *********************************************************************/ - -static int -lem_probe(device_t dev) -{ - char adapter_name[60]; - u16 pci_vendor_id = 0; - u16 pci_device_id = 0; - u16 pci_subvendor_id = 0; - u16 pci_subdevice_id = 0; - em_vendor_info_t *ent; - - INIT_DEBUGOUT("em_probe: begin"); - - pci_vendor_id = pci_get_vendor(dev); - if (pci_vendor_id != EM_VENDOR_ID) - return (ENXIO); - - pci_device_id = pci_get_device(dev); - pci_subvendor_id = pci_get_subvendor(dev); - pci_subdevice_id = pci_get_subdevice(dev); - - ent = lem_vendor_info_array; - while (ent->vendor_id != 0) { - if ((pci_vendor_id == ent->vendor_id) && - (pci_device_id == ent->device_id) && - - ((pci_subvendor_id == ent->subvendor_id) || - (ent->subvendor_id == PCI_ANY_ID)) && - - ((pci_subdevice_id == ent->subdevice_id) || - (ent->subdevice_id == PCI_ANY_ID))) { - sprintf(adapter_name, "%s %s", - lem_strings[ent->index], - lem_driver_version); - device_set_desc_copy(dev, adapter_name); - return (BUS_PROBE_DEFAULT); - } - ent++; - } - - return (ENXIO); -} - -/********************************************************************* - * Device initialization routine - * - * The attach entry point is called when the driver is being loaded. - * This routine identifies the type of hardware, allocates all resources - * and initializes the hardware. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -lem_attach(device_t dev) -{ - struct adapter *adapter; - int tsize, rsize; - int error = 0; - - INIT_DEBUGOUT("lem_attach: begin"); - - adapter = device_get_softc(dev); - adapter->dev = adapter->osdep.dev = dev; - EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); - EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); - EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); - - /* SYSCTL stuff */ - SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, - lem_sysctl_nvm_info, "I", "NVM Information"); - - callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); - callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); - - /* Determine hardware and mac info */ - lem_identify_hardware(adapter); - - /* Setup PCI resources */ - if (lem_allocate_pci_resources(adapter)) { - device_printf(dev, "Allocation of PCI resources failed\n"); - error = ENXIO; - goto err_pci; - } - - /* Do Shared Code initialization */ - if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { - device_printf(dev, "Setup of Shared code failed\n"); - error = ENXIO; - goto err_pci; - } - - e1000_get_bus_info(&adapter->hw); - - /* Set up some sysctls for the tunable interrupt delays */ - lem_add_int_delay_sysctl(adapter, "rx_int_delay", - "receive interrupt delay in usecs", &adapter->rx_int_delay, - E1000_REGISTER(&adapter->hw, E1000_RDTR), lem_rx_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "tx_int_delay", - "transmit interrupt delay in usecs", &adapter->tx_int_delay, - E1000_REGISTER(&adapter->hw, E1000_TIDV), lem_tx_int_delay_dflt); - if (adapter->hw.mac.type >= e1000_82540) { - lem_add_int_delay_sysctl(adapter, "rx_abs_int_delay", - "receive interrupt delay limit in usecs", - &adapter->rx_abs_int_delay, - E1000_REGISTER(&adapter->hw, E1000_RADV), - lem_rx_abs_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "tx_abs_int_delay", - "transmit interrupt delay limit in usecs", - &adapter->tx_abs_int_delay, - E1000_REGISTER(&adapter->hw, E1000_TADV), - lem_tx_abs_int_delay_dflt); - lem_add_int_delay_sysctl(adapter, "itr", - "interrupt delay limit in usecs/4", - &adapter->tx_itr, - E1000_REGISTER(&adapter->hw, E1000_ITR), - DEFAULT_ITR); - } - - /* Sysctls for limiting the amount of work done in the taskqueue */ - lem_add_rx_process_limit(adapter, "rx_processing_limit", - "max number of rx packets to process", &adapter->rx_process_limit, - lem_rx_process_limit); - -#ifdef NIC_SEND_COMBINING - /* Sysctls to control mitigation */ - lem_add_rx_process_limit(adapter, "sc_enable", - "driver TDT mitigation", &adapter->sc_enable, 0); -#endif /* NIC_SEND_COMBINING */ -#ifdef BATCH_DISPATCH - lem_add_rx_process_limit(adapter, "batch_enable", - "driver rx batch", &adapter->batch_enable, 0); -#endif /* BATCH_DISPATCH */ - - /* Sysctl for setting the interface flow control */ - lem_set_flow_cntrl(adapter, "flow_control", - "flow control setting", - &adapter->fc_setting, lem_fc_setting); - - /* - * Validate number of transmit and receive descriptors. It - * must not exceed hardware maximum, and must be multiple - * of E1000_DBA_ALIGN. - */ - if (((lem_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || - (adapter->hw.mac.type >= e1000_82544 && lem_txd > EM_MAX_TXD) || - (adapter->hw.mac.type < e1000_82544 && lem_txd > EM_MAX_TXD_82543) || - (lem_txd < EM_MIN_TXD)) { - device_printf(dev, "Using %d TX descriptors instead of %d!\n", - EM_DEFAULT_TXD, lem_txd); - adapter->num_tx_desc = EM_DEFAULT_TXD; - } else - adapter->num_tx_desc = lem_txd; - if (((lem_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || - (adapter->hw.mac.type >= e1000_82544 && lem_rxd > EM_MAX_RXD) || - (adapter->hw.mac.type < e1000_82544 && lem_rxd > EM_MAX_RXD_82543) || - (lem_rxd < EM_MIN_RXD)) { - device_printf(dev, "Using %d RX descriptors instead of %d!\n", - EM_DEFAULT_RXD, lem_rxd); - adapter->num_rx_desc = EM_DEFAULT_RXD; - } else - adapter->num_rx_desc = lem_rxd; - - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_wait_to_complete = FALSE; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - adapter->rx_buffer_len = 2048; - - e1000_init_script_state_82541(&adapter->hw, TRUE); - e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); - - /* Copper options */ - if (adapter->hw.phy.media_type == e1000_media_type_copper) { - adapter->hw.phy.mdix = AUTO_ALL_MODES; - adapter->hw.phy.disable_polarity_correction = FALSE; - adapter->hw.phy.ms_type = EM_MASTER_SLAVE; - } - - /* - * Set the frame limits assuming - * standard ethernet sized frames. - */ - adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; - adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; - - /* - * This controls when hardware reports transmit completion - * status. - */ - adapter->hw.mac.report_tx_early = 1; - - /* - * It seems that the descriptor DMA engine on some PCI cards - * fetches memory past the end of the last descriptor in the - * ring. These reads are problematic when VT-d (DMAR) busdma - * is used. Allocate the scratch space to avoid getting - * faults from DMAR, by requesting scratch memory for one more - * descriptor. - */ - tsize = roundup2((adapter->num_tx_desc + 1) * - sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); - - /* Allocate Transmit Descriptor ring */ - if (lem_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { - device_printf(dev, "Unable to allocate tx_desc memory\n"); - error = ENOMEM; - goto err_tx_desc; - } - adapter->tx_desc_base = - (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; - - /* - * See comment above txdma allocation for rationale behind +1. - */ - rsize = roundup2((adapter->num_rx_desc + 1) * - sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); - - /* Allocate Receive Descriptor ring */ - if (lem_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { - device_printf(dev, "Unable to allocate rx_desc memory\n"); - error = ENOMEM; - goto err_rx_desc; - } - adapter->rx_desc_base = - (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; - - /* Allocate multicast array memory. */ - adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * - MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); - if (adapter->mta == NULL) { - device_printf(dev, "Can not allocate multicast setup array\n"); - error = ENOMEM; - goto err_hw_init; - } - - /* - ** Start from a known state, this is - ** important in reading the nvm and - ** mac from that. - */ - e1000_reset_hw(&adapter->hw); - - /* Make sure we have a good EEPROM before we read from it */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - /* - ** Some PCI-E parts fail the first check due to - ** the link being in sleep state, call it again, - ** if it fails a second time its a real issue. - */ - if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { - device_printf(dev, - "The EEPROM Checksum Is Not Valid\n"); - error = EIO; - goto err_hw_init; - } - } - - /* Copy the permanent MAC address out of the EEPROM */ - if (e1000_read_mac_addr(&adapter->hw) < 0) { - device_printf(dev, "EEPROM read error while reading MAC" - " address\n"); - error = EIO; - goto err_hw_init; - } - - if (!lem_is_valid_ether_addr(adapter->hw.mac.addr)) { - device_printf(dev, "Invalid MAC address\n"); - error = EIO; - goto err_hw_init; - } - - /* Initialize the hardware */ - if (lem_hardware_init(adapter)) { - device_printf(dev, "Unable to initialize the hardware\n"); - error = EIO; - goto err_hw_init; - } - - /* Allocate transmit descriptors and buffers */ - if (lem_allocate_transmit_structures(adapter)) { - device_printf(dev, "Could not setup transmit structures\n"); - error = ENOMEM; - goto err_tx_struct; - } - - /* Allocate receive descriptors and buffers */ - if (lem_allocate_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - error = ENOMEM; - goto err_rx_struct; - } - - /* - ** Do interrupt configuration - */ - error = lem_allocate_irq(adapter); - if (error) - goto err_rx_struct; - - /* - * Get Wake-on-Lan and Management info for later use - */ - lem_get_wakeup(dev); - - /* Setup OS specific network interface */ - if (lem_setup_interface(dev, adapter) != 0) - goto err_rx_struct; - - /* Initialize statistics */ - lem_update_stats_counters(adapter); - - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - - /* Indicate SOL/IDER usage */ - if (e1000_check_reset_block(&adapter->hw)) - device_printf(dev, - "PHY reset is blocked due to SOL/IDER session.\n"); - - /* Do we need workaround for 82544 PCI-X adapter? */ - if (adapter->hw.bus.type == e1000_bus_type_pcix && - adapter->hw.mac.type == e1000_82544) - adapter->pcix_82544 = TRUE; - else - adapter->pcix_82544 = FALSE; - - /* Register for VLAN events */ - adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, - lem_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); - adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, - lem_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); - - lem_add_hw_stats(adapter); - - /* Non-AMT based hardware can now take control from firmware */ - if (adapter->has_manage && !adapter->has_amt) - lem_get_hw_control(adapter); - - /* Tell the stack that the interface is not active */ - if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_OACTIVE | IFF_DRV_RUNNING); - - adapter->led_dev = led_create(lem_led_func, adapter, - device_get_nameunit(dev)); - -#ifdef DEV_NETMAP - lem_netmap_attach(adapter); -#endif /* DEV_NETMAP */ - INIT_DEBUGOUT("lem_attach: end"); - - return (0); - -err_rx_struct: - lem_free_transmit_structures(adapter); -err_tx_struct: -err_hw_init: - lem_release_hw_control(adapter); - lem_dma_free(adapter, &adapter->rxdma); -err_rx_desc: - lem_dma_free(adapter, &adapter->txdma); -err_tx_desc: -err_pci: - if (adapter->ifp != (void *)NULL) - if_free(adapter->ifp); - lem_free_pci_resources(adapter); - free(adapter->mta, M_DEVBUF); - EM_TX_LOCK_DESTROY(adapter); - EM_RX_LOCK_DESTROY(adapter); - EM_CORE_LOCK_DESTROY(adapter); - - return (error); -} - -/********************************************************************* - * Device removal routine - * - * The detach entry point is called when the driver is being removed. - * This routine stops the adapter and deallocates all the resources - * that were allocated for driver operation. - * - * return 0 on success, positive on failure - *********************************************************************/ - -static int -lem_detach(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - - INIT_DEBUGOUT("em_detach: begin"); - - /* Make sure VLANS are not using driver */ - if (if_vlantrunkinuse(ifp)) { - device_printf(dev,"Vlan in use, detach first\n"); - return (EBUSY); - } - -#ifdef DEVICE_POLLING - if (if_getcapenable(ifp) & IFCAP_POLLING) - ether_poll_deregister(ifp); -#endif - - if (adapter->led_dev != NULL) - led_destroy(adapter->led_dev); - - EM_CORE_LOCK(adapter); - EM_TX_LOCK(adapter); - adapter->in_detach = 1; - lem_stop(adapter); - e1000_phy_hw_reset(&adapter->hw); - - lem_release_manageability(adapter); - - EM_TX_UNLOCK(adapter); - EM_CORE_UNLOCK(adapter); - - /* Unregister VLAN events */ - if (adapter->vlan_attach != NULL) - EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); - if (adapter->vlan_detach != NULL) - EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); - - ether_ifdetach(adapter->ifp); - callout_drain(&adapter->timer); - callout_drain(&adapter->tx_fifo_timer); - -#ifdef DEV_NETMAP - netmap_detach(ifp); -#endif /* DEV_NETMAP */ - lem_free_pci_resources(adapter); - bus_generic_detach(dev); - if_free(ifp); - - lem_free_transmit_structures(adapter); - lem_free_receive_structures(adapter); - - /* Free Transmit Descriptor ring */ - if (adapter->tx_desc_base) { - lem_dma_free(adapter, &adapter->txdma); - adapter->tx_desc_base = NULL; - } - - /* Free Receive Descriptor ring */ - if (adapter->rx_desc_base) { - lem_dma_free(adapter, &adapter->rxdma); - adapter->rx_desc_base = NULL; - } - - lem_release_hw_control(adapter); - free(adapter->mta, M_DEVBUF); - EM_TX_LOCK_DESTROY(adapter); - EM_RX_LOCK_DESTROY(adapter); - EM_CORE_LOCK_DESTROY(adapter); - - return (0); -} - -/********************************************************************* - * - * Shutdown entry point - * - **********************************************************************/ - -static int -lem_shutdown(device_t dev) -{ - return lem_suspend(dev); -} - -/* - * Suspend/resume device methods. - */ -static int -lem_suspend(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - - EM_CORE_LOCK(adapter); - - lem_release_manageability(adapter); - lem_release_hw_control(adapter); - lem_enable_wakeup(dev); - - EM_CORE_UNLOCK(adapter); - - return bus_generic_suspend(dev); -} - -static int -lem_resume(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - - EM_CORE_LOCK(adapter); - lem_init_locked(adapter); - lem_init_manageability(adapter); - EM_CORE_UNLOCK(adapter); - lem_start(ifp); - - return bus_generic_resume(dev); -} - - -static void -lem_start_locked(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct mbuf *m_head; - - EM_TX_LOCK_ASSERT(adapter); - - if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != - IFF_DRV_RUNNING) - return; - if (!adapter->link_active) - return; - - /* - * Force a cleanup if number of TX descriptors - * available hits the threshold - */ - if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { - lem_txeof(adapter); - /* Now do we at least have a minimal? */ - if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { - adapter->no_tx_desc_avail1++; - return; - } - } - - while (!if_sendq_empty(ifp)) { - m_head = if_dequeue(ifp); - - if (m_head == NULL) - break; - /* - * Encapsulation can modify our pointer, and or make it - * NULL on failure. In that event, we can't requeue. - */ - if (lem_xmit(adapter, &m_head)) { - if (m_head == NULL) - break; - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); - if_sendq_prepend(ifp, m_head); - break; - } - - /* Send a copy of the frame to the BPF listener */ - if_etherbpfmtap(ifp, m_head); - - /* Set timeout in case hardware has problems transmitting. */ - adapter->watchdog_check = TRUE; - adapter->watchdog_time = ticks; - } - if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) - if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); - - return; -} - -static void -lem_start(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - - EM_TX_LOCK(adapter); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); -} - -/********************************************************************* - * Ioctl entry point - * - * em_ioctl is called when the user wants to configure the - * interface. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -lem_ioctl(if_t ifp, u_long command, caddr_t data) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifreq *ifr = (struct ifreq *)data; -#if defined(INET) || defined(INET6) - struct ifaddr *ifa = (struct ifaddr *)data; -#endif - bool avoid_reset = FALSE; - int error = 0; - - if (adapter->in_detach) - return (error); - - switch (command) { - case SIOCSIFADDR: -#ifdef INET - if (ifa->ifa_addr->sa_family == AF_INET) - avoid_reset = TRUE; -#endif -#ifdef INET6 - if (ifa->ifa_addr->sa_family == AF_INET6) - avoid_reset = TRUE; -#endif - /* - ** Calling init results in link renegotiation, - ** so we avoid doing it when possible. - */ - if (avoid_reset) { - if_setflagbits(ifp, IFF_UP, 0); - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - lem_init(adapter); -#ifdef INET - if (!(if_getflags(ifp) & IFF_NOARP)) - arp_ifinit(ifp, ifa); -#endif - } else - error = ether_ioctl(ifp, command, data); - break; - case SIOCSIFMTU: - { - int max_frame_size; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); - - EM_CORE_LOCK(adapter); - switch (adapter->hw.mac.type) { - case e1000_82542: - max_frame_size = ETHER_MAX_LEN; - break; - default: - max_frame_size = MAX_JUMBO_FRAME_SIZE; - } - if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - - ETHER_CRC_LEN) { - EM_CORE_UNLOCK(adapter); - error = EINVAL; - break; - } - - if_setmtu(ifp, ifr->ifr_mtu); - adapter->max_frame_size = - if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - break; - } - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd:\ - SIOCSIFFLAGS (Set Interface Flags)"); - EM_CORE_LOCK(adapter); - if (if_getflags(ifp) & IFF_UP) { - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { - if ((if_getflags(ifp) ^ adapter->if_flags) & - (IFF_PROMISC | IFF_ALLMULTI)) { - lem_disable_promisc(adapter); - lem_set_promisc(adapter); - } - } else - lem_init_locked(adapter); - } else - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - } - adapter->if_flags = if_getflags(ifp); - EM_CORE_UNLOCK(adapter); - break; - case SIOCADDMULTI: - case SIOCDELMULTI: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - EM_CORE_LOCK(adapter); - lem_disable_intr(adapter); - lem_set_multi(adapter); - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - lem_initialize_receive_unit(adapter); - } -#ifdef DEVICE_POLLING - if (!(if_getcapenable(ifp) & IFCAP_POLLING)) -#endif - lem_enable_intr(adapter); - EM_CORE_UNLOCK(adapter); - } - break; - case SIOCSIFMEDIA: - /* Check SOL/IDER usage */ - EM_CORE_LOCK(adapter); - if (e1000_check_reset_block(&adapter->hw)) { - EM_CORE_UNLOCK(adapter); - device_printf(adapter->dev, "Media change is" - " blocked due to SOL/IDER session.\n"); - break; - } - EM_CORE_UNLOCK(adapter); - case SIOCGIFMEDIA: - IOCTL_DEBUGOUT("ioctl rcv'd: \ - SIOCxIFMEDIA (Get/Set Interface Media)"); - error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); - break; - case SIOCSIFCAP: - { - int mask, reinit; - - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); - reinit = 0; - mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); -#ifdef DEVICE_POLLING - if (mask & IFCAP_POLLING) { - if (ifr->ifr_reqcap & IFCAP_POLLING) { - error = ether_poll_register(lem_poll, ifp); - if (error) - return (error); - EM_CORE_LOCK(adapter); - lem_disable_intr(adapter); - if_setcapenablebit(ifp, IFCAP_POLLING, 0); - EM_CORE_UNLOCK(adapter); - } else { - error = ether_poll_deregister(ifp); - /* Enable interrupt even in error case */ - EM_CORE_LOCK(adapter); - lem_enable_intr(adapter); - if_setcapenablebit(ifp, 0, IFCAP_POLLING); - EM_CORE_UNLOCK(adapter); - } - } -#endif - if (mask & IFCAP_HWCSUM) { - if_togglecapenable(ifp, IFCAP_HWCSUM); - reinit = 1; - } - if (mask & IFCAP_VLAN_HWTAGGING) { - if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); - reinit = 1; - } - if ((mask & IFCAP_WOL) && - (if_getcapabilities(ifp) & IFCAP_WOL) != 0) { - if (mask & IFCAP_WOL_MCAST) - if_togglecapenable(ifp, IFCAP_WOL_MCAST); - if (mask & IFCAP_WOL_MAGIC) - if_togglecapenable(ifp, IFCAP_WOL_MAGIC); - } - if (reinit && (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - lem_init(adapter); - if_vlancap(ifp); - break; - } - - default: - error = ether_ioctl(ifp, command, data); - break; - } - - return (error); -} - - -/********************************************************************* - * Init entry point - * - * This routine is used in two ways. It is used by the stack as - * init entry point in network interface structure. It is also used - * by the driver as a hw/sw initialization routine to get to a - * consistent state. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static void -lem_init_locked(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - u32 pba; - - INIT_DEBUGOUT("lem_init: begin"); - - EM_CORE_LOCK_ASSERT(adapter); - - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - - /* - * Packet Buffer Allocation (PBA) - * Writing PBA sets the receive portion of the buffer - * the remainder is used for the transmit buffer. - * - * Devices before the 82547 had a Packet Buffer of 64K. - * Default allocation: PBA=48K for Rx, leaving 16K for Tx. - * After the 82547 the buffer was reduced to 40K. - * Default allocation: PBA=30K for Rx, leaving 10K for Tx. - * Note: default does not leave enough room for Jumbo Frame >10k. - */ - switch (adapter->hw.mac.type) { - case e1000_82547: - case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ - if (adapter->max_frame_size > 8192) - pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ - else - pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ - adapter->tx_fifo_head = 0; - adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; - adapter->tx_fifo_size = - (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; - break; - default: - /* Devices before 82547 had a Packet Buffer of 64K. */ - if (adapter->max_frame_size > 8192) - pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ - else - pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ - } - - INIT_DEBUGOUT1("lem_init: pba=%dK",pba); - E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); - - /* Get the latest mac address, User can use a LAA */ - bcopy(if_getlladdr(adapter->ifp), adapter->hw.mac.addr, - ETHER_ADDR_LEN); - - /* Put the address into the Receive Address Array */ - e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); - - /* Initialize the hardware */ - if (lem_hardware_init(adapter)) { - device_printf(dev, "Unable to initialize the hardware\n"); - return; - } - lem_update_link_status(adapter); - - /* Setup VLAN support, basic and offload if available */ - E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); - - /* Set hardware offload abilities */ - if_clearhwassist(ifp); - if (adapter->hw.mac.type >= e1000_82543) { - if (if_getcapenable(ifp) & IFCAP_TXCSUM) - if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); - } - - /* Configure for OS presence */ - lem_init_manageability(adapter); - - /* Prepare transmit descriptors and buffers */ - lem_setup_transmit_structures(adapter); - lem_initialize_transmit_unit(adapter); - - /* Setup Multicast table */ - lem_set_multi(adapter); - - /* Prepare receive descriptors and buffers */ - if (lem_setup_receive_structures(adapter)) { - device_printf(dev, "Could not setup receive structures\n"); - EM_TX_LOCK(adapter); - lem_stop(adapter); - EM_TX_UNLOCK(adapter); - return; - } - lem_initialize_receive_unit(adapter); - - /* Use real VLAN Filter support? */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) { - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - /* Use real VLAN Filter support */ - lem_setup_vlan_hw_support(adapter); - else { - u32 ctrl; - ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); - ctrl |= E1000_CTRL_VME; - E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); - } - } - - /* Don't lose promiscuous settings */ - lem_set_promisc(adapter); - - if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); - - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - e1000_clear_hw_cntrs_base_generic(&adapter->hw); - -#ifdef DEVICE_POLLING - /* - * Only enable interrupts if we are not polling, make sure - * they are off otherwise. - */ - if (if_getcapenable(ifp) & IFCAP_POLLING) - lem_disable_intr(adapter); - else -#endif /* DEVICE_POLLING */ - lem_enable_intr(adapter); - - /* AMT based hardware can now take control from firmware */ - if (adapter->has_manage && adapter->has_amt) - lem_get_hw_control(adapter); -} - -static void -lem_init(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - - -#ifdef DEVICE_POLLING -/********************************************************************* - * - * Legacy polling routine - * - *********************************************************************/ -static int -lem_poll(if_t ifp, enum poll_cmd cmd, int count) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 reg_icr, rx_done = 0; - - EM_CORE_LOCK(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) { - EM_CORE_UNLOCK(adapter); - return (rx_done); - } - - if (cmd == POLL_AND_CHECK_STATUS) { - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - callout_reset(&adapter->timer, hz, - lem_local_timer, adapter); - } - } - EM_CORE_UNLOCK(adapter); - - lem_rxeof(adapter, count, &rx_done); - - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if(!if_sendq_empty(ifp)) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - return (rx_done); -} -#endif /* DEVICE_POLLING */ - -/********************************************************************* - * - * Legacy Interrupt Service routine - * - *********************************************************************/ -static void -lem_intr(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - u32 reg_icr; - - - if ((if_getcapenable(ifp) & IFCAP_POLLING) || - ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) - return; - - EM_CORE_LOCK(adapter); - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - - if ((reg_icr == 0xffffffff) || (reg_icr == 0)) { - EM_CORE_UNLOCK(adapter); - return; - } - - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - callout_stop(&adapter->timer); - adapter->hw.mac.get_link_status = 1; - lem_update_link_status(adapter); - /* Deal with TX cruft when link lost */ - lem_tx_purge(adapter); - callout_reset(&adapter->timer, hz, - lem_local_timer, adapter); - EM_CORE_UNLOCK(adapter); - return; - } - - EM_CORE_UNLOCK(adapter); - lem_rxeof(adapter, -1, NULL); - - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) && - (!if_sendq_empty(ifp))) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - return; -} - - -static void -lem_handle_link(void *context, int pending) -{ - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - - if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) - return; - - EM_CORE_LOCK(adapter); - callout_stop(&adapter->timer); - lem_update_link_status(adapter); - /* Deal with TX cruft when link lost */ - lem_tx_purge(adapter); - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - EM_CORE_UNLOCK(adapter); -} - - -/* Combined RX/TX handler, used by Legacy and MSI */ -static void -lem_handle_rxtx(void *context, int pending) -{ - struct adapter *adapter = context; - if_t ifp = adapter->ifp; - - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - bool more = lem_rxeof(adapter, adapter->rx_process_limit, NULL); - EM_TX_LOCK(adapter); - lem_txeof(adapter); - if(!if_sendq_empty(ifp)) - lem_start_locked(ifp); - EM_TX_UNLOCK(adapter); - if (more) { - taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); - return; - } - } - - if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) - lem_enable_intr(adapter); -} - -/********************************************************************* - * - * Fast Legacy/MSI Combined Interrupt Service routine - * - *********************************************************************/ -static int -lem_irq_fast(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp; - u32 reg_icr; - - ifp = adapter->ifp; - - reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); - - /* Hot eject? */ - if (reg_icr == 0xffffffff) - return FILTER_STRAY; - - /* Definitely not our interrupt. */ - if (reg_icr == 0x0) - return FILTER_STRAY; - - /* - * Mask interrupts until the taskqueue is finished running. This is - * cheap, just assume that it is needed. This also works around the - * MSI message reordering errata on certain systems. - */ - lem_disable_intr(adapter); - taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); - - /* Link status change */ - if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { - adapter->hw.mac.get_link_status = 1; - taskqueue_enqueue(taskqueue_fast, &adapter->link_task); - } - - if (reg_icr & E1000_ICR_RXO) - adapter->rx_overruns++; - return FILTER_HANDLED; -} - - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called whenever the user queries the status of - * the interface using ifconfig. - * - **********************************************************************/ -static void -lem_media_status(if_t ifp, struct ifmediareq *ifmr) -{ - struct adapter *adapter = if_getsoftc(ifp); - u_char fiber_type = IFM_1000_SX; - - INIT_DEBUGOUT("lem_media_status: begin"); - - EM_CORE_LOCK(adapter); - lem_update_link_status(adapter); - - ifmr->ifm_status = IFM_AVALID; - ifmr->ifm_active = IFM_ETHER; - - if (!adapter->link_active) { - EM_CORE_UNLOCK(adapter); - return; - } - - ifmr->ifm_status |= IFM_ACTIVE; - - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - if (adapter->hw.mac.type == e1000_82545) - fiber_type = IFM_1000_LX; - ifmr->ifm_active |= fiber_type | IFM_FDX; - } else { - switch (adapter->link_speed) { - case 10: - ifmr->ifm_active |= IFM_10_T; - break; - case 100: - ifmr->ifm_active |= IFM_100_TX; - break; - case 1000: - ifmr->ifm_active |= IFM_1000_T; - break; - } - if (adapter->link_duplex == FULL_DUPLEX) - ifmr->ifm_active |= IFM_FDX; - else - ifmr->ifm_active |= IFM_HDX; - } - EM_CORE_UNLOCK(adapter); -} - -/********************************************************************* - * - * Media Ioctl callback - * - * This routine is called when the user changes speed/duplex using - * media/mediopt option with ifconfig. - * - **********************************************************************/ -static int -lem_media_change(if_t ifp) -{ - struct adapter *adapter = if_getsoftc(ifp); - struct ifmedia *ifm = &adapter->media; - - INIT_DEBUGOUT("lem_media_change: begin"); - - if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) - return (EINVAL); - - EM_CORE_LOCK(adapter); - switch (IFM_SUBTYPE(ifm->ifm_media)) { - case IFM_AUTO: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; - break; - case IFM_1000_LX: - case IFM_1000_SX: - case IFM_1000_T: - adapter->hw.mac.autoneg = DO_AUTO_NEG; - adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; - break; - case IFM_100_TX: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; - break; - case IFM_10_T: - adapter->hw.mac.autoneg = FALSE; - adapter->hw.phy.autoneg_advertised = 0; - if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; - else - adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; - break; - default: - device_printf(adapter->dev, "Unsupported media type\n"); - } - - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); - - return (0); -} - -/********************************************************************* - * - * This routine maps the mbufs to tx descriptors. - * - * return 0 on success, positive on failure - **********************************************************************/ - -static int -lem_xmit(struct adapter *adapter, struct mbuf **m_headp) -{ - bus_dma_segment_t segs[EM_MAX_SCATTER]; - bus_dmamap_t map; - struct em_buffer *tx_buffer, *tx_buffer_mapped; - struct e1000_tx_desc *ctxd = NULL; - struct mbuf *m_head; - u32 txd_upper, txd_lower, txd_used, txd_saved; - int error, nsegs, i, j, first, last = 0; - - m_head = *m_headp; - txd_upper = txd_lower = txd_used = txd_saved = 0; - - /* - ** When doing checksum offload, it is critical to - ** make sure the first mbuf has more than header, - ** because that routine expects data to be present. - */ - if ((m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) && - (m_head->m_len < ETHER_HDR_LEN + sizeof(struct ip))) { - m_head = m_pullup(m_head, ETHER_HDR_LEN + sizeof(struct ip)); - *m_headp = m_head; - if (m_head == NULL) - return (ENOBUFS); - } - - /* - * Map the packet for DMA - * - * Capture the first descriptor index, - * this descriptor will have the index - * of the EOP which is the only one that - * now gets a DONE bit writeback. - */ - first = adapter->next_avail_tx_desc; - tx_buffer = &adapter->tx_buffer_area[first]; - tx_buffer_mapped = tx_buffer; - map = tx_buffer->map; - - error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - /* - * There are two types of errors we can (try) to handle: - * - EFBIG means the mbuf chain was too long and bus_dma ran - * out of segments. Defragment the mbuf chain and try again. - * - ENOMEM means bus_dma could not obtain enough bounce buffers - * at this point in time. Defer sending and try again later. - * All other errors, in particular EINVAL, are fatal and prevent the - * mbuf chain from ever going through. Drop it and report error. - */ - if (error == EFBIG) { - struct mbuf *m; - - m = m_collapse(*m_headp, M_NOWAIT, EM_MAX_SCATTER); - if (m == NULL) { - adapter->mbuf_defrag_failed++; - m_freem(*m_headp); - *m_headp = NULL; - return (ENOBUFS); - } - *m_headp = m; - - /* Try it again */ - error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, - *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - - if (error) { - adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; - return (error); - } - } else if (error != 0) { - adapter->no_tx_dma_setup++; - return (error); - } - - if (adapter->num_tx_desc_avail < (nsegs + 2)) { - adapter->no_tx_desc_avail2++; - bus_dmamap_unload(adapter->txtag, map); - return (ENOBUFS); - } - m_head = *m_headp; - - /* Do hardware assists */ - if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) - lem_transmit_checksum_setup(adapter, m_head, - &txd_upper, &txd_lower); - - i = adapter->next_avail_tx_desc; - if (adapter->pcix_82544) - txd_saved = i; - - /* Set up our transmit descriptors */ - for (j = 0; j < nsegs; j++) { - bus_size_t seg_len; - bus_addr_t seg_addr; - /* If adapter is 82544 and on PCIX bus */ - if(adapter->pcix_82544) { - DESC_ARRAY desc_array; - u32 array_elements, counter; - /* - * Check the Address and Length combination and - * split the data accordingly - */ - array_elements = lem_fill_descriptors(segs[j].ds_addr, - segs[j].ds_len, &desc_array); - for (counter = 0; counter < array_elements; counter++) { - if (txd_used == adapter->num_tx_desc_avail) { - adapter->next_avail_tx_desc = txd_saved; - adapter->no_tx_desc_avail2++; - bus_dmamap_unload(adapter->txtag, map); - return (ENOBUFS); - } - tx_buffer = &adapter->tx_buffer_area[i]; - ctxd = &adapter->tx_desc_base[i]; - ctxd->buffer_addr = htole64( - desc_array.descriptor[counter].address); - ctxd->lower.data = htole32( - (adapter->txd_cmd | txd_lower | (u16) - desc_array.descriptor[counter].length)); - ctxd->upper.data = - htole32((txd_upper)); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - txd_used++; - } - } else { - tx_buffer = &adapter->tx_buffer_area[i]; - ctxd = &adapter->tx_desc_base[i]; - seg_addr = segs[j].ds_addr; - seg_len = segs[j].ds_len; - ctxd->buffer_addr = htole64(seg_addr); - ctxd->lower.data = htole32( - adapter->txd_cmd | txd_lower | seg_len); - ctxd->upper.data = - htole32(txd_upper); - last = i; - if (++i == adapter->num_tx_desc) - i = 0; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - } - } - - adapter->next_avail_tx_desc = i; - - if (adapter->pcix_82544) - adapter->num_tx_desc_avail -= txd_used; - else - adapter->num_tx_desc_avail -= nsegs; - - if (m_head->m_flags & M_VLANTAG) { - /* Set the vlan id. */ - ctxd->upper.fields.special = - htole16(m_head->m_pkthdr.ether_vtag); - /* Tell hardware to add tag */ - ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); - } - - tx_buffer->m_head = m_head; - tx_buffer_mapped->map = tx_buffer->map; - tx_buffer->map = map; - bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); - - /* - * Last Descriptor of Packet - * needs End Of Packet (EOP) - * and Report Status (RS) - */ - ctxd->lower.data |= - htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); - /* - * Keep track in the first buffer which - * descriptor will be written back - */ - tx_buffer = &adapter->tx_buffer_area[first]; - tx_buffer->next_eop = last; - adapter->watchdog_time = ticks; - - /* - * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 - * that this frame is available to transmit. - */ - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - -#ifdef NIC_SEND_COMBINING - if (adapter->sc_enable) { - if (adapter->shadow_tdt & MIT_PENDING_INT) { - /* signal intr and data pending */ - adapter->shadow_tdt = MIT_PENDING_TDT | (i & 0xffff); - return (0); - } else { - adapter->shadow_tdt = MIT_PENDING_INT; - } - } -#endif /* NIC_SEND_COMBINING */ - - if (adapter->hw.mac.type == e1000_82547 && - adapter->link_duplex == HALF_DUPLEX) - lem_82547_move_tail(adapter); - else { - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); - if (adapter->hw.mac.type == e1000_82547) - lem_82547_update_fifo_head(adapter, - m_head->m_pkthdr.len); - } - - return (0); -} - -/********************************************************************* - * - * 82547 workaround to avoid controller hang in half-duplex environment. - * The workaround is to avoid queuing a large packet that would span - * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers - * in this case. We do that only when FIFO is quiescent. - * - **********************************************************************/ -static void -lem_82547_move_tail(void *arg) -{ - struct adapter *adapter = arg; - struct e1000_tx_desc *tx_desc; - u16 hw_tdt, sw_tdt, length = 0; - bool eop = 0; - - EM_TX_LOCK_ASSERT(adapter); - - hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); - sw_tdt = adapter->next_avail_tx_desc; - - while (hw_tdt != sw_tdt) { - tx_desc = &adapter->tx_desc_base[hw_tdt]; - length += tx_desc->lower.flags.length; - eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; - if (++hw_tdt == adapter->num_tx_desc) - hw_tdt = 0; - - if (eop) { - if (lem_82547_fifo_workaround(adapter, length)) { - adapter->tx_fifo_wrk_cnt++; - callout_reset(&adapter->tx_fifo_timer, 1, - lem_82547_move_tail, adapter); - break; - } - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); - lem_82547_update_fifo_head(adapter, length); - length = 0; - } - } -} - -static int -lem_82547_fifo_workaround(struct adapter *adapter, int len) -{ - int fifo_space, fifo_pkt_len; - - fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); - - if (adapter->link_duplex == HALF_DUPLEX) { - fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; - - if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { - if (lem_82547_tx_fifo_reset(adapter)) - return (0); - else - return (1); - } - } - - return (0); -} - -static void -lem_82547_update_fifo_head(struct adapter *adapter, int len) -{ - int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); - - /* tx_fifo_head is always 16 byte aligned */ - adapter->tx_fifo_head += fifo_pkt_len; - if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { - adapter->tx_fifo_head -= adapter->tx_fifo_size; - } -} - - -static int -lem_82547_tx_fifo_reset(struct adapter *adapter) -{ - u32 tctl; - - if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == - E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && - (E1000_READ_REG(&adapter->hw, E1000_TDFT) == - E1000_READ_REG(&adapter->hw, E1000_TDFH)) && - (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == - E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && - (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { - /* Disable TX unit */ - tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, - tctl & ~E1000_TCTL_EN); - - /* Reset FIFO pointers */ - E1000_WRITE_REG(&adapter->hw, E1000_TDFT, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFH, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, - adapter->tx_head_addr); - E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, - adapter->tx_head_addr); - - /* Re-enable TX unit */ - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); - E1000_WRITE_FLUSH(&adapter->hw); - - adapter->tx_fifo_head = 0; - adapter->tx_fifo_reset_cnt++; - - return (TRUE); - } - else { - return (FALSE); - } -} - -static void -lem_set_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; - - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - - if (if_getflags(ifp) & IFF_PROMISC) { - reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); - /* Turn this on if you want to see bad packets */ - if (lem_debug_sbp) - reg_rctl |= E1000_RCTL_SBP; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else if (if_getflags(ifp) & IFF_ALLMULTI) { - reg_rctl |= E1000_RCTL_MPE; - reg_rctl &= ~E1000_RCTL_UPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } -} - -static void -lem_disable_promisc(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl; - int mcnt = 0; - - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= (~E1000_RCTL_UPE); - if (if_getflags(ifp) & IFF_ALLMULTI) - mcnt = MAX_NUM_MULTICAST_ADDRESSES; - else - mcnt = if_multiaddr_count(ifp, MAX_NUM_MULTICAST_ADDRESSES); - - /* Don't disable if in MAX groups */ - if (mcnt < MAX_NUM_MULTICAST_ADDRESSES) - reg_rctl &= (~E1000_RCTL_MPE); - reg_rctl &= (~E1000_RCTL_SBP); - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); -} - - -/********************************************************************* - * Multicast Update - * - * This routine is called whenever multicast address list is updated. - * - **********************************************************************/ - -static void -lem_set_multi(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u32 reg_rctl = 0; - u8 *mta; /* Multicast array memory */ - int mcnt = 0; - - IOCTL_DEBUGOUT("lem_set_multi: begin"); - - mta = adapter->mta; - bzero(mta, sizeof(u8) * ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES); - - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) - e1000_pci_clear_mwi(&adapter->hw); - reg_rctl |= E1000_RCTL_RST; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - msec_delay(5); - } - - if_multiaddr_array(ifp, mta, &mcnt, MAX_NUM_MULTICAST_ADDRESSES); - - if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - } else - e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); - - if (adapter->hw.mac.type == e1000_82542 && - adapter->hw.revision_id == E1000_REVISION_2) { - reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - reg_rctl &= ~E1000_RCTL_RST; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); - msec_delay(5); - if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) - e1000_pci_set_mwi(&adapter->hw); - } -} - - -/********************************************************************* - * Timer routine - * - * This routine checks for link status and updates statistics. - * - **********************************************************************/ - -static void -lem_local_timer(void *arg) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK_ASSERT(adapter); - - lem_update_link_status(adapter); - lem_update_stats_counters(adapter); - - lem_smartspeed(adapter); - - /* - * We check the watchdog: the time since - * the last TX descriptor was cleaned. - * This implies a functional TX engine. - */ - if ((adapter->watchdog_check == TRUE) && - (ticks - adapter->watchdog_time > EM_WATCHDOG)) - goto hung; - - callout_reset(&adapter->timer, hz, lem_local_timer, adapter); - return; -hung: - device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - if_setdrvflagbits(adapter->ifp, 0, IFF_DRV_RUNNING); - adapter->watchdog_events++; - lem_init_locked(adapter); -} - -static void -lem_update_link_status(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - if_t ifp = adapter->ifp; - device_t dev = adapter->dev; - u32 link_check = 0; - - /* Get the cached link value or read phy for real */ - switch (hw->phy.media_type) { - case e1000_media_type_copper: - if (hw->mac.get_link_status) { - /* Do the work to read phy */ - e1000_check_for_link(hw); - link_check = !hw->mac.get_link_status; - if (link_check) /* ESB2 fix */ - e1000_cfg_on_link_up(hw); - } else - link_check = TRUE; - break; - case e1000_media_type_fiber: - e1000_check_for_link(hw); - link_check = (E1000_READ_REG(hw, E1000_STATUS) & - E1000_STATUS_LU); - break; - case e1000_media_type_internal_serdes: - e1000_check_for_link(hw); - link_check = adapter->hw.mac.serdes_has_link; - break; - default: - case e1000_media_type_unknown: - break; - } - - /* Now check for a transition */ - if (link_check && (adapter->link_active == 0)) { - e1000_get_speed_and_duplex(hw, &adapter->link_speed, - &adapter->link_duplex); - if (bootverbose) - device_printf(dev, "Link is up %d Mbps %s\n", - adapter->link_speed, - ((adapter->link_duplex == FULL_DUPLEX) ? - "Full Duplex" : "Half Duplex")); - adapter->link_active = 1; - adapter->smartspeed = 0; - if_setbaudrate(ifp, adapter->link_speed * 1000000); - if_link_state_change(ifp, LINK_STATE_UP); - } else if (!link_check && (adapter->link_active == 1)) { - if_setbaudrate(ifp, 0); - adapter->link_speed = 0; - adapter->link_duplex = 0; - if (bootverbose) - device_printf(dev, "Link is Down\n"); - adapter->link_active = 0; - /* Link down, disable watchdog */ - adapter->watchdog_check = FALSE; - if_link_state_change(ifp, LINK_STATE_DOWN); - } -} - -/********************************************************************* - * - * This routine disables all traffic on the adapter by issuing a - * global reset on the MAC and deallocates TX/RX buffers. - * - * This routine should always be called with BOTH the CORE - * and TX locks. - **********************************************************************/ - -static void -lem_stop(void *arg) -{ - struct adapter *adapter = arg; - if_t ifp = adapter->ifp; - - EM_CORE_LOCK_ASSERT(adapter); - EM_TX_LOCK_ASSERT(adapter); - - INIT_DEBUGOUT("lem_stop: begin"); - - lem_disable_intr(adapter); - callout_stop(&adapter->timer); - callout_stop(&adapter->tx_fifo_timer); - - /* Tell the stack that the interface is no longer active */ - if_setdrvflagbits(ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)); - - e1000_reset_hw(&adapter->hw); - if (adapter->hw.mac.type >= e1000_82544) - E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); - - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); -} - - -/********************************************************************* - * - * Determine hardware revision. - * - **********************************************************************/ -static void -lem_identify_hardware(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - /* Make sure our PCI config space has the necessary stuff set */ - pci_enable_busmaster(dev); - adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); - - /* Save off the information about this board */ - adapter->hw.vendor_id = pci_get_vendor(dev); - adapter->hw.device_id = pci_get_device(dev); - adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); - adapter->hw.subsystem_vendor_id = - pci_read_config(dev, PCIR_SUBVEND_0, 2); - adapter->hw.subsystem_device_id = - pci_read_config(dev, PCIR_SUBDEV_0, 2); - - /* Do Shared Code Init and Setup */ - if (e1000_set_mac_type(&adapter->hw)) { - device_printf(dev, "Setup init failure\n"); - return; - } -} - -static int -lem_allocate_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int val, rid, error = E1000_SUCCESS; - - rid = PCIR_BAR(0); - adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, - &rid, RF_ACTIVE); - if (adapter->memory == NULL) { - device_printf(dev, "Unable to allocate bus resource: memory\n"); - return (ENXIO); - } - adapter->osdep.mem_bus_space_tag = - rman_get_bustag(adapter->memory); - adapter->osdep.mem_bus_space_handle = - rman_get_bushandle(adapter->memory); - adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; - - /* Only older adapters use IO mapping */ - if (adapter->hw.mac.type > e1000_82543) { - /* Figure our where our IO BAR is ? */ - for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { - val = pci_read_config(dev, rid, 4); - if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { - adapter->io_rid = rid; - break; - } - rid += 4; - /* check for 64bit BAR */ - if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) - rid += 4; - } - if (rid >= PCIR_CIS) { - device_printf(dev, "Unable to locate IO BAR\n"); - return (ENXIO); - } - adapter->ioport = bus_alloc_resource_any(dev, - SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); - if (adapter->ioport == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "ioport\n"); - return (ENXIO); - } - adapter->hw.io_base = 0; - adapter->osdep.io_bus_space_tag = - rman_get_bustag(adapter->ioport); - adapter->osdep.io_bus_space_handle = - rman_get_bushandle(adapter->ioport); - } - - adapter->hw.back = &adapter->osdep; - - return (error); -} - -/********************************************************************* - * - * Setup the Legacy or MSI Interrupt handler - * - **********************************************************************/ -int -lem_allocate_irq(struct adapter *adapter) -{ - device_t dev = adapter->dev; - int error, rid = 0; - - /* Manually turn off all interrupts */ - E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); - - /* We allocate a single interrupt resource */ - adapter->res[0] = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (adapter->res[0] == NULL) { - device_printf(dev, "Unable to allocate bus resource: " - "interrupt\n"); - return (ENXIO); - } - - /* Do Legacy setup? */ - if (lem_use_legacy_irq) { - if ((error = bus_setup_intr(dev, adapter->res[0], - INTR_TYPE_NET | INTR_MPSAFE, NULL, lem_intr, adapter, - &adapter->tag[0])) != 0) { - device_printf(dev, - "Failed to register interrupt handler"); - return (error); - } - return (0); - } - - /* - * Use a Fast interrupt and the associated - * deferred processing contexts. - */ - TASK_INIT(&adapter->rxtx_task, 0, lem_handle_rxtx, adapter); - TASK_INIT(&adapter->link_task, 0, lem_handle_link, adapter); - adapter->tq = taskqueue_create_fast("lem_taskq", M_NOWAIT, - taskqueue_thread_enqueue, &adapter->tq); - taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", - device_get_nameunit(adapter->dev)); - if ((error = bus_setup_intr(dev, adapter->res[0], - INTR_TYPE_NET, lem_irq_fast, NULL, adapter, - &adapter->tag[0])) != 0) { - device_printf(dev, "Failed to register fast interrupt " - "handler: %d\n", error); - taskqueue_free(adapter->tq); - adapter->tq = NULL; - return (error); - } - - return (0); -} - - -static void -lem_free_pci_resources(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - - if (adapter->tag[0] != NULL) { - bus_teardown_intr(dev, adapter->res[0], - adapter->tag[0]); - adapter->tag[0] = NULL; - } - - if (adapter->res[0] != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, - 0, adapter->res[0]); - } - - if (adapter->memory != NULL) - bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); - - if (adapter->ioport != NULL) - bus_release_resource(dev, SYS_RES_IOPORT, - adapter->io_rid, adapter->ioport); -} - - -/********************************************************************* - * - * Initialize the hardware to a configuration - * as specified by the adapter structure. - * - **********************************************************************/ -static int -lem_hardware_init(struct adapter *adapter) -{ - device_t dev = adapter->dev; - u16 rx_buffer_size; - - INIT_DEBUGOUT("lem_hardware_init: begin"); - - /* Issue a global reset */ - e1000_reset_hw(&adapter->hw); - - /* When hardware is reset, fifo_head is also reset */ - adapter->tx_fifo_head = 0; - - /* - * These parameters control the automatic generation (Tx) and - * response (Rx) to Ethernet PAUSE frames. - * - High water mark should allow for at least two frames to be - * received after sending an XOFF. - * - Low water mark works best when it is very near the high water mark. - * This allows the receiver to restart by sending XON when it has - * drained a bit. Here we use an arbitrary value of 1500 which will - * restart after one full frame is pulled from the buffer. There - * could be several smaller frames in the buffer and if so they will - * not trigger the XON until their total number reduces the buffer - * by 1500. - * - The pause time is fairly large at 1000 x 512ns = 512 usec. - */ - rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & - 0xffff) << 10 ); - - adapter->hw.fc.high_water = rx_buffer_size - - roundup2(adapter->max_frame_size, 1024); - adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; - - adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; - adapter->hw.fc.send_xon = TRUE; - - /* Set Flow control, use the tunable location if sane */ - if ((lem_fc_setting >= 0) && (lem_fc_setting < 4)) - adapter->hw.fc.requested_mode = lem_fc_setting; - else - adapter->hw.fc.requested_mode = e1000_fc_none; - - if (e1000_init_hw(&adapter->hw) < 0) { - device_printf(dev, "Hardware Initialization Failed\n"); - return (EIO); - } - - e1000_check_for_link(&adapter->hw); - - return (0); -} - -/********************************************************************* - * - * Setup networking device structure and register an interface. - * - **********************************************************************/ -static int -lem_setup_interface(device_t dev, struct adapter *adapter) -{ - if_t ifp; - - INIT_DEBUGOUT("lem_setup_interface: begin"); - - ifp = adapter->ifp = if_gethandle(IFT_ETHER); - if (ifp == (void *)NULL) { - device_printf(dev, "can not allocate ifnet structure\n"); - return (-1); - } - if_initname(ifp, device_get_name(dev), device_get_unit(dev)); - if_setinitfn(ifp, lem_init); - if_setsoftc(ifp, adapter); - if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); - if_setioctlfn(ifp, lem_ioctl); - if_setstartfn(ifp, lem_start); - if_setgetcounterfn(ifp, lem_get_counter); - if_setsendqlen(ifp, adapter->num_tx_desc - 1); - if_setsendqready(ifp); - - ether_ifattach(ifp, adapter->hw.mac.addr); - - if_setcapabilities(ifp, 0); - - if (adapter->hw.mac.type >= e1000_82543) { - if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); - if_setcapenablebit(ifp, IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM, 0); - } - - /* - * Tell the upper layer(s) we support long frames. - */ - if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); - if_setcapenablebit(ifp, IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU, 0); - - /* - ** Dont turn this on by default, if vlans are - ** created on another pseudo device (eg. lagg) - ** then vlan events are not passed thru, breaking - ** operation, but with HW FILTER off it works. If - ** using vlans directly on the em driver you can - ** enable this and get full hardware tag filtering. - */ - if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); - -#ifdef DEVICE_POLLING - if_setcapabilitiesbit(ifp, IFCAP_POLLING, 0); -#endif - - /* Enable only WOL MAGIC by default */ - if (adapter->wol) { - if_setcapabilitiesbit(ifp, IFCAP_WOL, 0); - if_setcapenablebit(ifp, IFCAP_WOL_MAGIC, 0); - } - - /* - * Specify the media types supported by this adapter and register - * callbacks to update media and link information - */ - ifmedia_init(&adapter->media, IFM_IMASK, - lem_media_change, lem_media_status); - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { - u_char fiber_type = IFM_1000_SX; /* default type */ - - if (adapter->hw.mac.type == e1000_82545) - fiber_type = IFM_1000_LX; - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); - } else { - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, - 0, NULL); - ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, - 0, NULL); - if (adapter->hw.phy.type != e1000_phy_ife) { - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); - ifmedia_add(&adapter->media, - IFM_ETHER | IFM_1000_T, 0, NULL); - } - } - ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); - ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); - return (0); -} - - -/********************************************************************* - * - * Workaround for SmartSpeed on 82541 and 82547 controllers - * - **********************************************************************/ -static void -lem_smartspeed(struct adapter *adapter) -{ - u16 phy_tmp; - - if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || - adapter->hw.mac.autoneg == 0 || - (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) - return; - - if (adapter->smartspeed == 0) { - /* If Master/Slave config fault is asserted twice, - * we assume back-to-back */ - e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); - if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) - return; - e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); - if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { - e1000_read_phy_reg(&adapter->hw, - PHY_1000T_CTRL, &phy_tmp); - if(phy_tmp & CR_1000T_MS_ENABLE) { - phy_tmp &= ~CR_1000T_MS_ENABLE; - e1000_write_phy_reg(&adapter->hw, - PHY_1000T_CTRL, phy_tmp); - adapter->smartspeed++; - if(adapter->hw.mac.autoneg && - !e1000_copper_link_autoneg(&adapter->hw) && - !e1000_read_phy_reg(&adapter->hw, - PHY_CONTROL, &phy_tmp)) { - phy_tmp |= (MII_CR_AUTO_NEG_EN | - MII_CR_RESTART_AUTO_NEG); - e1000_write_phy_reg(&adapter->hw, - PHY_CONTROL, phy_tmp); - } - } - } - return; - } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { - /* If still no link, perhaps using 2/3 pair cable */ - e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); - phy_tmp |= CR_1000T_MS_ENABLE; - e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); - if(adapter->hw.mac.autoneg && - !e1000_copper_link_autoneg(&adapter->hw) && - !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { - phy_tmp |= (MII_CR_AUTO_NEG_EN | - MII_CR_RESTART_AUTO_NEG); - e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); - } - } - /* Restart process after EM_SMARTSPEED_MAX iterations */ - if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) - adapter->smartspeed = 0; -} - - -/* - * Manage DMA'able memory. - */ -static void -lem_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) -{ - if (error) - return; - *(bus_addr_t *) arg = segs[0].ds_addr; -} - -static int -lem_dma_malloc(struct adapter *adapter, bus_size_t size, - struct em_dma_alloc *dma, int mapflags) -{ - int error; - - error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ - EM_DBA_ALIGN, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - size, /* maxsize */ - 1, /* nsegments */ - size, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &dma->dma_tag); - if (error) { - device_printf(adapter->dev, - "%s: bus_dma_tag_create failed: %d\n", - __func__, error); - goto fail_0; - } - - error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, - BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); - if (error) { - device_printf(adapter->dev, - "%s: bus_dmamem_alloc(%ju) failed: %d\n", - __func__, (uintmax_t)size, error); - goto fail_2; - } - - dma->dma_paddr = 0; - error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, - size, lem_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); - if (error || dma->dma_paddr == 0) { - device_printf(adapter->dev, - "%s: bus_dmamap_load failed: %d\n", - __func__, error); - goto fail_3; - } - - return (0); - -fail_3: - bus_dmamap_unload(dma->dma_tag, dma->dma_map); -fail_2: - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - bus_dma_tag_destroy(dma->dma_tag); -fail_0: - dma->dma_tag = NULL; - - return (error); -} - -static void -lem_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) -{ - if (dma->dma_tag == NULL) - return; - if (dma->dma_paddr != 0) { - bus_dmamap_sync(dma->dma_tag, dma->dma_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(dma->dma_tag, dma->dma_map); - dma->dma_paddr = 0; - } - if (dma->dma_vaddr != NULL) { - bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); - dma->dma_vaddr = NULL; - } - bus_dma_tag_destroy(dma->dma_tag); - dma->dma_tag = NULL; -} - - -/********************************************************************* - * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. - * - **********************************************************************/ -static int -lem_allocate_transmit_structures(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct em_buffer *tx_buffer; - int error; - - /* - * Create DMA tags for tx descriptors - */ - if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MCLBYTES * EM_MAX_SCATTER, /* maxsize */ - EM_MAX_SCATTER, /* nsegments */ - MCLBYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &adapter->txtag)) != 0) { - device_printf(dev, "Unable to allocate TX DMA tag\n"); - goto fail; - } - - adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * - adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->tx_buffer_area == NULL) { - device_printf(dev, "Unable to allocate tx_buffer memory\n"); - error = ENOMEM; - goto fail; - } - - /* Create the descriptor buffer dma maps */ - for (int i = 0; i < adapter->num_tx_desc; i++) { - tx_buffer = &adapter->tx_buffer_area[i]; - error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); - if (error != 0) { - device_printf(dev, "Unable to create TX DMA map\n"); - goto fail; - } - tx_buffer->next_eop = -1; - } - - return (0); -fail: - lem_free_transmit_structures(adapter); - return (error); -} - -/********************************************************************* - * - * (Re)Initialize transmit structures. - * - **********************************************************************/ -static void -lem_setup_transmit_structures(struct adapter *adapter) -{ - struct em_buffer *tx_buffer; -#ifdef DEV_NETMAP - /* we are already locked */ - struct netmap_adapter *na = netmap_getna(adapter->ifp); - struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); -#endif /* DEV_NETMAP */ - - /* Clear the old ring contents */ - bzero(adapter->tx_desc_base, - (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); - - /* Free any existing TX buffers */ - for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { - tx_buffer = &adapter->tx_buffer_area[i]; - bus_dmamap_sync(adapter->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; -#ifdef DEV_NETMAP - if (slot) { - /* the i-th NIC entry goes to slot si */ - int si = netmap_idx_n2k(&na->tx_rings[0], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - adapter->tx_desc_base[i].buffer_addr = htole64(paddr); - /* reload the map for netmap mode */ - netmap_load_map(na, adapter->txtag, tx_buffer->map, addr); - } -#endif /* DEV_NETMAP */ - tx_buffer->next_eop = -1; - } - - /* Reset state */ - adapter->last_hw_offload = 0; - adapter->next_avail_tx_desc = 0; - adapter->next_tx_to_clean = 0; - adapter->num_tx_desc_avail = adapter->num_tx_desc; - - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - return; -} - -/********************************************************************* - * - * Enable transmit unit. - * - **********************************************************************/ -static void -lem_initialize_transmit_unit(struct adapter *adapter) -{ - u32 tctl, tipg = 0; - u64 bus_addr; - - INIT_DEBUGOUT("lem_initialize_transmit_unit: begin"); - /* Setup the Base and Length of the Tx Descriptor Ring */ - bus_addr = adapter->txdma.dma_paddr; - E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), - adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); - E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), - (u32)(bus_addr >> 32)); - E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), - (u32)bus_addr); - /* Setup the HW Tx Head and Tail descriptor pointers */ - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); - E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); - - HW_DEBUGOUT2("Base = %x, Length = %x\n", - E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), - E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); - - /* Set the default values for the Tx Inter Packet Gap timer */ - switch (adapter->hw.mac.type) { - case e1000_82542: - tipg = DEFAULT_82542_TIPG_IPGT; - tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; - tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; - break; - default: - if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || - (adapter->hw.phy.media_type == - e1000_media_type_internal_serdes)) - tipg = DEFAULT_82543_TIPG_IPGT_FIBER; - else - tipg = DEFAULT_82543_TIPG_IPGT_COPPER; - tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; - tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; - } - - E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); - E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); - if(adapter->hw.mac.type >= e1000_82540) - E1000_WRITE_REG(&adapter->hw, E1000_TADV, - adapter->tx_abs_int_delay.value); - - /* Program the Transmit Control Register */ - tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); - tctl &= ~E1000_TCTL_CT; - tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | - (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); - - /* This write will effectively turn on the transmit unit. */ - E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); - - /* Setup Transmit Descriptor Base Settings */ - adapter->txd_cmd = E1000_TXD_CMD_IFCS; - - if (adapter->tx_int_delay.value > 0) - adapter->txd_cmd |= E1000_TXD_CMD_IDE; -} - -/********************************************************************* - * - * Free all transmit related data structures. - * - **********************************************************************/ -static void -lem_free_transmit_structures(struct adapter *adapter) -{ - struct em_buffer *tx_buffer; - - INIT_DEBUGOUT("free_transmit_structures: begin"); - - if (adapter->tx_buffer_area != NULL) { - for (int i = 0; i < adapter->num_tx_desc; i++) { - tx_buffer = &adapter->tx_buffer_area[i]; - if (tx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->txtag, tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } else if (tx_buffer->map != NULL) - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - if (tx_buffer->map != NULL) { - bus_dmamap_destroy(adapter->txtag, - tx_buffer->map); - tx_buffer->map = NULL; - } - } - } - if (adapter->tx_buffer_area != NULL) { - free(adapter->tx_buffer_area, M_DEVBUF); - adapter->tx_buffer_area = NULL; - } - if (adapter->txtag != NULL) { - bus_dma_tag_destroy(adapter->txtag); - adapter->txtag = NULL; - } -} - -/********************************************************************* - * - * The offload context needs to be set when we transfer the first - * packet of a particular protocol (TCP/UDP). This routine has been - * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) - * - * Added back the old method of keeping the current context type - * and not setting if unnecessary, as this is reported to be a - * big performance win. -jfv - **********************************************************************/ -static void -lem_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, - u32 *txd_upper, u32 *txd_lower) -{ - struct e1000_context_desc *TXD = NULL; - struct em_buffer *tx_buffer; - struct ether_vlan_header *eh; - struct ip *ip = NULL; - struct ip6_hdr *ip6; - int curr_txd, ehdrlen; - u32 cmd, hdr_len, ip_hlen; - u16 etype; - u8 ipproto; - - - cmd = hdr_len = ipproto = 0; - *txd_upper = *txd_lower = 0; - curr_txd = adapter->next_avail_tx_desc; - - /* - * Determine where frame payload starts. - * Jump over vlan headers if already present, - * helpful for QinQ too. - */ - eh = mtod(mp, struct ether_vlan_header *); - if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { - etype = ntohs(eh->evl_proto); - ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; - } else { - etype = ntohs(eh->evl_encap_proto); - ehdrlen = ETHER_HDR_LEN; - } - - /* - * We only support TCP/UDP for IPv4 and IPv6 for the moment. - * TODO: Support SCTP too when it hits the tree. - */ - switch (etype) { - case ETHERTYPE_IP: - ip = (struct ip *)(mp->m_data + ehdrlen); - ip_hlen = ip->ip_hl << 2; - - /* Setup of IP header checksum. */ - if (mp->m_pkthdr.csum_flags & CSUM_IP) { - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->lower_setup.ip_fields.ipcss = ehdrlen; - TXD->lower_setup.ip_fields.ipcse = - htole16(ehdrlen + ip_hlen); - TXD->lower_setup.ip_fields.ipcso = - ehdrlen + offsetof(struct ip, ip_sum); - cmd |= E1000_TXD_CMD_IP; - *txd_upper |= E1000_TXD_POPTS_IXSM << 8; - } - - hdr_len = ehdrlen + ip_hlen; - ipproto = ip->ip_p; - - break; - case ETHERTYPE_IPV6: - ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); - ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ - - /* IPv6 doesn't have a header checksum. */ - - hdr_len = ehdrlen + ip_hlen; - ipproto = ip6->ip6_nxt; - break; - - default: - return; - } - - switch (ipproto) { - case IPPROTO_TCP: - if (mp->m_pkthdr.csum_flags & CSUM_TCP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - /* no need for context if already set */ - if (adapter->last_hw_offload == CSUM_TCP) - return; - adapter->last_hw_offload = CSUM_TCP; - /* - * Start offset for payload checksum calculation. - * End offset for payload checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = - hdr_len + offsetof(struct tcphdr, th_sum); - cmd |= E1000_TXD_CMD_TCP; - } - break; - case IPPROTO_UDP: - { - if (mp->m_pkthdr.csum_flags & CSUM_UDP) { - *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; - *txd_upper |= E1000_TXD_POPTS_TXSM << 8; - /* no need for context if already set */ - if (adapter->last_hw_offload == CSUM_UDP) - return; - adapter->last_hw_offload = CSUM_UDP; - /* - * Start offset for header checksum calculation. - * End offset for header checksum calculation. - * Offset of place to put the checksum. - */ - TXD = (struct e1000_context_desc *) - &adapter->tx_desc_base[curr_txd]; - TXD->upper_setup.tcp_fields.tucss = hdr_len; - TXD->upper_setup.tcp_fields.tucse = htole16(0); - TXD->upper_setup.tcp_fields.tucso = - hdr_len + offsetof(struct udphdr, uh_sum); - } - /* Fall Thru */ - } - default: - break; - } - - if (TXD == NULL) - return; - TXD->tcp_seg_setup.data = htole32(0); - TXD->cmd_and_length = - htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); - tx_buffer = &adapter->tx_buffer_area[curr_txd]; - tx_buffer->m_head = NULL; - tx_buffer->next_eop = -1; - - if (++curr_txd == adapter->num_tx_desc) - curr_txd = 0; - - adapter->num_tx_desc_avail--; - adapter->next_avail_tx_desc = curr_txd; -} - - -/********************************************************************** - * - * Examine each tx_buffer in the used queue. If the hardware is done - * processing the packet then free associated resources. The - * tx_buffer is put back on the free queue. - * - **********************************************************************/ -static void -lem_txeof(struct adapter *adapter) -{ - int first, last, done, num_avail; - struct em_buffer *tx_buffer; - struct e1000_tx_desc *tx_desc, *eop_desc; - if_t ifp = adapter->ifp; - - EM_TX_LOCK_ASSERT(adapter); - -#ifdef DEV_NETMAP - if (netmap_tx_irq(ifp, 0)) - return; -#endif /* DEV_NETMAP */ - if (adapter->num_tx_desc_avail == adapter->num_tx_desc) - return; - - num_avail = adapter->num_tx_desc_avail; - first = adapter->next_tx_to_clean; - tx_desc = &adapter->tx_desc_base[first]; - tx_buffer = &adapter->tx_buffer_area[first]; - last = tx_buffer->next_eop; - eop_desc = &adapter->tx_desc_base[last]; - - /* - * What this does is get the index of the - * first descriptor AFTER the EOP of the - * first packet, that way we can do the - * simple comparison on the inner while loop. - */ - if (++last == adapter->num_tx_desc) - last = 0; - done = last; - - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_POSTREAD); - - while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { - /* We clean the range of the packet */ - while (first != done) { - tx_desc->upper.data = 0; - tx_desc->lower.data = 0; - tx_desc->buffer_addr = 0; - ++num_avail; - - if (tx_buffer->m_head) { - if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); - bus_dmamap_sync(adapter->txtag, - tx_buffer->map, - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(adapter->txtag, - tx_buffer->map); - - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } - tx_buffer->next_eop = -1; - adapter->watchdog_time = ticks; - - if (++first == adapter->num_tx_desc) - first = 0; - - tx_buffer = &adapter->tx_buffer_area[first]; - tx_desc = &adapter->tx_desc_base[first]; - } - /* See if we can continue to the next packet */ - last = tx_buffer->next_eop; - if (last != -1) { - eop_desc = &adapter->tx_desc_base[last]; - /* Get new done point */ - if (++last == adapter->num_tx_desc) last = 0; - done = last; - } else - break; - } - bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - adapter->next_tx_to_clean = first; - adapter->num_tx_desc_avail = num_avail; - -#ifdef NIC_SEND_COMBINING - if ((adapter->shadow_tdt & MIT_PENDING_TDT) == MIT_PENDING_TDT) { - /* a tdt write is pending, do it */ - E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), - 0xffff & adapter->shadow_tdt); - adapter->shadow_tdt = MIT_PENDING_INT; - } else { - adapter->shadow_tdt = 0; // disable - } -#endif /* NIC_SEND_COMBINING */ - /* - * If we have enough room, clear IFF_DRV_OACTIVE to - * tell the stack that it is OK to send packets. - * If there are no pending descriptors, clear the watchdog. - */ - if (adapter->num_tx_desc_avail > EM_TX_CLEANUP_THRESHOLD) { - if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); - if (adapter->num_tx_desc_avail == adapter->num_tx_desc) { - adapter->watchdog_check = FALSE; - return; - } - } -} - -/********************************************************************* - * - * When Link is lost sometimes there is work still in the TX ring - * which may result in a watchdog, rather than allow that we do an - * attempted cleanup and then reinit here. Note that this has been - * seens mostly with fiber adapters. - * - **********************************************************************/ -static void -lem_tx_purge(struct adapter *adapter) -{ - if ((!adapter->link_active) && (adapter->watchdog_check)) { - EM_TX_LOCK(adapter); - lem_txeof(adapter); - EM_TX_UNLOCK(adapter); - if (adapter->watchdog_check) /* Still outstanding? */ - lem_init_locked(adapter); - } -} - -/********************************************************************* - * - * Get a buffer from system mbuf buffer pool. - * - **********************************************************************/ -static int -lem_get_buf(struct adapter *adapter, int i) -{ - struct mbuf *m; - bus_dma_segment_t segs[1]; - bus_dmamap_t map; - struct em_buffer *rx_buffer; - int error, nsegs; - - m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); - if (m == NULL) { - adapter->mbuf_cluster_failed++; - return (ENOBUFS); - } - m->m_len = m->m_pkthdr.len = MCLBYTES; - - if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) - m_adj(m, ETHER_ALIGN); - - /* - * Using memory from the mbuf cluster pool, invoke the - * bus_dma machinery to arrange the memory mapping. - */ - error = bus_dmamap_load_mbuf_sg(adapter->rxtag, - adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); - if (error != 0) { - m_free(m); - return (error); - } - - /* If nsegs is wrong then the stack is corrupt. */ - KASSERT(nsegs == 1, ("Too many segments returned!")); - - rx_buffer = &adapter->rx_buffer_area[i]; - if (rx_buffer->m_head != NULL) - bus_dmamap_unload(adapter->rxtag, rx_buffer->map); - - map = rx_buffer->map; - rx_buffer->map = adapter->rx_sparemap; - adapter->rx_sparemap = map; - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); - rx_buffer->m_head = m; - - adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); - return (0); -} - -/********************************************************************* - * - * Allocate memory for rx_buffer structures. Since we use one - * rx_buffer per received packet, the maximum number of rx_buffer's - * that we'll need is equal to the number of receive descriptors - * that we've allocated. - * - **********************************************************************/ -static int -lem_allocate_receive_structures(struct adapter *adapter) -{ - device_t dev = adapter->dev; - struct em_buffer *rx_buffer; - int i, error; - - adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * - adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); - if (adapter->rx_buffer_area == NULL) { - device_printf(dev, "Unable to allocate rx_buffer memory\n"); - return (ENOMEM); - } - - error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ - 1, 0, /* alignment, bounds */ - BUS_SPACE_MAXADDR, /* lowaddr */ - BUS_SPACE_MAXADDR, /* highaddr */ - NULL, NULL, /* filter, filterarg */ - MCLBYTES, /* maxsize */ - 1, /* nsegments */ - MCLBYTES, /* maxsegsize */ - 0, /* flags */ - NULL, /* lockfunc */ - NULL, /* lockarg */ - &adapter->rxtag); - if (error) { - device_printf(dev, "%s: bus_dma_tag_create failed %d\n", - __func__, error); - goto fail; - } - - /* Create the spare map (used by getbuf) */ - error = bus_dmamap_create(adapter->rxtag, 0, &adapter->rx_sparemap); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - error = bus_dmamap_create(adapter->rxtag, 0, &rx_buffer->map); - if (error) { - device_printf(dev, "%s: bus_dmamap_create failed: %d\n", - __func__, error); - goto fail; - } - } - - return (0); - -fail: - lem_free_receive_structures(adapter); - return (error); -} - -/********************************************************************* - * - * (Re)initialize receive structures. - * - **********************************************************************/ -static int -lem_setup_receive_structures(struct adapter *adapter) -{ - struct em_buffer *rx_buffer; - int i, error; -#ifdef DEV_NETMAP - /* we are already under lock */ - struct netmap_adapter *na = netmap_getna(adapter->ifp); - struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); -#endif - - /* Reset descriptor ring */ - bzero(adapter->rx_desc_base, - (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); - - /* Free current RX buffers. */ - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - if (rx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(adapter->rxtag, rx_buffer->map); - m_freem(rx_buffer->m_head); - rx_buffer->m_head = NULL; - } - } - - /* Allocate new ones. */ - for (i = 0; i < adapter->num_rx_desc; i++) { -#ifdef DEV_NETMAP - if (slot) { - /* the i-th NIC entry goes to slot si */ - int si = netmap_idx_n2k(&na->rx_rings[0], i); - uint64_t paddr; - void *addr; - - addr = PNMB(na, slot + si, &paddr); - netmap_load_map(na, adapter->rxtag, rx_buffer->map, addr); - /* Update descriptor */ - adapter->rx_desc_base[i].buffer_addr = htole64(paddr); - continue; - } -#endif /* DEV_NETMAP */ - error = lem_get_buf(adapter, i); - if (error) - return (error); - } - - /* Setup our descriptor pointers */ - adapter->next_rx_desc_to_check = 0; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - return (0); -} - -/********************************************************************* - * - * Enable receive unit. - * - **********************************************************************/ - -static void -lem_initialize_receive_unit(struct adapter *adapter) -{ - if_t ifp = adapter->ifp; - u64 bus_addr; - u32 rctl, rxcsum; - - INIT_DEBUGOUT("lem_initialize_receive_unit: begin"); - - /* - * Make sure receives are disabled while setting - * up the descriptor ring - */ - rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); - - if (adapter->hw.mac.type >= e1000_82540) { - E1000_WRITE_REG(&adapter->hw, E1000_RADV, - adapter->rx_abs_int_delay.value); - /* - * Set the interrupt throttling rate. Value is calculated - * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) - */ - E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); - } - - /* Setup the Base and Length of the Rx Descriptor Ring */ - bus_addr = adapter->rxdma.dma_paddr; - E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), - adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); - E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), - (u32)(bus_addr >> 32)); - E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), - (u32)bus_addr); - - /* Setup the Receive Control Register */ - rctl &= ~(3 << E1000_RCTL_MO_SHIFT); - rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | - E1000_RCTL_RDMTS_HALF | - (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); - - /* Make sure VLAN Filters are off */ - rctl &= ~E1000_RCTL_VFE; - - if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) - rctl |= E1000_RCTL_SBP; - else - rctl &= ~E1000_RCTL_SBP; - - switch (adapter->rx_buffer_len) { - default: - case 2048: - rctl |= E1000_RCTL_SZ_2048; - break; - case 4096: - rctl |= E1000_RCTL_SZ_4096 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - case 8192: - rctl |= E1000_RCTL_SZ_8192 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - case 16384: - rctl |= E1000_RCTL_SZ_16384 | - E1000_RCTL_BSEX | E1000_RCTL_LPE; - break; - } - - if (if_getmtu(ifp) > ETHERMTU) - rctl |= E1000_RCTL_LPE; - else - rctl &= ~E1000_RCTL_LPE; - - /* Enable 82543 Receive Checksum Offload for TCP and UDP */ - if ((adapter->hw.mac.type >= e1000_82543) && - (if_getcapenable(ifp) & IFCAP_RXCSUM)) { - rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); - rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); - E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); - } - - /* Enable Receives */ - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); - - /* - * Setup the HW Rx Head and - * Tail Descriptor Pointers - */ - E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); - rctl = adapter->num_rx_desc - 1; /* default RDT value */ -#ifdef DEV_NETMAP - /* preserve buffers already made available to clients */ - if (if_getcapenable(ifp) & IFCAP_NETMAP) { - struct netmap_adapter *na = netmap_getna(adapter->ifp); - rctl -= nm_kr_rxspace(&na->rx_rings[0]); - } -#endif /* DEV_NETMAP */ - E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), rctl); - - return; -} - -/********************************************************************* - * - * Free receive related data structures. - * - **********************************************************************/ -static void -lem_free_receive_structures(struct adapter *adapter) -{ - struct em_buffer *rx_buffer; - int i; - - INIT_DEBUGOUT("free_receive_structures: begin"); - - if (adapter->rx_sparemap) { - bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); - adapter->rx_sparemap = NULL; - } - - /* Cleanup any existing buffers */ - if (adapter->rx_buffer_area != NULL) { - rx_buffer = adapter->rx_buffer_area; - for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { - if (rx_buffer->m_head != NULL) { - bus_dmamap_sync(adapter->rxtag, rx_buffer->map, - BUS_DMASYNC_POSTREAD); - bus_dmamap_unload(adapter->rxtag, - rx_buffer->map); - m_freem(rx_buffer->m_head); - rx_buffer->m_head = NULL; - } else if (rx_buffer->map != NULL) - bus_dmamap_unload(adapter->rxtag, - rx_buffer->map); - if (rx_buffer->map != NULL) { - bus_dmamap_destroy(adapter->rxtag, - rx_buffer->map); - rx_buffer->map = NULL; - } - } - } - - if (adapter->rx_buffer_area != NULL) { - free(adapter->rx_buffer_area, M_DEVBUF); - adapter->rx_buffer_area = NULL; - } - - if (adapter->rxtag != NULL) { - bus_dma_tag_destroy(adapter->rxtag); - adapter->rxtag = NULL; - } -} - -/********************************************************************* - * - * This routine executes in interrupt context. It replenishes - * the mbufs in the descriptor and sends data which has been - * dma'ed into host memory to upper layer. - * - * We loop at most count times if count is > 0, or until done if - * count < 0. - * - * For polling we also now return the number of cleaned packets - *********************************************************************/ -static bool -lem_rxeof(struct adapter *adapter, int count, int *done) -{ - if_t ifp = adapter->ifp; - struct mbuf *mp; - u8 status = 0, accept_frame = 0, eop = 0; - u16 len, desc_len, prev_len_adj; - int i, rx_sent = 0; - struct e1000_rx_desc *current_desc; - -#ifdef BATCH_DISPATCH - struct mbuf *mh = NULL, *mt = NULL; -#endif /* BATCH_DISPATCH */ - EM_RX_LOCK(adapter); - -#ifdef BATCH_DISPATCH - batch_again: -#endif /* BATCH_DISPATCH */ - i = adapter->next_rx_desc_to_check; - current_desc = &adapter->rx_desc_base[i]; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_POSTREAD); - -#ifdef DEV_NETMAP - if (netmap_rx_irq(ifp, 0, &rx_sent)) { - EM_RX_UNLOCK(adapter); - return (FALSE); - } -#endif /* DEV_NETMAP */ - - if (!((current_desc->status) & E1000_RXD_STAT_DD)) { - if (done != NULL) - *done = rx_sent; - EM_RX_UNLOCK(adapter); - return (FALSE); - } - - while (count != 0 && if_getdrvflags(ifp) & IFF_DRV_RUNNING) { - struct mbuf *m = NULL; - - status = current_desc->status; - if ((status & E1000_RXD_STAT_DD) == 0) { - break; - } - - mp = adapter->rx_buffer_area[i].m_head; - /* - * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT - * needs to access the last received byte in the mbuf. - */ - bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, - BUS_DMASYNC_POSTREAD); - - accept_frame = 1; - prev_len_adj = 0; - desc_len = le16toh(current_desc->length); - if (status & E1000_RXD_STAT_EOP) { - count--; - eop = 1; - if (desc_len < ETHER_CRC_LEN) { - len = 0; - prev_len_adj = ETHER_CRC_LEN - desc_len; - } else - len = desc_len - ETHER_CRC_LEN; - } else { - eop = 0; - len = desc_len; - } - - if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { - u8 last_byte; - u32 pkt_len = desc_len; - - if (adapter->fmp != NULL) - pkt_len += adapter->fmp->m_pkthdr.len; - - last_byte = *(mtod(mp, caddr_t) + desc_len - 1); - if (TBI_ACCEPT(&adapter->hw, status, - current_desc->errors, pkt_len, last_byte, - adapter->min_frame_size, adapter->max_frame_size)) { - e1000_tbi_adjust_stats_82543(&adapter->hw, - &adapter->stats, pkt_len, - adapter->hw.mac.addr, - adapter->max_frame_size); - if (len > 0) - len--; - } else - accept_frame = 0; - } - - if (accept_frame) { - if (lem_get_buf(adapter, i) != 0) { - if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); - goto discard; - } - - /* Assign correct length to the current fragment */ - mp->m_len = len; - - if (adapter->fmp == NULL) { - mp->m_pkthdr.len = len; - adapter->fmp = mp; /* Store the first mbuf */ - adapter->lmp = mp; - } else { - /* Chain mbuf's together */ - mp->m_flags &= ~M_PKTHDR; - /* - * Adjust length of previous mbuf in chain if - * we received less than 4 bytes in the last - * descriptor. - */ - if (prev_len_adj > 0) { - adapter->lmp->m_len -= prev_len_adj; - adapter->fmp->m_pkthdr.len -= - prev_len_adj; - } - adapter->lmp->m_next = mp; - adapter->lmp = adapter->lmp->m_next; - adapter->fmp->m_pkthdr.len += len; - } - - if (eop) { - if_setrcvif(adapter->fmp, ifp); - if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); - lem_receive_checksum(adapter, current_desc, - adapter->fmp); -#ifndef __NO_STRICT_ALIGNMENT - if (adapter->max_frame_size > - (MCLBYTES - ETHER_ALIGN) && - lem_fixup_rx(adapter) != 0) - goto skip; -#endif - if (status & E1000_RXD_STAT_VP) { - adapter->fmp->m_pkthdr.ether_vtag = - le16toh(current_desc->special); - adapter->fmp->m_flags |= M_VLANTAG; - } -#ifndef __NO_STRICT_ALIGNMENT -skip: -#endif - m = adapter->fmp; - adapter->fmp = NULL; - adapter->lmp = NULL; - } - } else { - adapter->dropped_pkts++; -discard: - /* Reuse loaded DMA map and just update mbuf chain */ - mp = adapter->rx_buffer_area[i].m_head; - mp->m_len = mp->m_pkthdr.len = MCLBYTES; - mp->m_data = mp->m_ext.ext_buf; - mp->m_next = NULL; - if (adapter->max_frame_size <= - (MCLBYTES - ETHER_ALIGN)) - m_adj(mp, ETHER_ALIGN); - if (adapter->fmp != NULL) { - m_freem(adapter->fmp); - adapter->fmp = NULL; - adapter->lmp = NULL; - } - m = NULL; - } - - /* Zero out the receive descriptors status. */ - current_desc->status = 0; - bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, - BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - - /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) - i = 0; - /* Call into the stack */ - if (m != NULL) { -#ifdef BATCH_DISPATCH - if (adapter->batch_enable) { - if (mh == NULL) - mh = mt = m; - else - mt->m_nextpkt = m; - mt = m; - m->m_nextpkt = NULL; - rx_sent++; - current_desc = &adapter->rx_desc_base[i]; - continue; - } -#endif /* BATCH_DISPATCH */ - adapter->next_rx_desc_to_check = i; - EM_RX_UNLOCK(adapter); - if_input(ifp, m); - EM_RX_LOCK(adapter); - rx_sent++; - i = adapter->next_rx_desc_to_check; - } - current_desc = &adapter->rx_desc_base[i]; - } - adapter->next_rx_desc_to_check = i; -#ifdef BATCH_DISPATCH - if (mh) { - EM_RX_UNLOCK(adapter); - while ( (mt = mh) != NULL) { - mh = mh->m_nextpkt; - mt->m_nextpkt = NULL; - if_input(ifp, mt); - } - EM_RX_LOCK(adapter); - i = adapter->next_rx_desc_to_check; /* in case of interrupts */ - if (count > 0) - goto batch_again; - } -#endif /* BATCH_DISPATCH */ - - /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ - if (--i < 0) - i = adapter->num_rx_desc - 1; - E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); - if (done != NULL) - *done = rx_sent; - EM_RX_UNLOCK(adapter); - return ((status & E1000_RXD_STAT_DD) ? TRUE : FALSE); -} - -#ifndef __NO_STRICT_ALIGNMENT -/* - * When jumbo frames are enabled we should realign entire payload on - * architecures with strict alignment. This is serious design mistake of 8254x - * as it nullifies DMA operations. 8254x just allows RX buffer size to be - * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its - * payload. On architecures without strict alignment restrictions 8254x still - * performs unaligned memory access which would reduce the performance too. - * To avoid copying over an entire frame to align, we allocate a new mbuf and - * copy ethernet header to the new mbuf. The new mbuf is prepended into the - * existing mbuf chain. - * - * Be aware, best performance of the 8254x is achieved only when jumbo frame is - * not used at all on architectures with strict alignment. - */ -static int -lem_fixup_rx(struct adapter *adapter) -{ - struct mbuf *m, *n; - int error; - - error = 0; - m = adapter->fmp; - if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { - bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); - m->m_data += ETHER_HDR_LEN; - } else { - MGETHDR(n, M_NOWAIT, MT_DATA); - if (n != NULL) { - bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); - m->m_data += ETHER_HDR_LEN; - m->m_len -= ETHER_HDR_LEN; - n->m_len = ETHER_HDR_LEN; - M_MOVE_PKTHDR(n, m); - n->m_next = m; - adapter->fmp = n; - } else { - adapter->dropped_pkts++; - m_freem(adapter->fmp); - adapter->fmp = NULL; - error = ENOMEM; - } - } - - return (error); -} -#endif - -/********************************************************************* - * - * Verify that the hardware indicated that the checksum is valid. - * Inform the stack about the status of checksum so that stack - * doesn't spend time verifying the checksum. - * - *********************************************************************/ -static void -lem_receive_checksum(struct adapter *adapter, - struct e1000_rx_desc *rx_desc, struct mbuf *mp) -{ - /* 82543 or newer only */ - if ((adapter->hw.mac.type < e1000_82543) || - /* Ignore Checksum bit is set */ - (rx_desc->status & E1000_RXD_STAT_IXSM)) { - mp->m_pkthdr.csum_flags = 0; - return; - } - - if (rx_desc->status & E1000_RXD_STAT_IPCS) { - /* Did it pass? */ - if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { - /* IP Checksum Good */ - mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; - mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; - - } else { - mp->m_pkthdr.csum_flags = 0; - } - } - - if (rx_desc->status & E1000_RXD_STAT_TCPCS) { - /* Did it pass? */ - if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { - mp->m_pkthdr.csum_flags |= - (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); - mp->m_pkthdr.csum_data = htons(0xffff); - } - } -} - -/* - * This routine is run via an vlan - * config EVENT - */ -static void -lem_register_vlan(void *arg, if_t ifp, u16 vtag) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (if_getsoftc(ifp) != arg) /* Not our event */ - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ - return; - - EM_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] |= (1 << bit); - ++adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - -/* - * This routine is run via an vlan - * unconfig EVENT - */ -static void -lem_unregister_vlan(void *arg, if_t ifp, u16 vtag) -{ - struct adapter *adapter = if_getsoftc(ifp); - u32 index, bit; - - if (if_getsoftc(ifp) != arg) - return; - - if ((vtag == 0) || (vtag > 4095)) /* Invalid */ - return; - - EM_CORE_LOCK(adapter); - index = (vtag >> 5) & 0x7F; - bit = vtag & 0x1F; - adapter->shadow_vfta[index] &= ~(1 << bit); - --adapter->num_vlans; - /* Re-init to load the changes */ - if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER) - lem_init_locked(adapter); - EM_CORE_UNLOCK(adapter); -} - -static void -lem_setup_vlan_hw_support(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 reg; - - /* - ** We get here thru init_locked, meaning - ** a soft reset, this has already cleared - ** the VFTA and other state, so if there - ** have been no vlan's registered do nothing. - */ - if (adapter->num_vlans == 0) - return; - - /* - ** A soft reset zero's out the VFTA, so - ** we need to repopulate it now. - */ - for (int i = 0; i < EM_VFTA_SIZE; i++) - if (adapter->shadow_vfta[i] != 0) - E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, - i, adapter->shadow_vfta[i]); - - reg = E1000_READ_REG(hw, E1000_CTRL); - reg |= E1000_CTRL_VME; - E1000_WRITE_REG(hw, E1000_CTRL, reg); - - /* Enable the Filter Table */ - reg = E1000_READ_REG(hw, E1000_RCTL); - reg &= ~E1000_RCTL_CFIEN; - reg |= E1000_RCTL_VFE; - E1000_WRITE_REG(hw, E1000_RCTL, reg); -} - -static void -lem_enable_intr(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 ims_mask = IMS_ENABLE_MASK; - - E1000_WRITE_REG(hw, E1000_IMS, ims_mask); -} - -static void -lem_disable_intr(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - - E1000_WRITE_REG(hw, E1000_IMC, 0xffffffff); -} - -/* - * Bit of a misnomer, what this really means is - * to enable OS management of the system... aka - * to disable special hardware management features - */ -static void -lem_init_manageability(struct adapter *adapter) -{ - /* A shared code workaround */ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - /* disable hardware interception of ARP */ - manc &= ~(E1000_MANC_ARP_EN); - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * Give control back to hardware management - * controller if there is one. - */ -static void -lem_release_manageability(struct adapter *adapter) -{ - if (adapter->has_manage) { - int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); - - /* re-enable hardware interception of ARP */ - manc |= E1000_MANC_ARP_EN; - E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); - } -} - -/* - * lem_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means - * that the driver is loaded. For AMT version type f/w - * this means that the network i/f is open. - */ -static void -lem_get_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); - return; -} - -/* - * lem_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. - * For ASF and Pass Through versions of f/w this means that - * the driver is no longer loaded. For AMT versions of the - * f/w this means that the network i/f is closed. - */ -static void -lem_release_hw_control(struct adapter *adapter) -{ - u32 ctrl_ext; - - if (!adapter->has_manage) - return; - - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, - ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); - return; -} - -static int -lem_is_valid_ether_addr(u8 *addr) -{ - char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; - - if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { - return (FALSE); - } - - return (TRUE); -} - -/* -** Parse the interface capabilities with regard -** to both system management and wake-on-lan for -** later use. -*/ -static void -lem_get_wakeup(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - u16 eeprom_data = 0, device_id, apme_mask; - - adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); - apme_mask = EM_EEPROM_APME; - - switch (adapter->hw.mac.type) { - case e1000_82542: - case e1000_82543: - break; - case e1000_82544: - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL2_REG, 1, &eeprom_data); - apme_mask = EM_82544_APME; - break; - case e1000_82546: - case e1000_82546_rev_3: - if (adapter->hw.bus.func == 1) { - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); - break; - } else - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); - break; - default: - e1000_read_nvm(&adapter->hw, - NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); - break; - } - if (eeprom_data & apme_mask) - adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); - /* - * We have the eeprom settings, now apply the special cases - * where the eeprom may be wrong or the board won't support - * wake on lan on a particular port - */ - device_id = pci_get_device(dev); - switch (device_id) { - case E1000_DEV_ID_82546GB_PCIE: - adapter->wol = 0; - break; - case E1000_DEV_ID_82546EB_FIBER: - case E1000_DEV_ID_82546GB_FIBER: - /* Wake events only supported on port A for dual fiber - * regardless of eeprom setting */ - if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & - E1000_STATUS_FUNC_1) - adapter->wol = 0; - break; - case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: - /* if quad port adapter, disable WoL on all but port A */ - if (global_quad_port_a != 0) - adapter->wol = 0; - /* Reset for multiple quad port adapters */ - if (++global_quad_port_a == 4) - global_quad_port_a = 0; - break; - } - return; -} - - -/* - * Enable PCI Wake On Lan capability - */ -static void -lem_enable_wakeup(device_t dev) -{ - struct adapter *adapter = device_get_softc(dev); - if_t ifp = adapter->ifp; - u32 pmc, ctrl, ctrl_ext, rctl; - u16 status; - - if ((pci_find_cap(dev, PCIY_PMG, &pmc) != 0)) - return; - - /* Advertise the wakeup capability */ - ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); - ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); - E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - - /* Keep the laser running on Fiber adapters */ - if (adapter->hw.phy.media_type == e1000_media_type_fiber || - adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { - ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); - ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; - E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); - } - - /* - ** Determine type of Wakeup: note that wol - ** is set with all bits on by default. - */ - if ((if_getcapenable(ifp) & IFCAP_WOL_MAGIC) == 0) - adapter->wol &= ~E1000_WUFC_MAG; - - if ((if_getcapenable(ifp) & IFCAP_WOL_MCAST) == 0) - adapter->wol &= ~E1000_WUFC_MC; - else { - rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); - rctl |= E1000_RCTL_MPE; - E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); - } - - if (adapter->hw.mac.type == e1000_pchlan) { - if (lem_enable_phy_wakeup(adapter)) - return; - } else { - E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); - E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); - } - - - /* Request PME */ - status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); - status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); - if (if_getcapenable(ifp) & IFCAP_WOL) - status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; - pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); - - return; -} - -/* -** WOL in the newer chipset interfaces (pchlan) -** require thing to be copied into the phy -*/ -static int -lem_enable_phy_wakeup(struct adapter *adapter) -{ - struct e1000_hw *hw = &adapter->hw; - u32 mreg, ret = 0; - u16 preg; - - /* copy MAC RARs to PHY RARs */ - for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { - mreg = E1000_READ_REG(hw, E1000_RAL(i)); - e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_RAR_M(i), - (u16)((mreg >> 16) & 0xFFFF)); - mreg = E1000_READ_REG(hw, E1000_RAH(i)); - e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_RAR_CTRL(i), - (u16)((mreg >> 16) & 0xFFFF)); - } - - /* copy MAC MTA to PHY MTA */ - for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { - mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); - e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); - e1000_write_phy_reg(hw, BM_MTA(i) + 1, - (u16)((mreg >> 16) & 0xFFFF)); - } - - /* configure PHY Rx Control register */ - e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); - mreg = E1000_READ_REG(hw, E1000_RCTL); - if (mreg & E1000_RCTL_UPE) - preg |= BM_RCTL_UPE; - if (mreg & E1000_RCTL_MPE) - preg |= BM_RCTL_MPE; - preg &= ~(BM_RCTL_MO_MASK); - if (mreg & E1000_RCTL_MO_3) - preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) - << BM_RCTL_MO_SHIFT); - if (mreg & E1000_RCTL_BAM) - preg |= BM_RCTL_BAM; - if (mreg & E1000_RCTL_PMCF) - preg |= BM_RCTL_PMCF; - mreg = E1000_READ_REG(hw, E1000_CTRL); - if (mreg & E1000_CTRL_RFCE) - preg |= BM_RCTL_RFCE; - e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); - - /* enable PHY wakeup in MAC register */ - E1000_WRITE_REG(hw, E1000_WUC, - E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); - E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); - - /* configure and enable PHY wakeup in PHY registers */ - e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); - e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); - - /* activate PHY wakeup */ - ret = hw->phy.ops.acquire(hw); - if (ret) { - printf("Could not acquire PHY\n"); - return ret; - } - e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, - (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); - ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); - if (ret) { - printf("Could not read PHY page 769\n"); - goto out; - } - preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; - ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); - if (ret) - printf("Could not set PHY Host Wakeup bit\n"); -out: - hw->phy.ops.release(hw); - - return ret; -} - -static void -lem_led_func(void *arg, int onoff) -{ - struct adapter *adapter = arg; - - EM_CORE_LOCK(adapter); - if (onoff) { - e1000_setup_led(&adapter->hw); - e1000_led_on(&adapter->hw); - } else { - e1000_led_off(&adapter->hw); - e1000_cleanup_led(&adapter->hw); - } - EM_CORE_UNLOCK(adapter); -} - -/********************************************************************* -* 82544 Coexistence issue workaround. -* There are 2 issues. -* 1. Transmit Hang issue. -* To detect this issue, following equation can be used... -* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. -* If SUM[3:0] is in between 1 to 4, we will have this issue. -* -* 2. DAC issue. -* To detect this issue, following equation can be used... -* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. -* If SUM[3:0] is in between 9 to c, we will have this issue. -* -* -* WORKAROUND: -* Make sure we do not have ending address -* as 1,2,3,4(Hang) or 9,a,b,c (DAC) -* -*************************************************************************/ -static u32 -lem_fill_descriptors (bus_addr_t address, u32 length, - PDESC_ARRAY desc_array) -{ - u32 safe_terminator; - - /* Since issue is sensitive to length and address.*/ - /* Let us first check the address...*/ - if (length <= 4) { - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length; - desc_array->elements = 1; - return (desc_array->elements); - } - safe_terminator = (u32)((((u32)address & 0x7) + - (length & 0xF)) & 0xF); - /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ - if (safe_terminator == 0 || - (safe_terminator > 4 && - safe_terminator < 9) || - (safe_terminator > 0xC && - safe_terminator <= 0xF)) { - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length; - desc_array->elements = 1; - return (desc_array->elements); - } - - desc_array->descriptor[0].address = address; - desc_array->descriptor[0].length = length - 4; - desc_array->descriptor[1].address = address + (length - 4); - desc_array->descriptor[1].length = 4; - desc_array->elements = 2; - return (desc_array->elements); -} - -/********************************************************************** - * - * Update the board statistics counters. - * - **********************************************************************/ -static void -lem_update_stats_counters(struct adapter *adapter) -{ - - if(adapter->hw.phy.media_type == e1000_media_type_copper || - (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { - adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); - adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); - } - adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); - adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); - adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); - adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); - - adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); - adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); - adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); - adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); - adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); - adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); - adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); - adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); - adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); - adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); - adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); - adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); - adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); - adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); - adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); - adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); - adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); - adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); - adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); - adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); - - /* For the 64-bit byte counters the low dword must be read first. */ - /* Both registers clear on the read of the high dword */ - - adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) + - ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32); - adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) + - ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32); - - adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); - adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); - adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); - adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); - adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); - - adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); - adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); - - adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); - adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); - adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); - adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); - adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); - adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); - adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); - adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); - adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); - adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); - - if (adapter->hw.mac.type >= e1000_82543) { - adapter->stats.algnerrc += - E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); - adapter->stats.rxerrc += - E1000_READ_REG(&adapter->hw, E1000_RXERRC); - adapter->stats.tncrs += - E1000_READ_REG(&adapter->hw, E1000_TNCRS); - adapter->stats.cexterr += - E1000_READ_REG(&adapter->hw, E1000_CEXTERR); - adapter->stats.tsctc += - E1000_READ_REG(&adapter->hw, E1000_TSCTC); - adapter->stats.tsctfc += - E1000_READ_REG(&adapter->hw, E1000_TSCTFC); - } -} - -static uint64_t -lem_get_counter(if_t ifp, ift_counter cnt) -{ - struct adapter *adapter; - - adapter = if_getsoftc(ifp); - - switch (cnt) { - case IFCOUNTER_COLLISIONS: - return (adapter->stats.colc); - case IFCOUNTER_IERRORS: - return (adapter->dropped_pkts + adapter->stats.rxerrc + - adapter->stats.crcerrs + adapter->stats.algnerrc + - adapter->stats.ruc + adapter->stats.roc + - adapter->stats.mpc + adapter->stats.cexterr); - case IFCOUNTER_OERRORS: - return (adapter->stats.ecol + adapter->stats.latecol + - adapter->watchdog_events); - default: - return (if_get_counter_default(ifp, cnt)); - } -} - -/* Export a single 32-bit register via a read-only sysctl. */ -static int -lem_sysctl_reg_handler(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - u_int val; - - adapter = oidp->oid_arg1; - val = E1000_READ_REG(&adapter->hw, oidp->oid_arg2); - return (sysctl_handle_int(oidp, &val, 0, req)); -} - -/* - * Add sysctl variables, one per statistic, to the system. - */ -static void -lem_add_hw_stats(struct adapter *adapter) -{ - device_t dev = adapter->dev; - - struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); - struct sysctl_oid *tree = device_get_sysctl_tree(dev); - struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); - struct e1000_hw_stats *stats = &adapter->stats; - - struct sysctl_oid *stat_node; - struct sysctl_oid_list *stat_list; - - /* Driver Statistics */ - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "cluster_alloc_fail", - CTLFLAG_RD, &adapter->mbuf_cluster_failed, - "Std mbuf cluster failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", - CTLFLAG_RD, &adapter->mbuf_defrag_failed, - "Defragmenting mbuf chain failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", - CTLFLAG_RD, &adapter->dropped_pkts, - "Driver dropped packets"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail1", - CTLFLAG_RD, &adapter->no_tx_desc_avail1, - "Not enough tx descriptors failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_desc_fail2", - CTLFLAG_RD, &adapter->no_tx_desc_avail2, - "Not enough tx descriptors failure in xmit"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_overruns", - CTLFLAG_RD, &adapter->rx_overruns, - "RX overruns"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_timeouts", - CTLFLAG_RD, &adapter->watchdog_events, - "Watchdog timeouts"); - - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "device_control", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_CTRL, - lem_sysctl_reg_handler, "IU", - "Device Control Register"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rx_control", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RCTL, - lem_sysctl_reg_handler, "IU", - "Receiver Control Register"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", - CTLFLAG_RD, &adapter->hw.fc.high_water, 0, - "Flow Control High Watermark"); - SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", - CTLFLAG_RD, &adapter->hw.fc.low_water, 0, - "Flow Control Low Watermark"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_workaround", - CTLFLAG_RD, &adapter->tx_fifo_wrk_cnt, - "TX FIFO workaround events"); - SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "fifo_reset", - CTLFLAG_RD, &adapter->tx_fifo_reset_cnt, - "TX FIFO resets"); - - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDH(0), - lem_sysctl_reg_handler, "IU", - "Transmit Descriptor Head"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "txd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_TDT(0), - lem_sysctl_reg_handler, "IU", - "Transmit Descriptor Tail"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_head", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDH(0), - lem_sysctl_reg_handler, "IU", - "Receive Descriptor Head"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "rxd_tail", - CTLTYPE_UINT | CTLFLAG_RD, adapter, E1000_RDT(0), - lem_sysctl_reg_handler, "IU", - "Receive Descriptor Tail"); - - - /* MAC stats get their own sub node */ - - stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", - CTLFLAG_RD, NULL, "Statistics"); - stat_list = SYSCTL_CHILDREN(stat_node); - - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "excess_coll", - CTLFLAG_RD, &stats->ecol, - "Excessive collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "single_coll", - CTLFLAG_RD, &stats->scc, - "Single collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "multiple_coll", - CTLFLAG_RD, &stats->mcc, - "Multiple collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "late_coll", - CTLFLAG_RD, &stats->latecol, - "Late collisions"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "collision_count", - CTLFLAG_RD, &stats->colc, - "Collision Count"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "symbol_errors", - CTLFLAG_RD, &adapter->stats.symerrs, - "Symbol Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "sequence_errors", - CTLFLAG_RD, &adapter->stats.sec, - "Sequence Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "defer_count", - CTLFLAG_RD, &adapter->stats.dc, - "Defer Count"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "missed_packets", - CTLFLAG_RD, &adapter->stats.mpc, - "Missed Packets"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", - CTLFLAG_RD, &adapter->stats.rnbc, - "Receive No Buffers"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersize", - CTLFLAG_RD, &adapter->stats.ruc, - "Receive Undersize"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", - CTLFLAG_RD, &adapter->stats.rfc, - "Fragmented Packets Received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversize", - CTLFLAG_RD, &adapter->stats.roc, - "Oversized Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabber", - CTLFLAG_RD, &adapter->stats.rjc, - "Recevied Jabber"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_errs", - CTLFLAG_RD, &adapter->stats.rxerrc, - "Receive Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs", - CTLFLAG_RD, &adapter->stats.crcerrs, - "CRC errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "alignment_errs", - CTLFLAG_RD, &adapter->stats.algnerrc, - "Alignment Errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", - CTLFLAG_RD, &adapter->stats.cexterr, - "Collision/Carrier extension errors"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_recvd", - CTLFLAG_RD, &adapter->stats.xonrxc, - "XON Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xon_txd", - CTLFLAG_RD, &adapter->stats.xontxc, - "XON Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", - CTLFLAG_RD, &adapter->stats.xoffrxc, - "XOFF Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "xoff_txd", - CTLFLAG_RD, &adapter->stats.xofftxc, - "XOFF Transmitted"); - - /* Packet Reception Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", - CTLFLAG_RD, &adapter->stats.tpr, - "Total Packets Received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", - CTLFLAG_RD, &adapter->stats.gprc, - "Good Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", - CTLFLAG_RD, &adapter->stats.bprc, - "Broadcast Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", - CTLFLAG_RD, &adapter->stats.mprc, - "Multicast Packets Received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", - CTLFLAG_RD, &adapter->stats.prc64, - "64 byte frames received "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", - CTLFLAG_RD, &adapter->stats.prc127, - "65-127 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", - CTLFLAG_RD, &adapter->stats.prc255, - "128-255 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", - CTLFLAG_RD, &adapter->stats.prc511, - "256-511 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", - CTLFLAG_RD, &adapter->stats.prc1023, - "512-1023 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", - CTLFLAG_RD, &adapter->stats.prc1522, - "1023-1522 byte frames received"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", - CTLFLAG_RD, &adapter->stats.gorc, - "Good Octets Received"); - - /* Packet Transmission Stats */ - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd", - CTLFLAG_RD, &adapter->stats.gotc, - "Good Octets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", - CTLFLAG_RD, &adapter->stats.tpt, - "Total Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", - CTLFLAG_RD, &adapter->stats.gptc, - "Good Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", - CTLFLAG_RD, &adapter->stats.bptc, - "Broadcast Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", - CTLFLAG_RD, &adapter->stats.mptc, - "Multicast Packets Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", - CTLFLAG_RD, &adapter->stats.ptc64, - "64 byte frames transmitted "); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", - CTLFLAG_RD, &adapter->stats.ptc127, - "65-127 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", - CTLFLAG_RD, &adapter->stats.ptc255, - "128-255 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", - CTLFLAG_RD, &adapter->stats.ptc511, - "256-511 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", - CTLFLAG_RD, &adapter->stats.ptc1023, - "512-1023 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", - CTLFLAG_RD, &adapter->stats.ptc1522, - "1024-1522 byte frames transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_txd", - CTLFLAG_RD, &adapter->stats.tsctc, - "TSO Contexts Transmitted"); - SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", - CTLFLAG_RD, &adapter->stats.tsctfc, - "TSO Contexts Failed"); -} - -/********************************************************************** - * - * This routine provides a way to dump out the adapter eeprom, - * often a useful debug/service tool. This only dumps the first - * 32 words, stuff that matters is in that extent. - * - **********************************************************************/ - -static int -lem_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) -{ - struct adapter *adapter; - int error; - int result; - - result = -1; - error = sysctl_handle_int(oidp, &result, 0, req); - - if (error || !req->newptr) - return (error); - - /* - * This value will cause a hex dump of the - * first 32 16-bit words of the EEPROM to - * the screen. - */ - if (result == 1) { - adapter = (struct adapter *)arg1; - lem_print_nvm_info(adapter); - } - - return (error); -} - -static void -lem_print_nvm_info(struct adapter *adapter) -{ - u16 eeprom_data; - int i, j, row = 0; - - /* Its a bit crude, but it gets the job done */ - printf("\nInterface EEPROM Dump:\n"); - printf("Offset\n0x0000 "); - for (i = 0, j = 0; i < 32; i++, j++) { - if (j == 8) { /* Make the offset block */ - j = 0; ++row; - printf("\n0x00%x0 ",row); - } - e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); - printf("%04x ", eeprom_data); - } - printf("\n"); -} - -static int -lem_sysctl_int_delay(SYSCTL_HANDLER_ARGS) -{ - struct em_int_delay_info *info; - struct adapter *adapter; - u32 regval; - int error; - int usecs; - int ticks; - - info = (struct em_int_delay_info *)arg1; - usecs = info->value; - error = sysctl_handle_int(oidp, &usecs, 0, req); - if (error != 0 || req->newptr == NULL) - return (error); - if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) - return (EINVAL); - info->value = usecs; - ticks = EM_USECS_TO_TICKS(usecs); - if (info->offset == E1000_ITR) /* units are 256ns here */ - ticks *= 4; - - adapter = info->adapter; - - EM_CORE_LOCK(adapter); - regval = E1000_READ_OFFSET(&adapter->hw, info->offset); - regval = (regval & ~0xffff) | (ticks & 0xffff); - /* Handle a few special cases. */ - switch (info->offset) { - case E1000_RDTR: - break; - case E1000_TIDV: - if (ticks == 0) { - adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; - /* Don't write 0 into the TIDV register. */ - regval++; - } else - adapter->txd_cmd |= E1000_TXD_CMD_IDE; - break; - } - E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); - EM_CORE_UNLOCK(adapter); - return (0); -} - -static void -lem_add_int_delay_sysctl(struct adapter *adapter, const char *name, - const char *description, struct em_int_delay_info *info, - int offset, int value) -{ - info->adapter = adapter; - info->offset = offset; - info->value = value; - SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, - info, 0, lem_sysctl_int_delay, "I", description); -} - -static void -lem_set_flow_cntrl(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} - -static void -lem_add_rx_process_limit(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) -{ - *limit = value; - SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), - OID_AUTO, name, CTLFLAG_RW, limit, value, description); -} diff --git a/sys/dev/e1000/if_lem.h b/sys/dev/e1000/if_lem.h deleted file mode 100644 index 4a27c34bd772..000000000000 --- a/sys/dev/e1000/if_lem.h +++ /dev/null @@ -1,519 +0,0 @@ -/****************************************************************************** - - Copyright (c) 2001-2015, Intel Corporation - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of the Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -******************************************************************************/ -/*$FreeBSD$*/ - - -#ifndef _LEM_H_DEFINED_ -#define _LEM_H_DEFINED_ - - -/* Tunables */ - -/* - * EM_TXD: Maximum number of Transmit Descriptors - * Valid Range: 80-256 for 82542 and 82543-based adapters - * 80-4096 for others - * Default Value: 256 - * This value is the number of transmit descriptors allocated by the driver. - * Increasing this value allows the driver to queue more transmits. Each - * descriptor is 16 bytes. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define EM_MIN_TXD 80 -#define EM_MAX_TXD_82543 256 -#define EM_MAX_TXD 4096 -#define EM_DEFAULT_TXD EM_MAX_TXD_82543 - -/* - * EM_RXD - Maximum number of receive Descriptors - * Valid Range: 80-256 for 82542 and 82543-based adapters - * 80-4096 for others - * Default Value: 256 - * This value is the number of receive descriptors allocated by the driver. - * Increasing this value allows the driver to buffer more incoming packets. - * Each descriptor is 16 bytes. A receive buffer is also allocated for each - * descriptor. The maximum MTU size is 16110. - * Since TDLEN should be multiple of 128bytes, the number of transmit - * desscriptors should meet the following condition. - * (num_tx_desc * sizeof(struct e1000_tx_desc)) % 128 == 0 - */ -#define EM_MIN_RXD 80 -#define EM_MAX_RXD_82543 256 -#define EM_MAX_RXD 4096 -#define EM_DEFAULT_RXD EM_MAX_RXD_82543 - -/* - * EM_TIDV - Transmit Interrupt Delay Value - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value delays the generation of transmit interrupts in units of - * 1.024 microseconds. Transmit interrupt reduction can improve CPU - * efficiency if properly tuned for specific network traffic. If the - * system is reporting dropped transmits, this value may be set too high - * causing the driver to run out of available transmit descriptors. - */ -#define EM_TIDV 64 - -/* - * EM_TADV - Transmit Absolute Interrupt Delay Value - * (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * transmit interrupt is generated. Useful only if EM_TIDV is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is sent on the wire within the set amount of time. Proper tuning, - * along with EM_TIDV, may improve traffic throughput in specific - * network conditions. - */ -#define EM_TADV 64 - -/* - * EM_RDTR - Receive Interrupt Delay Timer (Packet Timer) - * Valid Range: 0-65535 (0=off) - * Default Value: 0 - * This value delays the generation of receive interrupts in units of 1.024 - * microseconds. Receive interrupt reduction can improve CPU efficiency if - * properly tuned for specific network traffic. Increasing this value adds - * extra latency to frame reception and can end up decreasing the throughput - * of TCP traffic. If the system is reporting dropped receives, this value - * may be set too high, causing the driver to run out of available receive - * descriptors. - * - * CAUTION: When setting EM_RDTR to a value other than 0, adapters - * may hang (stop transmitting) under certain network conditions. - * If this occurs a WATCHDOG message is logged in the system - * event log. In addition, the controller is automatically reset, - * restoring the network connection. To eliminate the potential - * for the hang ensure that EM_RDTR is set to 0. - */ -#define EM_RDTR 0 - -/* - * Receive Interrupt Absolute Delay Timer (Not valid for 82542/82543/82544) - * Valid Range: 0-65535 (0=off) - * Default Value: 64 - * This value, in units of 1.024 microseconds, limits the delay in which a - * receive interrupt is generated. Useful only if EM_RDTR is non-zero, - * this value ensures that an interrupt is generated after the initial - * packet is received within the set amount of time. Proper tuning, - * along with EM_RDTR, may improve traffic throughput in specific network - * conditions. - */ -#define EM_RADV 64 - -/* - * This parameter controls the max duration of transmit watchdog. - */ -#define EM_WATCHDOG (10 * hz) - -/* - * This parameter controls when the driver calls the routine to reclaim - * transmit descriptors. - */ -#define EM_TX_CLEANUP_THRESHOLD (adapter->num_tx_desc / 8) -#define EM_TX_OP_THRESHOLD (adapter->num_tx_desc / 32) - -/* - * This parameter controls whether or not autonegotation is enabled. - * 0 - Disable autonegotiation - * 1 - Enable autonegotiation - */ -#define DO_AUTO_NEG 1 - -/* - * This parameter control whether or not the driver will wait for - * autonegotiation to complete. - * 1 - Wait for autonegotiation to complete - * 0 - Don't wait for autonegotiation to complete - */ -#define WAIT_FOR_AUTO_NEG_DEFAULT 0 - -/* Tunables -- End */ - -#define AUTONEG_ADV_DEFAULT (ADVERTISE_10_HALF | ADVERTISE_10_FULL | \ - ADVERTISE_100_HALF | ADVERTISE_100_FULL | \ - ADVERTISE_1000_FULL) - -#define AUTO_ALL_MODES 0 - -/* PHY master/slave setting */ -#define EM_MASTER_SLAVE e1000_ms_hw_default - -/* - * Micellaneous constants - */ -#define EM_VENDOR_ID 0x8086 -#define EM_FLASH 0x0014 - -#define EM_JUMBO_PBA 0x00000028 -#define EM_DEFAULT_PBA 0x00000030 -#define EM_SMARTSPEED_DOWNSHIFT 3 -#define EM_SMARTSPEED_MAX 15 -#define EM_MAX_LOOP 10 - -#define MAX_NUM_MULTICAST_ADDRESSES 128 -#define PCI_ANY_ID (~0U) -#define ETHER_ALIGN 2 -#define EM_FC_PAUSE_TIME 0x0680 -#define EM_EEPROM_APME 0x400; -#define EM_82544_APME 0x0004; - -/* Code compatilbility between 6 and 7 */ -#ifndef ETHER_BPF_MTAP -#define ETHER_BPF_MTAP BPF_MTAP -#endif - -/* - * TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be - * multiple of 128 bytes. So we align TDBA/RDBA on 128 byte boundary. This will - * also optimize cache line size effect. H/W supports up to cache line size 128. - */ -#define EM_DBA_ALIGN 128 - -#define SPEED_MODE_BIT (1<<21) /* On PCI-E MACs only */ - -/* PCI Config defines */ -#define EM_BAR_TYPE(v) ((v) & EM_BAR_TYPE_MASK) -#define EM_BAR_TYPE_MASK 0x00000001 -#define EM_BAR_TYPE_MMEM 0x00000000 -#define EM_BAR_TYPE_IO 0x00000001 -#define EM_BAR_TYPE_FLASH 0x0014 -#define EM_BAR_MEM_TYPE(v) ((v) & EM_BAR_MEM_TYPE_MASK) -#define EM_BAR_MEM_TYPE_MASK 0x00000006 -#define EM_BAR_MEM_TYPE_32BIT 0x00000000 -#define EM_BAR_MEM_TYPE_64BIT 0x00000004 -#define EM_MSIX_BAR 3 /* On 82575 */ - -#if __FreeBSD_version < 900000 -#define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD -#endif - -/* Defines for printing debug information */ -#define DEBUG_INIT 0 -#define DEBUG_IOCTL 0 -#define DEBUG_HW 0 - -#define INIT_DEBUGOUT(S) if (DEBUG_INIT) printf(S "\n") -#define INIT_DEBUGOUT1(S, A) if (DEBUG_INIT) printf(S "\n", A) -#define INIT_DEBUGOUT2(S, A, B) if (DEBUG_INIT) printf(S "\n", A, B) -#define IOCTL_DEBUGOUT(S) if (DEBUG_IOCTL) printf(S "\n") -#define IOCTL_DEBUGOUT1(S, A) if (DEBUG_IOCTL) printf(S "\n", A) -#define IOCTL_DEBUGOUT2(S, A, B) if (DEBUG_IOCTL) printf(S "\n", A, B) -#define HW_DEBUGOUT(S) if (DEBUG_HW) printf(S "\n") -#define HW_DEBUGOUT1(S, A) if (DEBUG_HW) printf(S "\n", A) -#define HW_DEBUGOUT2(S, A, B) if (DEBUG_HW) printf(S "\n", A, B) - -#define EM_MAX_SCATTER 40 -#define EM_VFTA_SIZE 128 -#define EM_MSIX_MASK 0x01F00000 /* For 82574 use */ -#define ETH_ZLEN 60 -#define ETH_ADDR_LEN 6 -#define CSUM_OFFLOAD 7 /* Offload bits in mbuf flag */ - -/* - * 82574 has a nonstandard address for EIAC - * and since its only used in MSIX, and in - * the em driver only 82574 uses MSIX we can - * solve it just using this define. - */ -#define EM_EIAC 0x000DC - -/* Used in for 82547 10Mb Half workaround */ -#define EM_PBA_BYTES_SHIFT 0xA -#define EM_TX_HEAD_ADDR_SHIFT 7 -#define EM_PBA_TX_MASK 0xFFFF0000 -#define EM_FIFO_HDR 0x10 -#define EM_82547_PKT_THRESH 0x3e0 - -/* Precision Time Sync (IEEE 1588) defines */ -#define ETHERTYPE_IEEE1588 0x88F7 -#define PICOSECS_PER_TICK 20833 -#define TSYNC_PORT 319 /* UDP port for the protocol */ - -#ifdef NIC_PARAVIRT -#define E1000_PARA_SUBDEV 0x1101 /* special id */ -#define E1000_CSBAL 0x02830 /* csb phys. addr. low */ -#define E1000_CSBAH 0x02834 /* csb phys. addr. hi */ -#include -#endif /* NIC_PARAVIRT */ - -/* - * Bus dma allocation structure used by - * e1000_dma_malloc and e1000_dma_free. - */ -struct em_dma_alloc { - bus_addr_t dma_paddr; - caddr_t dma_vaddr; - bus_dma_tag_t dma_tag; - bus_dmamap_t dma_map; - bus_dma_segment_t dma_seg; - int dma_nseg; -}; - -struct adapter; - -struct em_int_delay_info { - struct adapter *adapter; /* Back-pointer to the adapter struct */ - int offset; /* Register offset to read/write */ - int value; /* Current value in usecs */ -}; - -/* Our adapter structure */ -struct adapter { - if_t ifp; - struct e1000_hw hw; - - /* FreeBSD operating-system-specific structures. */ - struct e1000_osdep osdep; - device_t dev; - struct cdev *led_dev; - - struct resource *memory; - struct resource *flash; - struct resource *msix; - - struct resource *ioport; - int io_rid; - - /* 82574 may use 3 int vectors */ - struct resource *res[3]; - void *tag[3]; - int rid[3]; - - struct ifmedia media; - struct callout timer; - struct callout tx_fifo_timer; - bool watchdog_check; - int watchdog_time; - int msi; - int if_flags; - int max_frame_size; - int min_frame_size; - struct mtx core_mtx; - struct mtx tx_mtx; - struct mtx rx_mtx; - int em_insert_vlan_header; - - /* Task for FAST handling */ - struct task link_task; - struct task rxtx_task; - struct task rx_task; - struct task tx_task; - struct taskqueue *tq; /* private task queue */ - - eventhandler_tag vlan_attach; - eventhandler_tag vlan_detach; - u32 num_vlans; - - /* Management and WOL features */ - u32 wol; - bool has_manage; - bool has_amt; - - /* Multicast array memory */ - u8 *mta; - - /* - ** Shadow VFTA table, this is needed because - ** the real vlan filter table gets cleared during - ** a soft reset and the driver needs to be able - ** to repopulate it. - */ - u32 shadow_vfta[EM_VFTA_SIZE]; - - /* Info about the interface */ - uint8_t link_active; - uint16_t link_speed; - uint16_t link_duplex; - uint32_t smartspeed; - uint32_t fc_setting; - - struct em_int_delay_info tx_int_delay; - struct em_int_delay_info tx_abs_int_delay; - struct em_int_delay_info rx_int_delay; - struct em_int_delay_info rx_abs_int_delay; - struct em_int_delay_info tx_itr; - - /* - * Transmit definitions - * - * We have an array of num_tx_desc descriptors (handled - * by the controller) paired with an array of tx_buffers - * (at tx_buffer_area). - * The index of the next available descriptor is next_avail_tx_desc. - * The number of remaining tx_desc is num_tx_desc_avail. - */ - struct em_dma_alloc txdma; /* bus_dma glue for tx desc */ - struct e1000_tx_desc *tx_desc_base; - uint32_t next_avail_tx_desc; - uint32_t next_tx_to_clean; - volatile uint16_t num_tx_desc_avail; - uint16_t num_tx_desc; - uint16_t last_hw_offload; - uint32_t txd_cmd; - struct em_buffer *tx_buffer_area; - bus_dma_tag_t txtag; /* dma tag for tx */ - uint32_t tx_tso; /* last tx was tso */ - - /* - * Receive definitions - * - * we have an array of num_rx_desc rx_desc (handled by the - * controller), and paired with an array of rx_buffers - * (at rx_buffer_area). - * The next pair to check on receive is at offset next_rx_desc_to_check - */ - struct em_dma_alloc rxdma; /* bus_dma glue for rx desc */ - struct e1000_rx_desc *rx_desc_base; - uint32_t next_rx_desc_to_check; - uint32_t rx_buffer_len; - uint16_t num_rx_desc; - int rx_process_limit; - struct em_buffer *rx_buffer_area; - bus_dma_tag_t rxtag; - bus_dmamap_t rx_sparemap; - - /* - * First/last mbuf pointers, for - * collecting multisegment RX packets. - */ - struct mbuf *fmp; - struct mbuf *lmp; - - /* Misc stats maintained by the driver */ - unsigned long dropped_pkts; - unsigned long link_irq; - unsigned long mbuf_cluster_failed; - unsigned long mbuf_defrag_failed; - unsigned long no_tx_desc_avail1; - unsigned long no_tx_desc_avail2; - unsigned long no_tx_dma_setup; - unsigned long no_tx_map_avail; - unsigned long watchdog_events; - unsigned long rx_irq; - unsigned long rx_overruns; - unsigned long tx_irq; - - /* 82547 workaround */ - uint32_t tx_fifo_size; - uint32_t tx_fifo_head; - uint32_t tx_fifo_head_addr; - uint64_t tx_fifo_reset_cnt; - uint64_t tx_fifo_wrk_cnt; - uint32_t tx_head_addr; - - /* For 82544 PCIX Workaround */ - boolean_t pcix_82544; - boolean_t in_detach; - -#ifdef NIC_SEND_COMBINING - /* 0 = idle; 1xxxx int-pending; 3xxxx int + d pending + tdt */ -#define MIT_PENDING_INT 0x10000 /* pending interrupt */ -#define MIT_PENDING_TDT 0x30000 /* both intr and tdt write are pending */ - uint32_t shadow_tdt; - uint32_t sc_enable; -#endif /* NIC_SEND_COMBINING */ -#ifdef BATCH_DISPATCH - uint32_t batch_enable; -#endif /* BATCH_DISPATCH */ - -#ifdef NIC_PARAVIRT - struct em_dma_alloc csb_mem; /* phys address */ - struct paravirt_csb *csb; /* virtual addr */ - uint32_t rx_retries; /* optimize rx loop */ - uint32_t tdt_csb_count;// XXX stat - uint32_t tdt_reg_count;// XXX stat - uint32_t tdt_int_count;// XXX stat - uint32_t guest_need_kick_count;// XXX stat -#endif /* NIC_PARAVIRT */ - - struct e1000_hw_stats stats; -}; - -/* ****************************************************************************** - * vendor_info_array - * - * This array contains the list of Subvendor/Subdevice IDs on which the driver - * should load. - * - * ******************************************************************************/ -typedef struct _em_vendor_info_t { - unsigned int vendor_id; - unsigned int device_id; - unsigned int subvendor_id; - unsigned int subdevice_id; - unsigned int index; -} em_vendor_info_t; - -struct em_buffer { - int next_eop; /* Index of the desc to watch */ - struct mbuf *m_head; - bus_dmamap_t map; /* bus_dma map for packet */ -}; - -/* For 82544 PCIX Workaround */ -typedef struct _ADDRESS_LENGTH_PAIR -{ - uint64_t address; - uint32_t length; -} ADDRESS_LENGTH_PAIR, *PADDRESS_LENGTH_PAIR; - -typedef struct _DESCRIPTOR_PAIR -{ - ADDRESS_LENGTH_PAIR descriptor[4]; - uint32_t elements; -} DESC_ARRAY, *PDESC_ARRAY; - -#define EM_CORE_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->core_mtx, _name, "EM Core Lock", MTX_DEF) -#define EM_TX_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->tx_mtx, _name, "EM TX Lock", MTX_DEF) -#define EM_RX_LOCK_INIT(_sc, _name) \ - mtx_init(&(_sc)->rx_mtx, _name, "EM RX Lock", MTX_DEF) -#define EM_CORE_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->core_mtx) -#define EM_TX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->tx_mtx) -#define EM_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) -#define EM_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) -#define EM_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) -#define EM_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) -#define EM_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) -#define EM_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) -#define EM_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) -#define EM_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_mtx) -#define EM_CORE_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->core_mtx, MA_OWNED) -#define EM_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_mtx, MA_OWNED) - -#endif /* _LEM_H_DEFINED_ */ diff --git a/sys/dev/e1000/igb_txrx.c b/sys/dev/e1000/igb_txrx.c new file mode 100644 index 000000000000..039269caad95 --- /dev/null +++ b/sys/dev/e1000/igb_txrx.c @@ -0,0 +1,620 @@ +/*- + * Copyright (c) 2016 Matt Macy + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* $FreeBSD$ */ +#include "if_em.h" + +#ifdef RSS +#include +#include +#endif + +#ifdef VERBOSE_DEBUG +#define DPRINTF device_printf +#else +#define DPRINTF(...) +#endif + +/********************************************************************* + * Local Function prototypes + *********************************************************************/ +static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); +static void igb_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); +static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx, bool clear); + +static void igb_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buf_len __unused); +static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); +static int igb_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, + int budget); +static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); + +static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); +static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); + +static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype); +static int igb_determine_rsstype(u16 pkt_info); + +extern void igb_if_enable_intr(if_ctx_t ctx); +extern int em_intr(void *arg); + +struct if_txrx igb_txrx = { + igb_isc_txd_encap, + igb_isc_txd_flush, + igb_isc_txd_credits_update, + igb_isc_rxd_available, + igb_isc_rxd_pkt_get, + igb_isc_rxd_refill, + igb_isc_rxd_flush, + em_intr +}; + +extern if_shared_ctx_t em_sctx; + +/********************************************************************** + * + * Setup work for hardware segmentation offload (TSO) on + * adapters using advanced tx descriptors + * + **********************************************************************/ +static int +igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) +{ + struct e1000_adv_tx_context_desc *TXD; + struct adapter *adapter = txr->adapter; + u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0; + u32 mss_l4len_idx = 0; + u32 paylen; + + switch(pi->ipi_etype) { + case ETHERTYPE_IPV6: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; + break; + case ETHERTYPE_IP: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; + /* Tell transmit desc to also do IPv4 checksum. */ + *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; + break; + default: + panic("%s: CSUM_TSO but no supported IP version (0x%04x)", + __func__, ntohs(pi->ipi_etype)); + break; + } + + TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; + + /* This is used in the transmit desc in encap */ + paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen; + + /* VLAN MACLEN IPLEN */ + if (pi->ipi_mflags & M_VLANTAG) { + vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); + } + + vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; + vlan_macip_lens |= pi->ipi_ip_hlen; + TXD->vlan_macip_lens = htole32(vlan_macip_lens); + + /* ADV DTYPE TUCMD */ + type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); + + /* MSS L4LEN IDX */ + mss_l4len_idx |= (pi->ipi_tso_segsz << E1000_ADVTXD_MSS_SHIFT); + mss_l4len_idx |= (pi->ipi_tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); + /* 82575 needs the queue index added */ + if (adapter->hw.mac.type == e1000_82575) + mss_l4len_idx |= txr->me << 4; + TXD->mss_l4len_idx = htole32(mss_l4len_idx); + + TXD->seqnum_seed = htole32(0); + *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; + *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; + *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; + + return (1); +} + +/********************************************************************* + * + * Advanced Context Descriptor setup for VLAN, CSUM or TSO + * + **********************************************************************/ +static int +igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) +{ + struct e1000_adv_tx_context_desc *TXD; + struct adapter *adapter = txr->adapter; + u32 vlan_macip_lens, type_tucmd_mlhl; + u32 mss_l4len_idx; + mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; + int offload = TRUE; + + /* First check if TSO is to be used */ + if (pi->ipi_csum_flags & CSUM_TSO) + return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); + + /* Indicate the whole packet as payload when not doing TSO */ + *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; + + /* Now ready a context descriptor */ + TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; + + /* + ** In advanced descriptors the vlan tag must + ** be placed into the context descriptor. Hence + ** we need to make one even if not doing offloads. + */ + if (pi->ipi_mflags & M_VLANTAG) { + vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); + } else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) { + return (0); + } + + /* Set the ether header length */ + vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; + + switch(pi->ipi_etype) { + case ETHERTYPE_IP: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; + break; + case ETHERTYPE_IPV6: + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; + break; + default: + offload = FALSE; + break; + } + + vlan_macip_lens |= pi->ipi_ip_hlen; + type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; + + switch (pi->ipi_ipproto) { + case IPPROTO_TCP: + #if __FreeBSD_version >= 1000000 + if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) +#else + if (pi->ipi_csum_flags & CSUM_TCP) +#endif + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; + break; + case IPPROTO_UDP: +#if __FreeBSD_version >= 1000000 + if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) +#else + if (pi->ipi_csum_flags & CSUM_UDP) +#endif + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; + break; + +#if __FreeBSD_version >= 800000 + case IPPROTO_SCTP: +#if __FreeBSD_version >= 1000000 + if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) +#else + if (pi->ipi_csum_flags & CSUM_SCTP) +#endif + type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; + break; +#endif + default: + offload = FALSE; + break; + } + + if (offload) /* For the TX descriptor setup */ + *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; + + /* 82575 needs the queue index added */ + if (adapter->hw.mac.type == e1000_82575) + mss_l4len_idx = txr->me << 4; + + /* Now copy bits into descriptor */ + TXD->vlan_macip_lens = htole32(vlan_macip_lens); + TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); + TXD->seqnum_seed = htole32(0); + TXD->mss_l4len_idx = htole32(mss_l4len_idx); + + return (1); +} + +static int +igb_isc_txd_encap(void *arg, if_pkt_info_t pi) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; + struct tx_ring *txr = &que->txr; + int nsegs = pi->ipi_nsegs; + bus_dma_segment_t *segs = pi->ipi_segs; + struct em_txbuffer *txbuf; + union e1000_adv_tx_desc *txd = NULL; + + int i, j, first, pidx_last; + u32 olinfo_status, cmd_type_len; + + pidx_last = olinfo_status = 0; + /* Basic descriptor defines */ + cmd_type_len = (E1000_ADVTXD_DTYP_DATA | + E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); + + if (pi->ipi_mflags & M_VLANTAG) + cmd_type_len |= E1000_ADVTXD_DCMD_VLE; + + first = i = pi->ipi_pidx; + + /* Consume the first descriptor */ + i += igb_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status); + if (i == scctx->isc_ntxd[0]) + i = 0; + + /* 82575 needs the queue index added */ + if (sc->hw.mac.type == e1000_82575) + olinfo_status |= txr->me << 4; + + for (j = 0; j < nsegs; j++) { + bus_size_t seglen; + bus_addr_t segaddr; + + txbuf = &txr->tx_buffers[i]; + txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; + seglen = segs[j].ds_len; + segaddr = htole64(segs[j].ds_addr); + + txd->read.buffer_addr = segaddr; + txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | + cmd_type_len | seglen); + txd->read.olinfo_status = htole32(olinfo_status); + pidx_last = i; + if (++i == scctx->isc_ntxd[0]) { + i = 0; + } + } + + txd->read.cmd_type_len |= + htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); + + /* Set the EOP descriptor that will be marked done */ + txbuf = &txr->tx_buffers[first]; + txbuf->eop = pidx_last; + + pi->ipi_new_pidx = i; + + return (0); +} + +static void +igb_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) +{ + struct adapter *adapter = arg; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); +} + +static int +igb_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_tx_queue *que = &adapter->tx_queues[txqid]; + struct tx_ring *txr = &que->txr; + + u32 cidx, ntxd, processed = 0; + + struct em_txbuffer *buf; + union e1000_adv_tx_desc *txd, *eop; + int limit; + + cidx = cidx_init; + + buf = &txr->tx_buffers[cidx]; + txd = (union e1000_adv_tx_desc *)&txr->tx_base[cidx]; + ntxd = scctx->isc_ntxd[0]; + limit = adapter->tx_process_limit; + + do { + if (buf->eop == -1) /* No work */ + break; + + eop = (union e1000_adv_tx_desc *)&txr->tx_base[buf->eop]; + if ((eop->wb.status & E1000_TXD_STAT_DD) == 0) + break; /* I/O not complete */ + + if (clear) + buf->eop = -1; /* clear indicate processed */ + + /* We clean the range if multi segment */ + while (txd != eop) { + ++txd; + ++buf; + /* wrap the ring? */ + if (++cidx == scctx->isc_ntxd[0]) { + cidx = 0; + buf = txr->tx_buffers; + txd = (union e1000_adv_tx_desc *)txr->tx_base; + } + + buf = &txr->tx_buffers[cidx]; + if (clear) + buf->eop = -1; + processed++; + } + processed++; + + /* Try the next packet */ + txd++; + buf++; + + /* reset with a wrap */ + if (++cidx == scctx->isc_ntxd[0]) { + cidx = 0; + buf = txr->tx_buffers; + txd = (union e1000_adv_tx_desc *)txr->tx_base; + } + prefetch(txd); + prefetch(txd+1); + } while (__predict_true(--limit) && cidx != cidx_init); + + return (processed); +} + +static void +igb_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, + uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, + uint16_t count, uint16_t buf_len __unused) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + union e1000_adv_rx_desc *rxd; + struct rx_ring *rxr = &que->rxr; + int i; + uint32_t next_pidx; + + for (i = 0, next_pidx = pidx; i < count; i++) { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[next_pidx]; + + rxd->read.pkt_addr = htole64(paddrs[i]); + if (++next_pidx == scctx->isc_nrxd[0]) + next_pidx = 0; + } +} + +static void +igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) +{ + struct adapter *sc = arg; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + + E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); +} + +static int +igb_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) +{ + struct adapter *sc = arg; + if_softc_ctx_t scctx = sc->shared; + struct em_rx_queue *que = &sc->rx_queues[rxqid]; + struct rx_ring *rxr = &que->rxr; + union e1000_adv_rx_desc *rxd; + u32 staterr = 0; + int cnt, i, iter; + + for (iter = cnt = 0, i = idx; iter < scctx->isc_nrxd[0] && iter <= budget;) { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; + staterr = le32toh(rxd->wb.upper.status_error); + + if ((staterr & E1000_RXD_STAT_DD) == 0) + break; + + if (++i == scctx->isc_nrxd[0]) { + i = 0; + } + + if (staterr & E1000_RXD_STAT_EOP) + cnt++; + iter++; + } + { + struct e1000_hw *hw = &sc->hw; + int rdt, rdh; + rdt = E1000_READ_REG(hw, E1000_RDT(rxr->me)); + rdh = E1000_READ_REG(hw, E1000_RDH(rxr->me)); + DPRINTF(iflib_get_dev(sc->ctx), "sidx:%d eidx:%d iter=%d pktcnt=%d RDT=%d RDH=%d\n", idx, i, iter, cnt, rdt, rdh); + } + return (cnt); +} + +/**************************************************************** + * Routine sends data which has been dma'ed into host memory + * to upper layer. Initialize ri structure. + * + * Returns 0 upon success, errno on failure + ***************************************************************/ + +static int +igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) +{ + struct adapter *adapter = arg; + if_softc_ctx_t scctx = adapter->shared; + struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; + struct rx_ring *rxr = &que->rxr; + struct ifnet *ifp = iflib_get_ifp(adapter->ctx); + union e1000_adv_rx_desc *rxd; + + u16 pkt_info, len; + u16 vtag = 0; + u32 ptype; + u32 staterr = 0; + bool eop; + int i = 0; + int cidx = ri->iri_cidx; + + do { + rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; + staterr = le32toh(rxd->wb.upper.status_error); + pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); + + MPASS ((staterr & E1000_RXD_STAT_DD) != 0); + + len = le16toh(rxd->wb.upper.length); + ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; + + ri->iri_len += len; + rxr->rx_bytes += ri->iri_len; + + rxd->wb.upper.status_error = 0; + eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); + + if (((adapter->hw.mac.type == e1000_i350) || + (adapter->hw.mac.type == e1000_i354)) && + (staterr & E1000_RXDEXT_STATERR_LB)) + vtag = be16toh(rxd->wb.upper.vlan); + else + vtag = le16toh(rxd->wb.upper.vlan); + + /* Make sure bad packets are discarded */ + if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { + adapter->dropped_pkts++; + ++rxr->rx_discarded; + return (EBADMSG); + } + ri->iri_frags[i].irf_flid = 0; + ri->iri_frags[i].irf_idx = cidx; + ri->iri_frags[i].irf_len = len; + + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; +#ifdef notyet + if (rxr->hdr_split == TRUE) { + ri->iri_frags[i].irf_flid = 1; + ri->iri_frags[i].irf_idx = cidx; + if (++cidx == scctx->isc_nrxd[0]) + cidx = 0; + } +#endif + i++; + } while (!eop); + + rxr->rx_packets++; + + if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) + igb_rx_checksum(staterr, ri, ptype); + + if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && + (staterr & E1000_RXD_STAT_VP) != 0) { + ri->iri_vtag = vtag; + ri->iri_flags |= M_VLANTAG; + } + ri->iri_flowid = + le32toh(rxd->wb.lower.hi_dword.rss); + ri->iri_rsstype = igb_determine_rsstype(pkt_info); + ri->iri_nfrags = i; + + return (0); +} + +/********************************************************************* + * + * Verify that the hardware indicated that the checksum is valid. + * Inform the stack about the status of checksum so that stack + * doesn't spend time verifying the checksum. + * + *********************************************************************/ +static void +igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype) +{ + u16 status = (u16)staterr; + u8 errors = (u8) (staterr >> 24); + bool sctp = FALSE; + + /* Ignore Checksum bit is set */ + if (status & E1000_RXD_STAT_IXSM) { + ri->iri_csum_flags = 0; + return; + } + + if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && + (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) + sctp = 1; + else + sctp = 0; + + if (status & E1000_RXD_STAT_IPCS) { + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_IPE)) { + /* IP Checksum Good */ + ri->iri_csum_flags = CSUM_IP_CHECKED; + ri->iri_csum_flags |= CSUM_IP_VALID; + } else + ri->iri_csum_flags = 0; + } + + if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { + u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); +#if __FreeBSD_version >= 800000 + if (sctp) /* reassign */ + type = CSUM_SCTP_VALID; +#endif + /* Did it pass? */ + if (!(errors & E1000_RXD_ERR_TCPE)) { + ri->iri_csum_flags |= type; + if (sctp == 0) + ri->iri_csum_data = htons(0xffff); + } + } + return; +} + +/******************************************************************** + * + * Parse the packet type to determine the appropriate hash + * + ******************************************************************/ +static int +igb_determine_rsstype(u16 pkt_info) +{ + switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { + case E1000_RXDADV_RSSTYPE_IPV4_TCP: + return M_HASHTYPE_RSS_TCP_IPV4; + case E1000_RXDADV_RSSTYPE_IPV4: + return M_HASHTYPE_RSS_IPV4; + case E1000_RXDADV_RSSTYPE_IPV6_TCP: + return M_HASHTYPE_RSS_TCP_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_EX: + return M_HASHTYPE_RSS_IPV6_EX; + case E1000_RXDADV_RSSTYPE_IPV6: + return M_HASHTYPE_RSS_IPV6; + case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: + return M_HASHTYPE_RSS_TCP_IPV6_EX; + default: + return M_HASHTYPE_OPAQUE; + } +} diff --git a/sys/dev/ntb/if_ntb/if_ntb.c b/sys/dev/ntb/if_ntb/if_ntb.c index c67ae0dc682f..ace4861e6a89 100644 --- a/sys/dev/ntb/if_ntb/if_ntb.c +++ b/sys/dev/ntb/if_ntb/if_ntb.c @@ -237,6 +237,11 @@ ntb_ioctl(if_t ifp, u_long command, caddr_t data) int error = 0; switch (command) { + case SIOCSIFFLAGS: + case SIOCADDMULTI: + case SIOCDELMULTI: + break; + case SIOCSIFMTU: { if (ifr->ifr_mtu > sc->mtu - ETHER_HDR_LEN) { diff --git a/sys/dev/rtwn/pci/rtwn_pci_attach.c b/sys/dev/rtwn/pci/rtwn_pci_attach.c index e021c401ad3c..3af8d176f790 100644 --- a/sys/dev/rtwn/pci/rtwn_pci_attach.c +++ b/sys/dev/rtwn/pci/rtwn_pci_attach.c @@ -94,20 +94,31 @@ static void rtwn_pci_beacon_update_end(struct rtwn_softc *, static void rtwn_pci_attach_methods(struct rtwn_softc *); -static int matched_chip = RTWN_CHIP_MAX_PCI; +static const struct rtwn_pci_ident * +rtwn_pci_probe_sub(device_t dev) +{ + const struct rtwn_pci_ident *ident; + int vendor_id, device_id; + + vendor_id = pci_get_vendor(dev); + device_id = pci_get_device(dev); + + for (ident = rtwn_pci_ident_table; ident->name != NULL; ident++) + if (vendor_id == ident->vendor && device_id == ident->device) + return (ident); + + return (NULL); +} static int rtwn_pci_probe(device_t dev) { const struct rtwn_pci_ident *ident; - for (ident = rtwn_pci_ident_table; ident->name != NULL; ident++) { - if (pci_get_vendor(dev) == ident->vendor && - pci_get_device(dev) == ident->device) { - matched_chip = ident->chip; - device_set_desc(dev, ident->name); - return (BUS_PROBE_DEFAULT); - } + ident = rtwn_pci_probe_sub(dev); + if (ident != NULL) { + device_set_desc(dev, ident->name); + return (BUS_PROBE_DEFAULT); } return (ENXIO); } @@ -591,13 +602,15 @@ rtwn_pci_attach_methods(struct rtwn_softc *sc) static int rtwn_pci_attach(device_t dev) { + const struct rtwn_pci_ident *ident; struct rtwn_pci_softc *pc = device_get_softc(dev); struct rtwn_softc *sc = &pc->pc_sc; struct ieee80211com *ic = &sc->sc_ic; uint32_t lcsr; int cap_off, i, error, rid; - if (matched_chip >= RTWN_CHIP_MAX_PCI) + ident = rtwn_pci_probe_sub(dev); + if (ident == NULL) return (ENXIO); /* @@ -649,8 +662,7 @@ rtwn_pci_attach(device_t dev) mtx_init(&sc->sc_mtx, ic->ic_name, MTX_NETWORK_LOCK, MTX_DEF); rtwn_pci_attach_methods(sc); - /* XXX something similar to USB_GET_DRIVER_INFO() */ - rtwn_pci_attach_private(pc, matched_chip); + rtwn_pci_attach_private(pc, ident->chip); /* Allocate Tx/Rx buffers. */ error = rtwn_pci_alloc_rx_list(sc); diff --git a/sys/dev/rtwn/usb/rtwn_usb_ep.c b/sys/dev/rtwn/usb/rtwn_usb_ep.c index 328e5bd36e1a..52c628d5a9a7 100644 --- a/sys/dev/rtwn/usb/rtwn_usb_ep.c +++ b/sys/dev/rtwn/usb/rtwn_usb_ep.c @@ -58,7 +58,7 @@ __FBSDID("$FreeBSD$"); #include -static struct usb_config rtwn_config[RTWN_N_TRANSFER] = { +static const struct usb_config rtwn_config_common[RTWN_N_TRANSFER] = { [RTWN_BULK_RX] = { .type = UE_BULK, .endpoint = UE_ADDR_ANY, @@ -161,6 +161,7 @@ rtwn_usb_setup_queues(struct rtwn_usb_softc *uc) int rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) { + struct usb_config *rtwn_config; struct rtwn_softc *sc = &uc->uc_sc; const uint8_t iface_index = RTWN_IFACE_INDEX; struct usb_endpoint *ep, *ep_end; @@ -197,6 +198,9 @@ rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) return (EINVAL); } + rtwn_config = malloc(sizeof(rtwn_config_common), M_TEMP, M_WAITOK); + memcpy(rtwn_config, rtwn_config_common, sizeof(rtwn_config_common)); + /* NB: keep in sync with rtwn_dma_init(). */ rtwn_config[RTWN_BULK_TX_VO].endpoint = addr[0]; switch (uc->ntx) { @@ -224,6 +228,8 @@ rtwn_usb_setup_endpoints(struct rtwn_usb_softc *uc) rtwn_config[RTWN_BULK_RX].bufsize = sc->rx_dma_size + 1024; error = usbd_transfer_setup(uc->uc_udev, &iface_index, uc->uc_xfer, rtwn_config, RTWN_N_TRANSFER, uc, &sc->sc_mtx); + free(rtwn_config, M_TEMP); + if (error) { device_printf(sc->sc_dev, "could not allocate USB transfers, " "err=%s\n", usbd_errstr(error)); diff --git a/sys/dev/sdhci/sdhci_acpi.c b/sys/dev/sdhci/sdhci_acpi.c new file mode 100644 index 000000000000..a2a2cab0ffd2 --- /dev/null +++ b/sys/dev/sdhci/sdhci_acpi.c @@ -0,0 +1,370 @@ +/*- + * Copyright (c) 2017 Oleksandr Tymoshenko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "sdhci.h" +#include "mmcbr_if.h" +#include "sdhci_if.h" + +static const struct sdhci_acpi_device { + const char* hid; + int uid; + const char *desc; + u_int quirks; +} sdhci_acpi_devices[] = { + { "80860F14", 1, "Intel Bay Trail eMMC 4.5 Controller", + SDHCI_QUIRK_ALL_SLOTS_NON_REMOVABLE | + SDHCI_QUIRK_INTEL_POWER_UP_RESET }, + { "80860F16", 0, "Intel Bay Trail SD Host Controller", + 0 }, + { NULL, 0, NULL, 0} +}; + +static char *sdhci_ids[] = { + "80860F14", + "80860F16", + NULL +}; + +struct sdhci_acpi_softc { + u_int quirks; /* Chip specific quirks */ + struct resource *irq_res; /* IRQ resource */ + void *intrhand; /* Interrupt handle */ + + struct sdhci_slot slot; + struct resource *mem_res; /* Memory resource */ +}; + +static void sdhci_acpi_intr(void *arg); +static int sdhci_acpi_detach(device_t dev); + +static uint8_t +sdhci_acpi_read_1(device_t dev, struct sdhci_slot *slot, bus_size_t off) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + return bus_read_1(sc->mem_res, off); +} + +static void +sdhci_acpi_write_1(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint8_t val) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + bus_write_1(sc->mem_res, off, val); +} + +static uint16_t +sdhci_acpi_read_2(device_t dev, struct sdhci_slot *slot, bus_size_t off) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + return bus_read_2(sc->mem_res, off); +} + +static void +sdhci_acpi_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint16_t val) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + bus_write_2(sc->mem_res, off, val); +} + +static uint32_t +sdhci_acpi_read_4(device_t dev, struct sdhci_slot *slot, bus_size_t off) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + return bus_read_4(sc->mem_res, off); +} + +static void +sdhci_acpi_write_4(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint32_t val) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_barrier(sc->mem_res, 0, 0xFF, + BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); + bus_write_4(sc->mem_res, off, val); +} + +static void +sdhci_acpi_read_multi_4(device_t dev, struct sdhci_slot *slot, + bus_size_t off, uint32_t *data, bus_size_t count) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_read_multi_stream_4(sc->mem_res, off, data, count); +} + +static void +sdhci_acpi_write_multi_4(device_t dev, struct sdhci_slot *slot, + bus_size_t off, uint32_t *data, bus_size_t count) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + bus_write_multi_stream_4(sc->mem_res, off, data, count); +} + +static const struct sdhci_acpi_device * +sdhci_acpi_find_device(device_t dev) +{ + const char *hid; + int i, uid; + ACPI_HANDLE handle; + ACPI_STATUS status; + + hid = ACPI_ID_PROBE(device_get_parent(dev), dev, sdhci_ids); + if (hid == NULL) + return (NULL); + + handle = acpi_get_handle(dev); + status = acpi_GetInteger(handle, "_UID", &uid); + if (ACPI_FAILURE(status)) + uid = 0; + + for (i = 0; sdhci_acpi_devices[i].hid != NULL; i++) { + if (strcmp(sdhci_acpi_devices[i].hid, hid) != 0) + continue; + if ((sdhci_acpi_devices[i].uid != 0) && + (sdhci_acpi_devices[i].uid != uid)) + continue; + return &sdhci_acpi_devices[i]; + } + + return (NULL); +} + +static int +sdhci_acpi_probe(device_t dev) +{ + const struct sdhci_acpi_device *acpi_dev; + + acpi_dev = sdhci_acpi_find_device(dev); + if (acpi_dev == NULL) + return (ENXIO); + + device_set_desc(dev, acpi_dev->desc); + + return (BUS_PROBE_DEFAULT); +} + +static int +sdhci_acpi_attach(device_t dev) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + int rid, err; + const struct sdhci_acpi_device *acpi_dev; + + acpi_dev = sdhci_acpi_find_device(dev); + if (acpi_dev == NULL) + return (ENXIO); + + sc->quirks = acpi_dev->quirks; + + /* Allocate IRQ. */ + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, "can't allocate IRQ\n"); + return (ENOMEM); + } + + rid = 0; + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &rid, RF_ACTIVE); + if (sc->mem_res == NULL) { + device_printf(dev, "can't allocate memory resource for slot\n"); + sdhci_acpi_detach(dev); + return (ENOMEM); + } + + sc->slot.quirks = sc->quirks; + + err = sdhci_init_slot(dev, &sc->slot, 0); + if (err) { + device_printf(dev, "failed to init slot\n"); + sdhci_acpi_detach(dev); + return (err); + } + + /* Activate the interrupt */ + err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE, + NULL, sdhci_acpi_intr, sc, &sc->intrhand); + if (err) { + device_printf(dev, "can't setup IRQ\n"); + sdhci_acpi_detach(dev); + return (err); + } + + /* Process cards detection. */ + sdhci_start_slot(&sc->slot); + + return (0); +} + +static int +sdhci_acpi_detach(device_t dev) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + + if (sc->intrhand) + bus_teardown_intr(dev, sc->irq_res, sc->intrhand); + if (sc->irq_res) + bus_release_resource(dev, SYS_RES_IRQ, + rman_get_rid(sc->irq_res), sc->irq_res); + + if (sc->mem_res) { + sdhci_cleanup_slot(&sc->slot); + bus_release_resource(dev, SYS_RES_MEMORY, + rman_get_rid(sc->mem_res), sc->mem_res); + } + + return (0); +} + +static int +sdhci_acpi_shutdown(device_t dev) +{ + + return (0); +} + +static int +sdhci_acpi_suspend(device_t dev) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + int err; + + err = bus_generic_suspend(dev); + if (err) + return (err); + sdhci_generic_suspend(&sc->slot); + return (0); +} + +static int +sdhci_acpi_resume(device_t dev) +{ + struct sdhci_acpi_softc *sc = device_get_softc(dev); + int err; + + sdhci_generic_resume(&sc->slot); + err = bus_generic_resume(dev); + if (err) + return (err); + return (0); +} + +static void +sdhci_acpi_intr(void *arg) +{ + struct sdhci_acpi_softc *sc = (struct sdhci_acpi_softc *)arg; + + sdhci_generic_intr(&sc->slot); +} + +static device_method_t sdhci_methods[] = { + /* device_if */ + DEVMETHOD(device_probe, sdhci_acpi_probe), + DEVMETHOD(device_attach, sdhci_acpi_attach), + DEVMETHOD(device_detach, sdhci_acpi_detach), + DEVMETHOD(device_shutdown, sdhci_acpi_shutdown), + DEVMETHOD(device_suspend, sdhci_acpi_suspend), + DEVMETHOD(device_resume, sdhci_acpi_resume), + + /* Bus interface */ + DEVMETHOD(bus_read_ivar, sdhci_generic_read_ivar), + DEVMETHOD(bus_write_ivar, sdhci_generic_write_ivar), + + /* mmcbr_if */ + DEVMETHOD(mmcbr_update_ios, sdhci_generic_update_ios), + DEVMETHOD(mmcbr_request, sdhci_generic_request), + DEVMETHOD(mmcbr_get_ro, sdhci_generic_get_ro), + DEVMETHOD(mmcbr_acquire_host, sdhci_generic_acquire_host), + DEVMETHOD(mmcbr_release_host, sdhci_generic_release_host), + + /* SDHCI registers accessors */ + DEVMETHOD(sdhci_read_1, sdhci_acpi_read_1), + DEVMETHOD(sdhci_read_2, sdhci_acpi_read_2), + DEVMETHOD(sdhci_read_4, sdhci_acpi_read_4), + DEVMETHOD(sdhci_read_multi_4, sdhci_acpi_read_multi_4), + DEVMETHOD(sdhci_write_1, sdhci_acpi_write_1), + DEVMETHOD(sdhci_write_2, sdhci_acpi_write_2), + DEVMETHOD(sdhci_write_4, sdhci_acpi_write_4), + DEVMETHOD(sdhci_write_multi_4, sdhci_acpi_write_multi_4), + + DEVMETHOD_END +}; + +static driver_t sdhci_acpi_driver = { + "sdhci_acpi", + sdhci_methods, + sizeof(struct sdhci_acpi_softc), +}; +static devclass_t sdhci_acpi_devclass; + +DRIVER_MODULE(sdhci_acpi, acpi, sdhci_acpi_driver, sdhci_acpi_devclass, NULL, + NULL); +MODULE_DEPEND(sdhci_acpi, sdhci, 1, 1, 1); +DRIVER_MODULE(mmc, sdhci_acpi, mmc_driver, mmc_devclass, NULL, NULL); +MODULE_DEPEND(sdhci_acpi, mmc, 1, 1, 1); diff --git a/sys/dev/sfxge/sfxge_tx.c b/sys/dev/sfxge/sfxge_tx.c index 4e31aeccbb2f..dd78e6e8a24f 100644 --- a/sys/dev/sfxge/sfxge_tx.c +++ b/sys/dev/sfxge/sfxge_tx.c @@ -363,8 +363,22 @@ static int sfxge_tx_queue_mbuf(struct sfxge_txq *txq, struct mbuf *mbuf) KASSERT(!txq->blocked, ("txq->blocked")); +#if SFXGE_TX_PARSE_EARLY + /* + * If software TSO is used, we still need to copy packet header, + * even if we have already parsed it early before enqueue. + */ + if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) && + (txq->tso_fw_assisted == 0)) + prefetch_read_many(mbuf->m_data); +#else + /* + * Prefetch packet header since we need to parse it and extract + * IP ID, TCP sequence number and flags. + */ if (mbuf->m_pkthdr.csum_flags & CSUM_TSO) prefetch_read_many(mbuf->m_data); +#endif if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) { rc = EINTR; diff --git a/sys/dev/usb/controller/ehci_imx.c b/sys/dev/usb/controller/ehci_imx.c index 07f73105f9ab..df4d202c38ef 100644 --- a/sys/dev/usb/controller/ehci_imx.c +++ b/sys/dev/usb/controller/ehci_imx.c @@ -157,6 +157,18 @@ struct imx_ehci_softc { struct resource *ehci_irq_res; /* EHCI core IRQ. */ }; +static void +imx_ehci_post_reset(struct ehci_softc *ehci_softc) +{ + uint32_t usbmode; + + /* Force HOST mode */ + usbmode = EOREAD4(ehci_softc, EHCI_USBMODE_NOLPM); + usbmode &= ~EHCI_UM_CM; + usbmode |= EHCI_UM_CM_HOST; + EOWRITE4(ehci_softc, EHCI_USBMODE_NOLPM, usbmode); +} + static int imx_ehci_probe(device_t dev) { @@ -282,8 +294,13 @@ imx_ehci_attach(device_t dev) esc->sc_id_vendor = USB_VENDOR_FREESCALE; strlcpy(esc->sc_vendor, "Freescale", sizeof(esc->sc_vendor)); - /* Set flags that affect ehci_init() behavior. */ - esc->sc_flags |= EHCI_SCFLG_DONTRESET | EHCI_SCFLG_NORESTERM; + /* + * Set flags that affect ehci_init() behavior, and hook our post-reset + * code into the standard controller code. + */ + esc->sc_flags |= EHCI_SCFLG_NORESTERM; + esc->sc_vendor_post_reset = imx_ehci_post_reset; + err = ehci_init(esc); if (err != 0) { device_printf(dev, "USB init failed, usb_err_t=%d\n", diff --git a/sys/dev/usb/misc/udbp.c b/sys/dev/usb/misc/udbp.c index 3c07931bc068..f8f7a4b6f44a 100644 --- a/sys/dev/usb/misc/udbp.c +++ b/sys/dev/usb/misc/udbp.c @@ -259,6 +259,7 @@ static driver_t udbp_driver = { }; static const STRUCT_USB_HOST_ID udbp_devs[] = { + {USB_VPI(USB_VENDOR_BELKIN, USB_PRODUCT_BELKIN_F5U258, 0)}, {USB_VPI(USB_VENDOR_NETCHIP, USB_PRODUCT_NETCHIP_TURBOCONNECT, 0)}, {USB_VPI(USB_VENDOR_NETCHIP, USB_PRODUCT_NETCHIP_GADGETZERO, 0)}, {USB_VPI(USB_VENDOR_PROLIFIC, USB_PRODUCT_PROLIFIC_PL2301, 0)}, diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs index a625c2bf4b2d..797077e16f80 100644 --- a/sys/dev/usb/usbdevs +++ b/sys/dev/usb/usbdevs @@ -1318,6 +1318,7 @@ product BELKIN RTL8188CU 0x1102 RTL8188CU Wireless Adapter product BELKIN F9L1103 0x1103 F9L1103 Wireless Adapter product BELKIN RTL8192CU 0x2102 RTL8192CU Wireless Adapter product BELKIN F7D2102 0x2103 F7D2102 Wireless Adapter +product BELKIN F5U258 0x258A F5U258 Host to Host cable product BELKIN ZD1211B 0x4050 ZD1211B product BELKIN F5D5055 0x5055 F5D5055 product BELKIN F5D7050 0x7050 F5D7050 Wireless Adapter diff --git a/sys/dev/usb/wlan/if_rsu.c b/sys/dev/usb/wlan/if_rsu.c index 9fe82eea8cab..abab5b5f6004 100644 --- a/sys/dev/usb/wlan/if_rsu.c +++ b/sys/dev/usb/wlan/if_rsu.c @@ -523,6 +523,12 @@ rsu_attach(device_t self) sc->sc_ntxstream = 2; rft = "2T2R"; break; + case 0x3: /* "green" NIC */ + sc->sc_rftype = RTL8712_RFCONFIG_1T2R; + sc->sc_nrxstream = 2; + sc->sc_ntxstream = 1; + rft = "1T2R ('green')"; + break; default: device_printf(sc->sc_dev, "%s: unknown board type (rfconfig=0x%02x)\n", diff --git a/sys/fs/cd9660/cd9660_vfsops.c b/sys/fs/cd9660/cd9660_vfsops.c index 7473f97dceef..96c7dcfbaee4 100644 --- a/sys/fs/cd9660/cd9660_vfsops.c +++ b/sys/fs/cd9660/cd9660_vfsops.c @@ -88,6 +88,7 @@ static struct vfsops cd9660_vfsops = { VFS_SET(cd9660_vfsops, cd9660, VFCF_READONLY); MODULE_VERSION(cd9660, 1); +static int cd9660_vfs_hash_cmp(struct vnode *vp, void *pino); static int iso_mountfs(struct vnode *devvp, struct mount *mp); /* @@ -649,12 +650,14 @@ cd9660_vget(mp, ino, flags, vpp) static int cd9660_vfs_hash_cmp(vp, pino) struct vnode *vp; - cd_ino_t *pino; + void *pino; { struct iso_node *ip; + cd_ino_t ino; ip = VTOI(vp); - return (ip->i_number != *pino); + ino = *(cd_ino_t *)pino; + return (ip->i_number != ino); } int diff --git a/sys/fs/cd9660/iso.h b/sys/fs/cd9660/iso.h index 53f005c870f0..0d3e46c9a739 100644 --- a/sys/fs/cd9660/iso.h +++ b/sys/fs/cd9660/iso.h @@ -222,7 +222,7 @@ enum ISO_FTYPE { ISO_FTYPE_DEFAULT, ISO_FTYPE_9660, ISO_FTYPE_RRIP, /* * When ino_t becomes 64-bit, we can remove this definition in favor of ino_t. */ -#define cd_ino_t uint64_t +typedef __uint64_t cd_ino_t; struct iso_mnt { uint64_t im_flags; diff --git a/sys/fs/pseudofs/pseudofs_vncache.c b/sys/fs/pseudofs/pseudofs_vncache.c index 1bec5a4c885c..093d80565eaa 100644 --- a/sys/fs/pseudofs/pseudofs_vncache.c +++ b/sys/fs/pseudofs/pseudofs_vncache.c @@ -51,6 +51,7 @@ static struct mtx pfs_vncache_mutex; static struct pfs_vdata *pfs_vncache; static eventhandler_tag pfs_exit_tag; static void pfs_exit(void *arg, struct proc *p); +static void pfs_purge_locked(struct pfs_node *pn, bool force); static SYSCTL_NODE(_vfs_pfs, OID_AUTO, vncache, CTLFLAG_RW, 0, "pseudofs vnode cache"); @@ -97,6 +98,9 @@ pfs_vncache_unload(void) { EVENTHANDLER_DEREGISTER(process_exit, pfs_exit_tag); + mtx_lock(&pfs_vncache_mutex); + pfs_purge_locked(NULL, true); + mtx_unlock(&pfs_vncache_mutex); KASSERT(pfs_vncache_entries == 0, ("%d vncache entries remaining", pfs_vncache_entries)); mtx_destroy(&pfs_vncache_mutex); @@ -272,7 +276,7 @@ pfs_vncache_free(struct vnode *vp) * used to implement the cache. */ static void -pfs_purge_locked(struct pfs_node *pn) +pfs_purge_locked(struct pfs_node *pn, bool force) { struct pfs_vdata *pvd; struct vnode *vnp; @@ -280,7 +284,8 @@ pfs_purge_locked(struct pfs_node *pn) mtx_assert(&pfs_vncache_mutex, MA_OWNED); pvd = pfs_vncache; while (pvd != NULL) { - if (pvd->pvd_dead || (pn != NULL && pvd->pvd_pn == pn)) { + if (force || pvd->pvd_dead || + (pn != NULL && pvd->pvd_pn == pn)) { vnp = pvd->pvd_vnode; vhold(vnp); mtx_unlock(&pfs_vncache_mutex); @@ -301,7 +306,7 @@ pfs_purge(struct pfs_node *pn) { mtx_lock(&pfs_vncache_mutex); - pfs_purge_locked(pn); + pfs_purge_locked(pn, false); mtx_unlock(&pfs_vncache_mutex); } @@ -321,6 +326,6 @@ pfs_exit(void *arg, struct proc *p) if (pvd->pvd_pid == p->p_pid) dead = pvd->pvd_dead = 1; if (dead) - pfs_purge_locked(NULL); + pfs_purge_locked(NULL, false); mtx_unlock(&pfs_vncache_mutex); } diff --git a/sys/i386/conf/GENERIC b/sys/i386/conf/GENERIC index cbb15a91933e..f536927fc2c5 100644 --- a/sys/i386/conf/GENERIC +++ b/sys/i386/conf/GENERIC @@ -235,7 +235,6 @@ device puc # Multi I/O cards and multi-channel UARTs device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family -device igb # Intel PRO/1000 PCIE Server Gigabit Family device ixgb # Intel PRO/10GbE Ethernet Card device le # AMD Am7900 LANCE and Am79C9xx PCnet device ti # Alteon Networks Tigon I/II gigabit Ethernet diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index c5296f69654a..b4eef419a698 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -4905,7 +4905,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { pd_entry_t oldpde, *pde; pt_entry_t *pte; - vm_offset_t pdnxt; + vm_offset_t va, pdnxt; vm_page_t m; boolean_t anychanged, pv_lists_locked; @@ -4966,11 +4966,11 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } if (pdnxt > eva) pdnxt = eva; + va = pdnxt; for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++, sva += PAGE_SIZE) { - if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | - PG_V)) - continue; + if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) + goto maybe_invlrng; else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (advice == MADV_DONTNEED) { /* @@ -4985,12 +4985,21 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } else if ((*pte & PG_A) != 0) atomic_clear_int((u_int *)pte, PG_A); else - continue; - if ((*pte & PG_G) != 0) - pmap_invalidate_page(pmap, sva); - else + goto maybe_invlrng; + if ((*pte & PG_G) != 0) { + if (va == pdnxt) + va = sva; + } else anychanged = TRUE; + continue; +maybe_invlrng: + if (va != pdnxt) { + pmap_invalidate_range(pmap, va, sva); + va = pdnxt; + } } + if (va != pdnxt) + pmap_invalidate_range(pmap, va, sva); } if (anychanged) pmap_invalidate_all(pmap); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index b746eb93eb4d..c4498a191f55 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -3905,10 +3905,8 @@ biodone(struct bio *bp) bp->bio_flags |= BIO_DONE; wakeup(bp); mtx_unlock(mtxp); - } else { - bp->bio_flags |= BIO_DONE; + } else done(bp); - } } /* diff --git a/sys/mips/conf/OCTEON1 b/sys/mips/conf/OCTEON1 index d3ca58ca7faf..34f8ef1d6408 100644 --- a/sys/mips/conf/OCTEON1 +++ b/sys/mips/conf/OCTEON1 @@ -195,7 +195,6 @@ device octm # PCI Ethernet NICs. device de # DEC/Intel DC21x4x (``Tulip'') device em # Intel PRO/1000 Gigabit Ethernet Family -device igb # Intel PRO/1000 PCIE Server Gigabit Family device ix # Intel PRO/10GbE PF PCIE Ethernet Family device ixv # Intel PRO/10GbE VF PCIE Ethernet Family device le # AMD Am7900 LANCE and Am79C9xx PCnet diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 69e5fc4d64e4..0e016f7ce820 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -163,7 +163,6 @@ SUBDIR= \ if_tun \ if_vlan \ if_vxlan \ - ${_igb} \ ${_iir} \ imgact_binmisc \ ${_intelspi} \ @@ -333,6 +332,7 @@ SUBDIR= \ scc \ ${_scsi_low} \ sdhci \ + ${_sdhci_acpi} \ sdhci_pci \ sem \ send \ @@ -544,7 +544,6 @@ _cxgb= cxgb .if ${MACHINE_CPUARCH} == "aarch64" _armv8crypto= armv8crypto _em= em -_igb= igb .endif .if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64" @@ -572,7 +571,6 @@ _fe= fe _ibcore= ibcore .endif _if_ndis= if_ndis -_igb= igb _io= io .if ${MK_OFED} != "no" || defined(ALL_MODULES) _ipoib= ipoib @@ -668,6 +666,7 @@ _padlock_rng= padlock_rng _rdrand_rng= rdrand_rng .endif _s3= s3 +_sdhci_acpi= sdhci_acpi _tpm= tpm _twa= twa _vesa= vesa @@ -779,7 +778,6 @@ _nvram= powermac_nvram _auxio= auxio _em= em _epic= epic -_igb= igb .endif .if (${MACHINE_CPUARCH} == "amd64" || ${MACHINE_ARCH} == "armv6" || \ diff --git a/sys/modules/ahci/Makefile b/sys/modules/ahci/Makefile index 5666870532a1..cc947d3b910a 100644 --- a/sys/modules/ahci/Makefile +++ b/sys/modules/ahci/Makefile @@ -6,7 +6,7 @@ KMOD= ahci SRCS= ahci.c ahci_pci.c ahciem.c ahci.h device_if.h bus_if.h pci_if.h opt_cam.h .if ${MACHINE_CPUARCH} == "aarch64" -SRCS+= ahci_generic.c ofw_bus_if.h +SRCS+= ahci_generic.c acpi_if.h ofw_bus_if.h .endif .include diff --git a/sys/modules/em/Makefile b/sys/modules/em/Makefile index 1de7c62298e9..ead132978052 100644 --- a/sys/modules/em/Makefile +++ b/sys/modules/em/Makefile @@ -1,23 +1,28 @@ + # $FreeBSD$ .PATH: ${.CURDIR}/../../dev/e1000 KMOD = if_em SRCS = device_if.h bus_if.h pci_if.h opt_ddb.h opt_em.h opt_inet.h \ - opt_inet6.h + opt_inet6.h ifdi_if.h SRCS += $(CORE_SRC) $(LEGACY_SRC) SRCS += $(COMMON_SHARED) $(LEGACY_SHARED) $(PCIE_SHARED) -CORE_SRC = if_em.c e1000_osdep.c +CORE_SRC = if_em.c em_txrx.c e1000_osdep.c +CORE_SRC += igb_txrx.c # This is the Legacy, pre-PCIE source, it can be # undefined when using modular driver if not needed -LEGACY_SRC += if_lem.c COMMON_SHARED = e1000_api.c e1000_phy.c e1000_nvm.c e1000_mac.c \ e1000_manage.c e1000_vf.c e1000_mbx.c e1000_i210.c PCIE_SHARED = e1000_80003es2lan.c e1000_ich8lan.c e1000_82571.c e1000_82575.c LEGACY_SHARED = e1000_82540.c e1000_82542.c e1000_82541.c e1000_82543.c + CFLAGS += -I${.CURDIR}/../../dev/e1000 # DEVICE_POLLING for a non-interrupt-driven method #CFLAGS += -DDEVICE_POLLING +afterinstall: + ln -sf ${DESTDIR}${KMODDIR}/${KMOD}.ko ${DESTDIR}${KMODDIR}/if_igb.ko + .include diff --git a/sys/modules/igb/Makefile b/sys/modules/igb/Makefile deleted file mode 100644 index 4db960826593..000000000000 --- a/sys/modules/igb/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -#$FreeBSD$ - -.PATH: ${.CURDIR}/../../dev/e1000 -KMOD = if_igb -SRCS = device_if.h bus_if.h pci_if.h opt_inet.h opt_inet6.h opt_rss.h -SRCS += if_igb.c $(SHARED_SRCS) -SHARED_SRCS = e1000_api.c e1000_phy.c e1000_nvm.c e1000_mac.c e1000_manage.c -SHARED_SRCS += e1000_80003es2lan.c e1000_82542.c e1000_82541.c e1000_82543.c -SHARED_SRCS += e1000_82540.c e1000_ich8lan.c e1000_82571.c e1000_osdep.c -SHARED_SRCS += e1000_82575.c e1000_vf.c e1000_mbx.c e1000_i210.c - -CFLAGS += -I${.CURDIR}/../../dev/e1000 -DSMP - -# DEVICE_POLLING gives you non-interrupt handling -# not advisable since MSIX gives better results -#CFLAGS += -DDEVICE_POLLING - -# IGB_LEGACY_TX will override the stack if_transmit path and -# instead use the older if_start non-multiqueue capable interface. -# This might be desirable for testing, or to enable the use of -# ALTQ. -#CFLAGS += -DIGB_LEGACY_TX - -.include diff --git a/sys/modules/sdhci_acpi/Makefile b/sys/modules/sdhci_acpi/Makefile new file mode 100644 index 000000000000..0d348050d116 --- /dev/null +++ b/sys/modules/sdhci_acpi/Makefile @@ -0,0 +1,9 @@ +# $FreeBSD$ + +.PATH: ${.CURDIR}/../../dev/sdhci + +KMOD= sdhci_acpi +SRCS= sdhci_acpi.c sdhci.h sdhci_if.h +SRCS+= acpi_if.h device_if.h bus_if.h opt_acpi.h pci_if.h mmcbr_if.h + +.include diff --git a/sys/net/iflib.c b/sys/net/iflib.c index beeddcf6fabb..f1aee37ecd78 100644 --- a/sys/net/iflib.c +++ b/sys/net/iflib.c @@ -447,10 +447,6 @@ struct iflib_rxq { static int enable_msix = 1; -#define mtx_held(m) (((m)->mtx_lock & ~MTX_FLAGMASK) != (uintptr_t)0) - - - #define CTX_ACTIVE(ctx) ((if_getdrvflags((ctx)->ifc_ifp) & IFF_DRV_RUNNING)) #define CTX_LOCK_INIT(_sc, _name) mtx_init(&(_sc)->ifc_mtx, _name, "iflib ctx lock", MTX_DEF) diff --git a/sys/net80211/_ieee80211.h b/sys/net80211/_ieee80211.h index 13155ea399a3..12f3ad8c84d2 100644 --- a/sys/net80211/_ieee80211.h +++ b/sys/net80211/_ieee80211.h @@ -313,6 +313,12 @@ struct ieee80211_channel { #define IEEE80211_IS_CHAN_VHT(_c) \ (((_c)->ic_flags & IEEE80211_CHAN_VHT) != 0) +#define IEEE80211_IS_CHAN_VHT_2GHZ(_c) \ + (IEEE80211_IS_CHAN_2GHZ(_c) && \ + ((_c)->ic_flags & IEEE80211_CHAN_VHT) != 0) +#define IEEE80211_IS_CHAN_VHT_5GHZ(_c) \ + (IEEE80211_IS_CHAN_5GHZ(_c) && \ + ((_c)->ic_flags & IEEE80211_CHAN_VHT) != 0) #define IEEE80211_IS_CHAN_VHT20(_c) \ (((_c)->ic_flags & IEEE80211_CHAN_VHT20) != 0) #define IEEE80211_IS_CHAN_VHT40(_c) \ diff --git a/sys/net80211/ieee80211.c b/sys/net80211/ieee80211.c index e4990868f06c..487e1505ef08 100644 --- a/sys/net80211/ieee80211.c +++ b/sys/net80211/ieee80211.c @@ -70,6 +70,8 @@ const char *ieee80211_phymode_name[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_QUARTER] = "quarter", [IEEE80211_MODE_11NA] = "11na", [IEEE80211_MODE_11NG] = "11ng", + [IEEE80211_MODE_VHT_2GHZ] = "11acg", + [IEEE80211_MODE_VHT_5GHZ] = "11ac", }; /* map ieee80211_opmode to the corresponding capability bit */ const int ieee80211_opcap[IEEE80211_OPMODE_MAX] = { @@ -181,6 +183,10 @@ ieee80211_chan_init(struct ieee80211com *ic) setbit(ic->ic_modecaps, IEEE80211_MODE_11NA); if (IEEE80211_IS_CHAN_HTG(c)) setbit(ic->ic_modecaps, IEEE80211_MODE_11NG); + if (IEEE80211_IS_CHAN_VHTA(c)) + setbit(ic->ic_modecaps, IEEE80211_MODE_VHT_5GHZ); + if (IEEE80211_IS_CHAN_VHTG(c)) + setbit(ic->ic_modecaps, IEEE80211_MODE_VHT_2GHZ); } /* initialize candidate channels to all available */ memcpy(ic->ic_chan_active, ic->ic_chan_avail, @@ -208,6 +214,8 @@ ieee80211_chan_init(struct ieee80211com *ic) DEFAULTRATES(IEEE80211_MODE_QUARTER, ieee80211_rateset_quarter); DEFAULTRATES(IEEE80211_MODE_11NA, ieee80211_rateset_11a); DEFAULTRATES(IEEE80211_MODE_11NG, ieee80211_rateset_11g); + DEFAULTRATES(IEEE80211_MODE_VHT_2GHZ, ieee80211_rateset_11g); + DEFAULTRATES(IEEE80211_MODE_VHT_5GHZ, ieee80211_rateset_11a); /* * Setup required information to fill the mcsset field, if driver did @@ -1492,6 +1500,8 @@ addmedia(struct ifmedia *media, int caps, int addsta, int mode, int mword) [IEEE80211_MODE_QUARTER] = IFM_IEEE80211_11A, /* XXX */ [IEEE80211_MODE_11NA] = IFM_IEEE80211_11NA, [IEEE80211_MODE_11NG] = IFM_IEEE80211_11NG, + [IEEE80211_MODE_VHT_2GHZ] = IFM_IEEE80211_VHT2G, + [IEEE80211_MODE_VHT_5GHZ] = IFM_IEEE80211_VHT5G, }; u_int mopt; @@ -1604,6 +1614,19 @@ ieee80211_media_setup(struct ieee80211com *ic, if (rate > maxrate) maxrate = rate; } + + /* + * Add VHT media. + */ + for (; mode <= IEEE80211_MODE_VHT_5GHZ; mode++) { + if (isclr(ic->ic_modecaps, mode)) + continue; + addmedia(media, caps, addsta, mode, IFM_AUTO); + addmedia(media, caps, addsta, mode, IFM_IEEE80211_VHT); + + /* XXX TODO: VHT maxrate */ + } + return maxrate; } @@ -1883,7 +1906,11 @@ enum ieee80211_phymode ieee80211_chan2mode(const struct ieee80211_channel *chan) { - if (IEEE80211_IS_CHAN_HTA(chan)) + if (IEEE80211_IS_CHAN_VHT_2GHZ(chan)) + return IEEE80211_MODE_VHT_2GHZ; + else if (IEEE80211_IS_CHAN_VHT_5GHZ(chan)) + return IEEE80211_MODE_VHT_5GHZ; + else if (IEEE80211_IS_CHAN_HTA(chan)) return IEEE80211_MODE_11NA; else if (IEEE80211_IS_CHAN_HTG(chan)) return IEEE80211_MODE_11NG; diff --git a/sys/net80211/ieee80211.h b/sys/net80211/ieee80211.h index aa2ddb0968ec..93740eec36b3 100644 --- a/sys/net80211/ieee80211.h +++ b/sys/net80211/ieee80211.h @@ -798,37 +798,73 @@ struct ieee80211_ie_vht_operation { #define IEEE80211_VHTCAP_MAX_MPDU_LENGTH_7991 0x00000001 #define IEEE80211_VHTCAP_MAX_MPDU_LENGTH_11454 0x00000002 #define IEEE80211_VHTCAP_MAX_MPDU_MASK 0x00000003 -#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160MHZ 0x00000004 -#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ 0x00000008 +#define IEEE80211_VHTCAP_MAX_MPDU_MASK_S 0 + #define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK 0x0000000C +#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK_S 2 +#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_NONE 0 +#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160MHZ 1 +#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160_80P80MHZ 2 +#define IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_RESERVED 3 + #define IEEE80211_VHTCAP_RXLDPC 0x00000010 +#define IEEE80211_VHTCAP_RXLDPC_S 4 + #define IEEE80211_VHTCAP_SHORT_GI_80 0x00000020 +#define IEEE80211_VHTCAP_SHORT_GI_80_S 5 + #define IEEE80211_VHTCAP_SHORT_GI_160 0x00000040 +#define IEEE80211_VHTCAP_SHORT_GI_160_S 6 + #define IEEE80211_VHTCAP_TXSTBC 0x00000080 +#define IEEE80211_VHTCAP_TXSTBC_S 7 + #define IEEE80211_VHTCAP_RXSTBC_1 0x00000100 #define IEEE80211_VHTCAP_RXSTBC_2 0x00000200 #define IEEE80211_VHTCAP_RXSTBC_3 0x00000300 #define IEEE80211_VHTCAP_RXSTBC_4 0x00000400 #define IEEE80211_VHTCAP_RXSTBC_MASK 0x00000700 +#define IEEE80211_VHTCAP_RXSTBC_MASK_S 8 + #define IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE 0x00000800 +#define IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE_S 11 + #define IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE 0x00001000 +#define IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE_S 12 + #define IEEE80211_VHTCAP_BEAMFORMEE_STS_SHIFT 13 #define IEEE80211_VHTCAP_BEAMFORMEE_STS_MASK \ (7 << IEEE80211_VHTCAP_BEAMFORMEE_STS_SHIFT) +#define IEEE80211_VHTCAP_BEAMFORMEE_STS_MASK_S 13 + #define IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_SHIFT 16 #define IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_MASK \ (7 << IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_SHIFT) +#define IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_MASK_S 16 + #define IEEE80211_VHTCAP_MU_BEAMFORMER_CAPABLE 0x00080000 +#define IEEE80211_VHTCAP_MU_BEAMFORMER_CAPABLE_S 19 #define IEEE80211_VHTCAP_MU_BEAMFORMEE_CAPABLE 0x00100000 +#define IEEE80211_VHTCAP_MU_BEAMFORMEE_CAPABLE_S 20 #define IEEE80211_VHTCAP_VHT_TXOP_PS 0x00200000 +#define IEEE80211_VHTCAP_VHT_TXOP_PS_S 21 #define IEEE80211_VHTCAP_HTC_VHT 0x00400000 +#define IEEE80211_VHTCAP_HTC_VHT_S 22 + #define IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT 23 #define IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK \ (7 << IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT) +#define IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK_S 23 + +#define IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MASK 0x0c000000 #define IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_UNSOL_MFB 0x08000000 #define IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB 0x0c000000 +#define IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MASK_S 26 + #define IEEE80211_VHTCAP_RX_ANTENNA_PATTERN 0x10000000 +#define IEEE80211_VHTCAP_RX_ANTENNA_PATTERN_S 28 #define IEEE80211_VHTCAP_TX_ANTENNA_PATTERN 0x20000000 +#define IEEE80211_VHTCAP_TX_ANTENNA_PATTERN_S 29 /* * XXX TODO: add the rest of the bits diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c index 36070b7f1947..275401fa5747 100644 --- a/sys/net80211/ieee80211_proto.c +++ b/sys/net80211/ieee80211_proto.c @@ -842,6 +842,9 @@ setbasicrates(struct ieee80211_rateset *rs, [IEEE80211_MODE_11NA] = { 3, { 12, 24, 48 } }, /* NB: mixed b/g */ [IEEE80211_MODE_11NG] = { 4, { 2, 4, 11, 22 } }, + /* NB: mixed b/g */ + [IEEE80211_MODE_VHT_2GHZ] = { 4, { 2, 4, 11, 22 } }, + [IEEE80211_MODE_VHT_5GHZ] = { 3, { 12, 24, 48 } }, }; int i, j; @@ -906,6 +909,8 @@ static const struct phyParamType phyParamForAC_BE[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_QUARTER]= { 3, 4, 6, 0, 0 }, [IEEE80211_MODE_11NA] = { 3, 4, 6, 0, 0 }, [IEEE80211_MODE_11NG] = { 3, 4, 6, 0, 0 }, + [IEEE80211_MODE_VHT_2GHZ] = { 3, 4, 6, 0, 0 }, + [IEEE80211_MODE_VHT_5GHZ] = { 3, 4, 6, 0, 0 }, }; static const struct phyParamType phyParamForAC_BK[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_AUTO] = { 7, 4, 10, 0, 0 }, @@ -920,6 +925,8 @@ static const struct phyParamType phyParamForAC_BK[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_QUARTER]= { 7, 4, 10, 0, 0 }, [IEEE80211_MODE_11NA] = { 7, 4, 10, 0, 0 }, [IEEE80211_MODE_11NG] = { 7, 4, 10, 0, 0 }, + [IEEE80211_MODE_VHT_2GHZ] = { 7, 4, 10, 0, 0 }, + [IEEE80211_MODE_VHT_5GHZ] = { 7, 4, 10, 0, 0 }, }; static const struct phyParamType phyParamForAC_VI[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_AUTO] = { 1, 3, 4, 94, 0 }, @@ -934,6 +941,8 @@ static const struct phyParamType phyParamForAC_VI[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_QUARTER]= { 1, 3, 4, 94, 0 }, [IEEE80211_MODE_11NA] = { 1, 3, 4, 94, 0 }, [IEEE80211_MODE_11NG] = { 1, 3, 4, 94, 0 }, + [IEEE80211_MODE_VHT_2GHZ] = { 1, 3, 4, 94, 0 }, + [IEEE80211_MODE_VHT_5GHZ] = { 1, 3, 4, 94, 0 }, }; static const struct phyParamType phyParamForAC_VO[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_AUTO] = { 1, 2, 3, 47, 0 }, @@ -948,6 +957,8 @@ static const struct phyParamType phyParamForAC_VO[IEEE80211_MODE_MAX] = { [IEEE80211_MODE_QUARTER]= { 1, 2, 3, 47, 0 }, [IEEE80211_MODE_11NA] = { 1, 2, 3, 47, 0 }, [IEEE80211_MODE_11NG] = { 1, 2, 3, 47, 0 }, + [IEEE80211_MODE_VHT_2GHZ] = { 1, 2, 3, 47, 0 }, + [IEEE80211_MODE_VHT_5GHZ] = { 1, 2, 3, 47, 0 }, }; static const struct phyParamType bssPhyParamForAC_BE[IEEE80211_MODE_MAX] = { @@ -1123,6 +1134,8 @@ ieee80211_wme_updateparams_locked(struct ieee80211vap *vap) [IEEE80211_MODE_QUARTER] = { 2, 4, 10, 64, 0 }, [IEEE80211_MODE_11NA] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/ [IEEE80211_MODE_11NG] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/ + [IEEE80211_MODE_VHT_2GHZ] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/ + [IEEE80211_MODE_VHT_5GHZ] = { 2, 4, 10, 64, 0 }, /* XXXcheck*/ }; struct ieee80211com *ic = vap->iv_ic; struct ieee80211_wme_state *wme = &ic->ic_wme; @@ -1243,6 +1256,8 @@ ieee80211_wme_updateparams_locked(struct ieee80211vap *vap) [IEEE80211_MODE_QUARTER] = 3, [IEEE80211_MODE_11NA] = 3, [IEEE80211_MODE_11NG] = 3, + [IEEE80211_MODE_VHT_2GHZ] = 3, + [IEEE80211_MODE_VHT_5GHZ] = 3, }; chanp = &wme->wme_chanParams.cap_wmeParams[WME_AC_BE]; bssp = &wme->wme_bssChanParams.cap_wmeParams[WME_AC_BE]; diff --git a/sys/net80211/ieee80211_tdma.c b/sys/net80211/ieee80211_tdma.c index aac8c7fd55c1..93eba3b2efe8 100644 --- a/sys/net80211/ieee80211_tdma.c +++ b/sys/net80211/ieee80211_tdma.c @@ -176,6 +176,8 @@ ieee80211_tdma_vattach(struct ieee80211vap *vap) settxparms(vap, IEEE80211_MODE_11NG, TDMA_TXRATE_11NG_DEFAULT); settxparms(vap, IEEE80211_MODE_HALF, TDMA_TXRATE_HALF_DEFAULT); settxparms(vap, IEEE80211_MODE_QUARTER, TDMA_TXRATE_QUARTER_DEFAULT); + settxparms(vap, IEEE80211_MODE_VHT_2GHZ, TDMA_TXRATE_11NG_DEFAULT); + settxparms(vap, IEEE80211_MODE_VHT_5GHZ, TDMA_TXRATE_11NA_DEFAULT); setackpolicy(vap->iv_ic, 1); /* disable ACK's */ diff --git a/sys/net80211/ieee80211_vht.c b/sys/net80211/ieee80211_vht.c index 274d94b25208..e50c11c5edcd 100644 --- a/sys/net80211/ieee80211_vht.c +++ b/sys/net80211/ieee80211_vht.c @@ -70,15 +70,67 @@ __FBSDID("$FreeBSD$"); frm += 4; \ } while (0) +/* + * Immediate TODO: + * + * + handle WLAN_ACTION_VHT_OPMODE_NOTIF and other VHT action frames + * + ensure vhtinfo/vhtcap parameters correctly use the negotiated + * capabilities and ratesets + * + group ID management operation + */ + /* * XXX TODO: handle WLAN_ACTION_VHT_OPMODE_NOTIF * * Look at mac80211/vht.c:ieee80211_vht_handle_opmode() for further details. */ +static int +vht_recv_action_placeholder(struct ieee80211_node *ni, + const struct ieee80211_frame *wh, + const uint8_t *frm, const uint8_t *efrm) +{ + +#ifdef IEEE80211_DEBUG + ieee80211_note(ni->ni_vap, "%s: called; fc=0x%.2x/0x%.2x", + __func__, + wh->i_fc[0], + wh->i_fc[1]); +#endif + return (0); +} + +static int +vht_send_action_placeholder(struct ieee80211_node *ni, + int category, int action, void *arg0) +{ + +#ifdef IEEE80211_DEBUG + ieee80211_note(ni->ni_vap, "%s: called; category=%d, action=%d", + __func__, + category, + action); +#endif + return (EINVAL); +} + static void ieee80211_vht_init(void) { + + ieee80211_recv_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_COMPRESSED_BF, vht_recv_action_placeholder); + ieee80211_recv_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_GROUPID_MGMT, vht_recv_action_placeholder); + ieee80211_recv_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_OPMODE_NOTIF, vht_recv_action_placeholder); + + ieee80211_send_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_COMPRESSED_BF, vht_send_action_placeholder); + ieee80211_send_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_GROUPID_MGMT, vht_send_action_placeholder); + ieee80211_send_action_register(IEEE80211_ACTION_CAT_VHT, + WLAN_ACTION_VHT_OPMODE_NOTIF, vht_send_action_placeholder); } SYSINIT(wlan_vht, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_vht_init, NULL); @@ -153,9 +205,9 @@ ieee80211_vht_announce(struct ieee80211com *ic) /* Channel width */ ic_printf(ic, "[VHT] Channel Widths: 20MHz, 40MHz, 80MHz"); - if (ic->ic_vhtcaps & IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ) + if (MS(ic->ic_vhtcaps, IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK) == 2) printf(" 80+80MHz"); - if (ic->ic_vhtcaps & IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_160MHZ) + if (MS(ic->ic_vhtcaps, IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK) >= 1) printf(" 160MHz"); printf("\n"); @@ -280,10 +332,342 @@ ieee80211_vht_node_leave(struct ieee80211_node *ni) "%s: called", __func__); } +/* + * Calculate the VHTCAP IE for a given node. + * + * This includes calculating the capability intersection based on the + * current operating mode and intersection of the TX/RX MCS maps. + * + * The standard only makes it clear about MCS rate negotiation + * and MCS basic rates (which must be a subset of the general + * negotiated rates). It doesn't make it clear that the AP should + * figure out the minimum functional overlap with the STA and + * support that. + * + * Note: this is in host order, not in 802.11 endian order. + * + * TODO: ensure I re-read 9.7.11 Rate Selection for VHT STAs. + * + * TODO: investigate what we should negotiate for MU-MIMO beamforming + * options. + * + * opmode is '1' for "vhtcap as if I'm a STA", 0 otherwise. + */ +void +ieee80211_vht_get_vhtcap_ie(struct ieee80211_node *ni, + struct ieee80211_ie_vhtcap *vhtcap, int opmode) +{ + struct ieee80211vap *vap = ni->ni_vap; +// struct ieee80211com *ic = vap->iv_ic; + uint32_t val, val1, val2; + uint32_t new_vhtcap; + int i; + + vhtcap->ie = IEEE80211_ELEMID_VHT_CAP; + vhtcap->len = sizeof(struct ieee80211_ie_vhtcap) - 2; + + /* + * Capabilities - it depends on whether we are a station + * or not. + */ + new_vhtcap = 0; + + /* + * Station - use our desired configuration based on + * local config, local device bits and the already-learnt + * vhtcap/vhtinfo IE in the node. + */ + + /* Limit MPDU size to the smaller of the two */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_MAX_MPDU_MASK); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_MAX_MPDU_MASK); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_MAX_MPDU_MASK); + + /* Limit supp channel config */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK); + } + if ((val2 == 2) && + ((vap->iv_flags_vht & IEEE80211_FVHT_USEVHT80P80) == 0)) + val2 = 1; + if ((val2 == 1) && + ((vap->iv_flags_vht & IEEE80211_FVHT_USEVHT160) == 0)) + val2 = 0; + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SUPP_CHAN_WIDTH_MASK); + + /* RX LDPC */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_RXLDPC); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_RXLDPC); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_RXLDPC); + + /* Short-GI 80 */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_SHORT_GI_80); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_SHORT_GI_80); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SHORT_GI_80); + + /* Short-GI 160 */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_SHORT_GI_160); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_SHORT_GI_160); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SHORT_GI_160); + + /* + * STBC is slightly more complicated. + * + * In non-STA mode, we just announce our capabilities and that + * is that. + * + * In STA mode, we should calculate our capabilities based on + * local capabilities /and/ what the remote says. So: + * + * + Only TX STBC if we support it and the remote supports RX STBC; + * + Only announce RX STBC if we support it and the remote supports + * TX STBC; + * + RX STBC should be the minimum of local and remote RX STBC; + */ + + /* TX STBC */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_TXSTBC); + if (opmode == 1) { + /* STA mode - enable it only if node RXSTBC is non-zero */ + val2 = !! MS(ni->ni_vhtcap, IEEE80211_VHTCAP_RXSTBC_MASK); + } + val = MIN(val1, val2); + /* XXX For now, use the 11n config flag */ + if ((vap->iv_flags_ht & IEEE80211_FHT_STBC_TX) == 0) + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_TXSTBC); + + /* RX STBC1..4 */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_RXSTBC_MASK); + if (opmode == 1) { + /* STA mode - enable it only if node TXSTBC is non-zero */ + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_TXSTBC); + } + val = MIN(val1, val2); + /* XXX For now, use the 11n config flag */ + if ((vap->iv_flags_ht & IEEE80211_FHT_STBC_RX) == 0) + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_RXSTBC_MASK); + + /* + * Finally - if RXSTBC is 0, then don't enable TXSTBC. + * Strictly speaking a device can TXSTBC and not RXSTBC, but + * it would be silly. + */ + if (val == 0) + new_vhtcap &= ~IEEE80211_VHTCAP_TXSTBC; + + /* + * Some of these fields require other fields to exist. + * So before using it, the parent field needs to be checked + * otherwise the overridden value may be wrong. + * + * For example, if SU beamformee is set to 0, then BF STS + * needs to be 0. + */ + + /* SU Beamformer capable */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE); + + /* SU Beamformee capable */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE); + } + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE); + + /* Beamformee STS capability - only if SU beamformee capable */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_BEAMFORMEE_STS_MASK); + if (opmode == 1) { + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_BEAMFORMEE_STS_MASK); + } + val = MIN(val1, val2); + if ((new_vhtcap & IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE) == 0) + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_BEAMFORMEE_STS_MASK); + + /* Sounding dimensions - only if SU beamformer capable */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_MASK); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_MASK); + val = MIN(val1, val2); + if ((new_vhtcap & IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE) == 0) + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SOUNDING_DIMENSIONS_MASK); + + /* + * MU Beamformer capable - only if SU BFF capable, MU BFF capable + * and STA (not AP) + */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_MU_BEAMFORMER_CAPABLE); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_MU_BEAMFORMER_CAPABLE); + val = MIN(val1, val2); + if ((new_vhtcap & IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE) == 0) + val = 0; + if (opmode != 1) /* Only enable for STA mode */ + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SU_BEAMFORMER_CAPABLE); + + /* + * MU Beamformee capable - only if SU BFE capable, MU BFE capable + * and AP (not STA) + */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_MU_BEAMFORMEE_CAPABLE); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_MU_BEAMFORMEE_CAPABLE); + val = MIN(val1, val2); + if ((new_vhtcap & IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE) == 0) + val = 0; + if (opmode != 0) /* Only enable for AP mode */ + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_SU_BEAMFORMEE_CAPABLE); + + /* VHT TXOP PS */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_VHT_TXOP_PS); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_VHT_TXOP_PS); + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_VHT_TXOP_PS); + + /* HTC_VHT */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_HTC_VHT); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_HTC_VHT); + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_HTC_VHT); + + /* A-MPDU length max */ + /* XXX TODO: we need a userland config knob for this */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK); + val = MIN(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK); + + /* + * Link adaptation is only valid if HTC-VHT capable is 1. + * Otherwise, always set it to 0. + */ + val2 = val1 = MS(vap->iv_vhtcaps, + IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MASK); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, + IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MASK); + val = MIN(val1, val2); + if ((new_vhtcap & IEEE80211_VHTCAP_HTC_VHT) == 0) + val = 0; + new_vhtcap |= SM(val, IEEE80211_VHTCAP_VHT_LINK_ADAPTATION_VHT_MASK); + + /* + * The following two options are 0 if the pattern may change, 1 if it + * does not change. So, downgrade to the higher value. + */ + + /* RX antenna pattern */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_RX_ANTENNA_PATTERN); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_RX_ANTENNA_PATTERN); + val = MAX(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_RX_ANTENNA_PATTERN); + + /* TX antenna pattern */ + val2 = val1 = MS(vap->iv_vhtcaps, IEEE80211_VHTCAP_TX_ANTENNA_PATTERN); + if (opmode == 1) + val2 = MS(ni->ni_vhtcap, IEEE80211_VHTCAP_TX_ANTENNA_PATTERN); + val = MAX(val1, val2); + new_vhtcap |= SM(val, IEEE80211_VHTCAP_TX_ANTENNA_PATTERN); + + /* + * MCS set - again, we announce what we want to use + * based on configuration, device capabilities and + * already-learnt vhtcap/vhtinfo IE information. + */ + + /* MCS set - start with whatever the device supports */ + vhtcap->supp_mcs.rx_mcs_map = vap->iv_vht_mcsinfo.rx_mcs_map; + vhtcap->supp_mcs.rx_highest = 0; + vhtcap->supp_mcs.tx_mcs_map = vap->iv_vht_mcsinfo.tx_mcs_map; + vhtcap->supp_mcs.tx_highest = 0; + + vhtcap->vht_cap_info = new_vhtcap; + + /* + * Now, if we're a STA, mask off whatever the AP doesn't support. + * Ie, we continue to state we can receive whatever we can do, + * but we only announce that we will transmit rates that meet + * the AP requirement. + * + * Note: 0 - MCS0..7; 1 - MCS0..8; 2 - MCS0..9; 3 = not supported. + * We can't just use MIN() because '3' means "no", so special case it. + */ + if (opmode) { + for (i = 0; i < 8; i++) { + val1 = (vhtcap->supp_mcs.tx_mcs_map >> (i*2)) & 0x3; + val2 = (ni->ni_vht_mcsinfo.tx_mcs_map >> (i*2)) & 0x3; + val = MIN(val1, val2); + if (val1 == 3 || val2 == 3) + val = 3; + vhtcap->supp_mcs.tx_mcs_map &= ~(0x3 << (i*2)); + vhtcap->supp_mcs.tx_mcs_map |= (val << (i*2)); + } + } +} + +/* + * Add a VHTCAP field. + * + * If in station mode, we announce what we would like our + * desired configuration to be. + * + * Else, we announce our capabilities based on our current + * configuration. + */ uint8_t * ieee80211_add_vhtcap(uint8_t *frm, struct ieee80211_node *ni) { - uint32_t cap; + struct ieee80211_ie_vhtcap vhtcap; + int opmode; + + opmode = 0; + if (ni->ni_vap->iv_opmode == IEEE80211_M_STA) + opmode = 1; + + ieee80211_vht_get_vhtcap_ie(ni, &vhtcap, opmode); memset(frm, '\0', sizeof(struct ieee80211_ie_vhtcap)); @@ -291,27 +675,14 @@ ieee80211_add_vhtcap(uint8_t *frm, struct ieee80211_node *ni) frm[1] = sizeof(struct ieee80211_ie_vhtcap) - 2; frm += 2; - /* - * For now, don't do any configuration. - * Just populate the node configuration. - * We can worry about making it configurable later. - */ - - cap = ni->ni_vhtcap; - - /* - * XXX TODO: any capability changes required by - * configuration. - */ - /* 32-bit VHT capability */ - ADDWORD(frm, cap); + ADDWORD(frm, vhtcap.vht_cap_info); /* suppmcs */ - ADDSHORT(frm, ni->ni_vht_mcsinfo.rx_mcs_map); - ADDSHORT(frm, ni->ni_vht_mcsinfo.rx_highest); - ADDSHORT(frm, ni->ni_vht_mcsinfo.tx_mcs_map); - ADDSHORT(frm, ni->ni_vht_mcsinfo.tx_highest); + ADDSHORT(frm, vhtcap.supp_mcs.rx_mcs_map); + ADDSHORT(frm, vhtcap.supp_mcs.rx_highest); + ADDSHORT(frm, vhtcap.supp_mcs.tx_mcs_map); + ADDSHORT(frm, vhtcap.supp_mcs.tx_highest); return (frm); } @@ -370,17 +741,6 @@ ieee80211_add_vhtinfo(uint8_t *frm, struct ieee80211_node *ni) frm[1] = sizeof(struct ieee80211_ie_vht_operation) - 2; frm += 2; - /* - * XXX if it's a station, then see if we have a node - * channel or ANYC. If it's ANYC then assume we're - * scanning, and announce our capabilities. - * - * This should set the "20/40/80/160MHz wide config"; - * the 80/80 or 160MHz wide config is done in VHTCAP. - * - * Other modes - just limit it to the channel. - */ - /* 8-bit chanwidth */ *frm++ = ieee80211_vht_get_chwidth_ie(ni->ni_chan); @@ -475,3 +835,19 @@ ieee80211_vht_adjust_channel(struct ieee80211com *ic, #endif return (chan); } + +/* + * Calculate the VHT operation IE for a given node. + * + * This includes calculating the suitable channel width/parameters + * and basic MCS set. + * + * TODO: ensure I read 9.7.11 Rate Selection for VHT STAs. + * TODO: ensure I read 10.39.7 - BSS Basic VHT-MCS and NSS set operation. + */ +void +ieee80211_vht_get_vhtinfo_ie(struct ieee80211_node *ni, + struct ieee80211_ie_vht_operation *vhtop, int opmode) +{ + printf("%s: called; TODO!\n", __func__); +} diff --git a/sys/net80211/ieee80211_vht.h b/sys/net80211/ieee80211_vht.h index 9e68e7d258d8..791762b14a31 100644 --- a/sys/net80211/ieee80211_vht.h +++ b/sys/net80211/ieee80211_vht.h @@ -60,4 +60,9 @@ struct ieee80211_channel * ieee80211_vht_adjust_channel(struct ieee80211com *, struct ieee80211_channel *, int); +void ieee80211_vht_get_vhtcap_ie(struct ieee80211_node *ni, + struct ieee80211_ie_vhtcap *, int); +void ieee80211_vht_get_vhtinfo_ie(struct ieee80211_node *ni, + struct ieee80211_ie_vht_operation *, int); + #endif /* _NET80211_IEEE80211_VHT_H_ */ diff --git a/sys/netpfil/ipfw/ip_dummynet.c b/sys/netpfil/ipfw/ip_dummynet.c index f6d9c28237f5..7240a99e5c44 100644 --- a/sys/netpfil/ipfw/ip_dummynet.c +++ b/sys/netpfil/ipfw/ip_dummynet.c @@ -931,29 +931,35 @@ delete_schk(int i) static int copy_obj(char **start, char *end, void *_o, const char *msg, int i) { - struct dn_id *o = _o; + struct dn_id o; + union { + struct dn_link l; + struct dn_schk s; + } dn; int have = end - *start; - if (have < o->len || o->len == 0 || o->type == 0) { + memcpy(&o, _o, sizeof(o)); + if (have < o.len || o.len == 0 || o.type == 0) { D("(WARN) type %d %s %d have %d need %d", - o->type, msg, i, have, o->len); + o.type, msg, i, have, o.len); return 1; } - ND("type %d %s %d len %d", o->type, msg, i, o->len); - bcopy(_o, *start, o->len); - if (o->type == DN_LINK) { + ND("type %d %s %d len %d", o.type, msg, i, o.len); + if (o.type == DN_LINK) { + memcpy(&dn.l, _o, sizeof(dn.l)); /* Adjust burst parameter for link */ - struct dn_link *l = (struct dn_link *)*start; - l->burst = div64(l->burst, 8 * hz); - l->delay = l->delay * 1000 / hz; - } else if (o->type == DN_SCH) { - /* Set id->id to the number of instances */ - struct dn_schk *s = _o; - struct dn_id *id = (struct dn_id *)(*start); - id->id = (s->sch.flags & DN_HAVE_MASK) ? - dn_ht_entries(s->siht) : (s->siht ? 1 : 0); - } - *start += o->len; + dn.l.burst = div64(dn.l.burst, 8 * hz); + dn.l.delay = dn.l.delay * 1000 / hz; + memcpy(*start, &dn.l, sizeof(dn.l)); + } else if (o.type == DN_SCH) { + /* Set dn.s.sch.oid.id to the number of instances */ + memcpy(&dn.s, _o, sizeof(dn.s)); + dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ? + dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0); + memcpy(*start, &dn.s, sizeof(dn.s)); + } else + memcpy(*start, _o, o.len); + *start += o.len; return 0; } @@ -974,7 +980,7 @@ copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) return 1; } ND("type %d %s %d len %d", o->type, msg, i, len); - bcopy(_o, *start, len); + memcpy(*start, _o, len); ((struct dn_id*)(*start))->len = len; *start += len; return 0; @@ -1022,7 +1028,7 @@ copy_profile(struct copy_args *a, struct dn_profile *p) D("error have %d need %d", have, profile_len); return 1; } - bcopy(p, *a->start, profile_len); + memcpy(*a->start, p, profile_len); ((struct dn_id *)(*a->start))->len = profile_len; *a->start += profile_len; return 0; @@ -1584,6 +1590,9 @@ config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) { int i; struct dn_fsk *fs; +#ifdef NEW_AQM + struct dn_extra_parms *ep; +#endif if (nfs->oid.len != sizeof(*nfs)) { D("invalid flowset len %d", nfs->oid.len); @@ -1592,6 +1601,15 @@ config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) i = nfs->fs_nr; if (i <= 0 || i >= 3*DN_MAX_ID) return NULL; +#ifdef NEW_AQM + ep = NULL; + if (arg != NULL) { + ep = malloc(sizeof(*ep), M_TEMP, locked ? M_NOWAIT : M_WAITOK); + if (ep == NULL) + return (NULL); + memcpy(ep, arg, sizeof(*ep)); + } +#endif ND("flowset %d", i); /* XXX other sanity checks */ if (nfs->flags & DN_QSIZE_BYTES) { @@ -1630,12 +1648,15 @@ config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { ND("flowset %d unchanged", i); #ifdef NEW_AQM - /* reconfigure AQM as the parameters can be changed. - * we consider the flowsetis busy if it has scheduler instance(s) - */ - s = locate_scheduler(nfs->sched_nr); - config_aqm(fs, (struct dn_extra_parms *) arg, - s != NULL && s->siht != NULL); + if (ep != NULL) { + /* + * Reconfigure AQM as the parameters can be changed. + * We consider the flowset as busy if it has scheduler + * instance(s). + */ + s = locate_scheduler(nfs->sched_nr); + config_aqm(fs, ep, s != NULL && s->siht != NULL); + } #endif break; /* no change, nothing to do */ } @@ -1657,13 +1678,19 @@ config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) fs->fs = *nfs; /* copy configuration */ #ifdef NEW_AQM fs->aqmfp = NULL; - config_aqm(fs, (struct dn_extra_parms *) arg, s != NULL && s->siht != NULL); + if (ep != NULL) + config_aqm(fs, ep, s != NULL && + s->siht != NULL); #endif if (s != NULL) fsk_attach(fs, s); } while (0); if (!locked) DN_BH_WUNLOCK(); +#ifdef NEW_AQM + if (ep != NULL) + free(ep, M_TEMP); +#endif return fs; } @@ -1773,7 +1800,7 @@ config_sched(struct dn_sch *_nsch, struct dn_id *arg) D("cannot allocate profile"); goto error; //XXX } - bcopy(pf, s->profile, sizeof(*pf)); + memcpy(s->profile, pf, sizeof(*pf)); } } p.link_nr = 0; @@ -1795,7 +1822,7 @@ config_sched(struct dn_sch *_nsch, struct dn_id *arg) pf = malloc(sizeof(*pf), M_DUMMYNET, M_NOWAIT | M_ZERO); if (pf) /* XXX should issue a warning otherwise */ - bcopy(s->profile, pf, sizeof(*pf)); + memcpy(pf, s->profile, sizeof(*pf)); } /* remove from the hash */ dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); @@ -1917,7 +1944,7 @@ config_profile(struct dn_profile *pf, struct dn_id *arg) olen = s->profile->oid.len; if (olen < pf->oid.len) olen = pf->oid.len; - bcopy(pf, s->profile, pf->oid.len); + memcpy(s->profile, pf, pf->oid.len); s->profile->oid.len = olen; } DN_BH_WUNLOCK(); @@ -1953,30 +1980,35 @@ dummynet_flush(void) int do_config(void *p, int l) { - struct dn_id *next, *o; - int err = 0, err2 = 0; - struct dn_id *arg = NULL; - uintptr_t *a; + struct dn_id o; + union { + struct dn_profile profile; + struct dn_fs fs; + struct dn_link link; + struct dn_sch sched; + } *dn; + struct dn_id *arg; + uintptr_t a; + int err, err2, off; - o = p; - if (o->id != DN_API_VERSION) { - D("invalid api version got %d need %d", - o->id, DN_API_VERSION); + memcpy(&o, p, sizeof(o)); + if (o.id != DN_API_VERSION) { + D("invalid api version got %d need %d", o.id, DN_API_VERSION); return EINVAL; } - for (; l >= sizeof(*o); o = next) { - struct dn_id *prev = arg; - if (o->len < sizeof(*o) || l < o->len) { - D("bad len o->len %d len %d", o->len, l); + arg = NULL; + dn = NULL; + for (off = 0; l >= sizeof(o); memcpy(&o, (char *)p + off, sizeof(o))) { + if (o.len < sizeof(o) || l < o.len) { + D("bad len o.len %d len %d", o.len, l); err = EINVAL; break; } - l -= o->len; - next = (struct dn_id *)((char *)o + o->len); + l -= o.len; err = 0; - switch (o->type) { + switch (o.type) { default: - D("cmd %d not implemented", o->type); + D("cmd %d not implemented", o.type); break; #ifdef EMULATE_SYSCTL @@ -1994,31 +2026,30 @@ do_config(void *p, int l) case DN_CMD_DELETE: /* the argument is in the first uintptr_t after o */ - a = (uintptr_t *)(o+1); - if (o->len < sizeof(*o) + sizeof(*a)) { + if (o.len < sizeof(o) + sizeof(a)) { err = EINVAL; break; } - switch (o->subtype) { + memcpy(&a, (char *)p + off + sizeof(o), sizeof(a)); + switch (o.subtype) { case DN_LINK: /* delete base and derived schedulers */ DN_BH_WLOCK(); - err = delete_schk(*a); - err2 = delete_schk(*a + DN_MAX_ID); + err = delete_schk(a); + err2 = delete_schk(a + DN_MAX_ID); DN_BH_WUNLOCK(); if (!err) err = err2; break; default: - D("invalid delete type %d", - o->subtype); + D("invalid delete type %d", o.subtype); err = EINVAL; break; case DN_FS: - err = (*a <1 || *a >= DN_MAX_ID) ? - EINVAL : delete_fs(*a, 0) ; + err = (a < 1 || a >= DN_MAX_ID) ? + EINVAL : delete_fs(a, 0) ; break; } break; @@ -2028,28 +2059,47 @@ do_config(void *p, int l) dummynet_flush(); DN_BH_WUNLOCK(); break; - case DN_TEXT: /* store argument the next block */ - prev = NULL; - arg = o; + case DN_TEXT: /* store argument of next block */ + if (arg != NULL) + free(arg, M_TEMP); + arg = malloc(o.len, M_TEMP, M_WAITOK); + memcpy(arg, (char *)p + off, o.len); break; case DN_LINK: - err = config_link((struct dn_link *)o, arg); + if (dn == NULL) + dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); + memcpy(&dn->link, (char *)p + off, sizeof(dn->link)); + err = config_link(&dn->link, arg); break; case DN_PROFILE: - err = config_profile((struct dn_profile *)o, arg); + if (dn == NULL) + dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); + memcpy(&dn->profile, (char *)p + off, + sizeof(dn->profile)); + err = config_profile(&dn->profile, arg); break; case DN_SCH: - err = config_sched((struct dn_sch *)o, arg); + if (dn == NULL) + dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); + memcpy(&dn->sched, (char *)p + off, + sizeof(dn->sched)); + err = config_sched(&dn->sched, arg); break; case DN_FS: - err = (NULL==config_fs((struct dn_fs *)o, arg, 0)); + if (dn == NULL) + dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); + memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs)); + err = (NULL == config_fs(&dn->fs, arg, 0)); break; } - if (prev) - arg = NULL; if (err != 0) break; + off += o.len; } + if (arg != NULL) + free(arg, M_TEMP); + if (dn != NULL) + free(dn, M_TEMP); return err; } @@ -2261,7 +2311,7 @@ dummynet_get(struct sockopt *sopt, void **compat) a.type = cmd->subtype; if (compat == NULL) { - bcopy(cmd, start, sizeof(*cmd)); + memcpy(start, cmd, sizeof(*cmd)); ((struct dn_id*)(start))->len = sizeof(struct dn_id); buf = start + sizeof(*cmd); } else diff --git a/sys/powerpc/conf/GENERIC64 b/sys/powerpc/conf/GENERIC64 index 7ebcb7a8f21b..2325bb059d78 100644 --- a/sys/powerpc/conf/GENERIC64 +++ b/sys/powerpc/conf/GENERIC64 @@ -139,7 +139,6 @@ device uart_z8530 # Ethernet hardware device em # Intel PRO/1000 Gigabit Ethernet Family -device igb # Intel PRO/1000 PCIE Server Gigabit Family device ix # Intel PRO/10GbE PCIE PF Ethernet Family device ixv # Intel PRO/10GbE PCIE VF Ethernet Family device glc # Sony Playstation 3 Ethernet diff --git a/sys/powerpc/include/asm.h b/sys/powerpc/include/asm.h index 3aec5d3a4791..08aab9854c62 100644 --- a/sys/powerpc/include/asm.h +++ b/sys/powerpc/include/asm.h @@ -89,10 +89,11 @@ name: #ifdef __powerpc64__ -#define TOC_REF(name) __CONCAT(.L,name) +#define TOC_NAME_FOR_REF(name) __CONCAT(.L,name) +#define TOC_REF(name) TOC_NAME_FOR_REF(name)@toc #define TOC_ENTRY(name) \ .section ".toc","aw"; \ - TOC_REF(name): \ + TOC_NAME_FOR_REF(name): \ .tc name[TC],name #endif diff --git a/sys/sparc64/include/atomic.h b/sys/sparc64/include/atomic.h index c70e99aea70b..b2fd8cc5a101 100644 --- a/sys/sparc64/include/atomic.h +++ b/sys/sparc64/include/atomic.h @@ -219,6 +219,40 @@ atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \ } \ \ +static __inline int \ +atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s) \ +{ \ + vtype t; \ + \ + t = (vtype)atomic_cas((p), (*ep), (s), sz); \ + if (t == (*ep)) \ + return (1); \ + *ep = t; \ + return (0); \ +} \ +static __inline int \ +atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s) \ +{ \ + vtype t; \ + \ + t = (vtype)atomic_cas_acq((p), (*ep), (s), sz); \ + if (t == (*ep)) \ + return (1); \ + *ep = t; \ + return (0); \ +} \ +static __inline int \ +atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s) \ +{ \ + vtype t; \ + \ + t = (vtype)atomic_cas_rel((p), (*ep), (s), sz); \ + if (t == (*ep)) \ + return (1); \ + *ep = t; \ + return (0); \ +} \ + \ static __inline vtype \ atomic_load_ ## name(volatile ptype p) \ { \ diff --git a/sys/sys/cdefs.h b/sys/sys/cdefs.h index 453b395a806d..7db4f25542b3 100644 --- a/sys/sys/cdefs.h +++ b/sys/sys/cdefs.h @@ -543,22 +543,6 @@ __attribute__((__format__ (__strftime__, fmtarg, firstvararg))) #endif -/* - * FORTIFY_SOURCE, and perhaps other compiler-specific features, require - * the use of non-standard inlining. In general we should try to avoid - * using these but GCC-compatible compilers tend to support the extensions - * well enough to use them in limited cases. - */ -#if defined(__GNUC_GNU_INLINE__) || defined(__GNUC_STDC_INLINE__) -#if __GNUC_PREREQ__(4, 3) || __has_attribute(__artificial__) -#define __gnu_inline __attribute__((__gnu_inline__, __artificial__)) -#else -#define __gnu_inline __attribute__((__gnu_inline__)) -#endif /* artificial */ -#else -#define __gnu_inline -#endif - /* Compiler-dependent macros that rely on FreeBSD-specific extensions. */ #if defined(__FreeBSD_cc_version) && __FreeBSD_cc_version >= 300001 && \ defined(__GNUC__) && !defined(__INTEL_COMPILER) diff --git a/sys/tools/embed_mfs.sh b/sys/tools/embed_mfs.sh index 66c50c2da111..a7ac80c8f8d3 100644 --- a/sys/tools/embed_mfs.sh +++ b/sys/tools/embed_mfs.sh @@ -36,12 +36,12 @@ mfs_size=`stat -f '%z' $2 2> /dev/null` # If we can't determine MFS image size - bail. [ -z ${mfs_size} ] && echo "Can't determine MFS image size" && exit 1 -sec_info=`${CROSS_BINUTILS_PREFIX}objdump -h $1 2> /dev/null | grep " oldmfs "` +sec_info=`elfdump -c $1 2> /dev/null | grep -A 5 -E "sh_name: oldmfs$"` # If we can't find the mfs section within the given kernel - bail. [ -z "${sec_info}" ] && echo "Can't locate mfs section within kernel" && exit 1 -sec_size=`echo ${sec_info} | awk '{printf("%d", "0x" $3)}' 2> /dev/null` -sec_start=`echo ${sec_info} | awk '{printf("%d", "0x" $6)}' 2> /dev/null` +sec_size=`echo "${sec_info}" | awk '/sh_size/ {print $2}' 2> /dev/null` +sec_start=`echo "${sec_info}" | awk '/sh_offset/ {print $2}' 2> /dev/null` # If the mfs section size is smaller than the mfs image - bail. [ ${sec_size} -lt ${mfs_size} ] && echo "MFS image too large" && exit 1 diff --git a/tests/sys/geom/class/gate/1_test.sh b/tests/sys/geom/class/gate/1_test.sh deleted file mode 100644 index ba573bb6e122..000000000000 --- a/tests/sys/geom/class/gate/1_test.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -# $FreeBSD$ - -. `dirname $0`/conf.sh - -echo '1..2' - -base=`basename $0` -us=0 -while [ -c /dev/ggate${us} ]; do - : $(( us += 1 )) -done -pidfile=ggated.$$.pid -conf=`mktemp $base.XXXXXX` || exit 1 -port=33080 - -work=$(attach_md -t malloc -s 1M) -src=$(attach_md -t malloc -s 1M) - -test_cleanup() -{ - ggatec destroy -f -u $us - pkill -F $pidfile - geom_test_cleanup -} -trap test_cleanup ABRT EXIT INT TERM - -dd if=/dev/random of=/dev/$work bs=1m count=1 conv=sync -dd if=/dev/random of=/dev/$src bs=1m count=1 conv=sync -src_checksum=$(md5 -q /dev/$src) - -echo "127.0.0.1 RW /dev/$work" > $conf - -if ! ggated -p $port -F $pidfile $conf; then - echo 'ggated failed to start' - echo 'Bail out!' - exit 1 -fi -sleep 1 -if ! ggatec create -p $port -u $us 127.0.0.1 /dev/$work; then - echo 'ggatec create failed' - echo 'Bail out!' - exit 1 -fi -sleep 1 - -dd if=/dev/${src} of=/dev/ggate${us} bs=1m count=1 -sleep 1 - -work_checksum=$(md5 -q /dev/$work) -if [ "$work_checksum" != "$src_checksum" ]; then - echo "not ok 1 - md5 checksums didn't match ($work_checksum != $src_checksum)" - echo "not ok 2 # SKIP" -else - echo 'ok 1 - md5 checksum' - - ggate_checksum=$(md5 -q /dev/ggate${us}) - if [ "$ggate_checksum" != "$src_checksum" ]; then - echo "not ok 2 - md5 checksums didn't match ($ggate_checksum != $src_checksum)" - else - echo 'ok 2 - md5 checksum' - fi -fi diff --git a/tests/sys/geom/class/gate/2_test.sh b/tests/sys/geom/class/gate/2_test.sh deleted file mode 100644 index be89accfcb9c..000000000000 --- a/tests/sys/geom/class/gate/2_test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -# $FreeBSD$ - -. `dirname $0`/conf.sh - -base=`basename $0` -us=46 -work=`mktemp -u $base.XXXXXX` || exit 1 -src=`mktemp -u $base.XXXXXX` || exit 1 - -test_cleanup() -{ - ggatel destroy -f -u $us - rm -f $work $src - - geom_test_cleanup -} -trap test_cleanup ABRT EXIT INT TERM - -dd if=/dev/random of=$work bs=1m count=1 conv=sync -dd if=/dev/random of=$src bs=1m count=1 conv=sync - -if ! ggatel create -u $us $work; then - echo 'ggatel create failed' - echo 'Bail out!' - exit 1 -fi - -dd if=${src} of=/dev/ggate${us} bs=1m count=1 -sleep 1 - -echo '1..2' - -src_checksum=$(md5 -q $src) -work_checksum=$(md5 -q $work) -if [ "$work_checksum" != "$src_checksum" ]; then - echo "not ok 1 - md5 checksums didn't match ($work_checksum != $src_checksum) # TODO: bug 204616" - echo 'not ok 2 # SKIP' -else - echo 'ok 1 - md5 checksum' - - ggate_checksum=$(md5 -q /dev/ggate${us}) - if [ "$ggate_checksum" != "$src_checksum" ]; then - echo "not ok 2 - md5 checksums didn't match ($ggate_checksum != $src_checksum)" - else - echo 'ok 2 - md5 checksum' - fi -fi diff --git a/tests/sys/geom/class/gate/3_test.sh b/tests/sys/geom/class/gate/3_test.sh deleted file mode 100644 index 3511df761ca6..000000000000 --- a/tests/sys/geom/class/gate/3_test.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# $FreeBSD$ - -. `dirname $0`/conf.sh - -base=`basename $0` -us=47 - -test_cleanup() -{ - ggatel destroy -f -u $us - - geom_test_cleanup -} -trap test_cleanup ABRT EXIT INT TERM - -work=$(attach_md -t malloc -s 1M) -src=$(attach_md -t malloc -s 1M) - -dd if=/dev/random of=/dev/$work bs=1m count=1 conv=sync -dd if=/dev/random of=/dev/$src bs=1m count=1 conv=sync -src_checksum=$(md5 -q /dev/$src) - -if ! ggatel create -u $us /dev/$work; then - echo 'ggatel create failed' - echo 'Bail out!' - exit 1 -fi - -sleep 1 -dd if=/dev/${src} of=/dev/ggate${us} bs=1m count=1 conv=sync -sleep 1 - -echo '1..2' - -work_checksum=$(md5 -q /dev/$work) -if [ "$work_checksum" != "$src_checksum" ]; then - echo "not ok 1 - md5 checksums didn't match ($work_checksum != $src_checksum)" - echo 'not ok 2 # SKIP' -else - echo 'ok 1 - md5 checksum' - - ggate_checksum=$(md5 -q /dev/ggate${us}) - if [ "$ggate_checksum" != "$src_checksum" ]; then - echo "not ok 2 - md5 checksums didn't match ($ggate_checksum != $src_checksum)" - else - echo 'ok 2 - md5 checksum' - fi -fi diff --git a/tests/sys/geom/class/gate/Makefile b/tests/sys/geom/class/gate/Makefile index c034bf037efb..417fa7dfd197 100644 --- a/tests/sys/geom/class/gate/Makefile +++ b/tests/sys/geom/class/gate/Makefile @@ -4,14 +4,6 @@ PACKAGE= tests TESTSDIR= ${TESTSBASE}/sys/geom/class/${.CURDIR:T} -TAP_TESTS_SH+= 1_test -TAP_TESTS_SH+= 2_test -TAP_TESTS_SH+= 3_test - -${PACKAGE}FILES+= conf.sh - -.for t in ${TAP_TESTS_SH} -TEST_METADATA.$t+= required_user="root" -.endfor +ATF_TESTS_SH+= ggate_test .include diff --git a/tests/sys/geom/class/gate/conf.sh b/tests/sys/geom/class/gate/conf.sh deleted file mode 100755 index 7e22ce46af66..000000000000 --- a/tests/sys/geom/class/gate/conf.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# $FreeBSD$ - -name="$(mktemp -u gate.XXXXXX)" -class="gate" -base=`basename $0` - -. `dirname $0`/../geom_subr.sh diff --git a/tests/sys/geom/class/gate/ggate_test.sh b/tests/sys/geom/class/gate/ggate_test.sh new file mode 100755 index 000000000000..ffdb341a4b28 --- /dev/null +++ b/tests/sys/geom/class/gate/ggate_test.sh @@ -0,0 +1,193 @@ +# $FreeBSD$ + +PIDFILE=ggated.pid +PLAINFILES=plainfiles +PORT=33080 +CONF=gg.exports +RETRIES=16 + +atf_test_case ggated cleanup +ggated_head() +{ + atf_set "descr" "ggated can proxy geoms" + atf_set "require.progs" "ggatec ggated" + atf_set "require.user" "root" + atf_set "timeout" 60 +} + +ggated_body() +{ + us=$(alloc_ggate_dev) + work=$(alloc_md) + src=$(alloc_md) + + dd if=/dev/random of=/dev/$work bs=1m count=1 conv=notrunc + dd if=/dev/random of=/dev/$src bs=1m count=1 conv=notrunc + + echo $CONF >> $PLAINFILES + echo "127.0.0.1 RW /dev/$work" > $CONF + + atf_check ggated -p $PORT -F $PIDFILE $CONF + for try in `jot $RETRIES`; do + ggatec create -p $PORT -u $us 127.0.0.1 /dev/$work && break + # wait for ggated to be ready + sleep 0.25 + done + if [ "$try" -eq "$RETRIES" ]; then + atf_fail "ggatec create failed" + fi + + for try in `jot $RETRIES`; do + dd if=/dev/${src} of=/dev/ggate${us} bs=1m count=1 conv=notrunc\ + && break + # Wait for /dev/ggate${us} to be ready + sleep 0.25 + done + if [ "$try" -eq "$RETRIES" ]; then + atf_fail "dd failed; /dev/ggate${us} isn't working" + fi + + checksum /dev/$src /dev/$work +} + +ggated_cleanup() +{ + common_cleanup +} + +atf_test_case ggatel_file cleanup +ggatel_file_head() +{ + atf_set "descr" "ggatel can proxy files" + atf_set "require.progs" "ggatel" + atf_set "require.user" "root" + atf_set "timeout" 15 +} + +ggatel_file_body() +{ + us=$(alloc_ggate_dev) + + echo src work >> ${PLAINFILES} + dd if=/dev/random of=work bs=1m count=1 + dd if=/dev/random of=src bs=1m count=1 + + atf_check ggatel create -u $us work + + dd if=src of=/dev/ggate${us} bs=1m count=1 conv=notrunc + + checksum src work +} + +ggatel_file_cleanup() +{ + common_cleanup +} + +atf_test_case ggatel_md cleanup +ggatel_md_head() +{ + atf_set "descr" "ggatel can proxy files" + atf_set "require.progs" "ggatel" + atf_set "require.user" "root" + atf_set "timeout" 15 +} + +ggatel_md_body() +{ + us=$(alloc_ggate_dev) + work=$(alloc_md) + src=$(alloc_md) + + dd if=/dev/random of=$work bs=1m count=1 conv=notrunc + dd if=/dev/random of=$src bs=1m count=1 conv=notrunc + + atf_check ggatel create -u $us /dev/$work + + dd if=/dev/$src of=/dev/ggate${us} bs=1m count=1 conv=notrunc + + checksum /dev/$src /dev/$work +} + +ggatel_md_cleanup() +{ + common_cleanup +} + +atf_init_test_cases() +{ + atf_add_test_case ggated + atf_add_test_case ggatel_file + atf_add_test_case ggatel_md +} + +alloc_ggate_dev() +{ + local us + + us=0 + while [ -c /dev/ggate${us} ]; do + : $(( us += 1 )) + done + echo ${us} > ggate.devs + echo ${us} +} + +alloc_md() +{ + local md + + md=$(mdconfig -a -t malloc -s 1M) || \ + atf_fail "failed to allocate md device" + echo ${md} >> md.devs + echo ${md} +} + +checksum() +{ + local src work + src=$1 + work=$2 + + src_checksum=$(md5 -q $src) + work_checksum=$(md5 -q $work) + + if [ "$work_checksum" != "$src_checksum" ]; then + atf_fail "work md5 checksum didn't match" + fi + + ggate_checksum=$(md5 -q /dev/ggate${us}) + if [ "$ggate_checksum" != "$src_checksum" ]; then + atf_fail "ggate md5 checksum didn't match" + fi +} + +common_cleanup() +{ + if [ -f "ggate.devs" ]; then + while read test_ggate; do + ggatec destroy -f -u $test_ggate >/dev/null + done < ggate.devs + rm ggate.devs + fi + + if [ -f "$PIDFILE" ]; then + pkill -F "$PIDFILE" + rm $PIDFILE + fi + + if [ -f "PLAINFILES" ]; then + while read f; do + rm -f ${f} + done < ${PLAINFILES} + rm ${PLAINFILES} + fi + + if [ -f "md.devs" ]; then + while read test_md; do + mdconfig -d -u $test_md 2>/dev/null + done < md.devs + rm md.devs + fi + true +} diff --git a/usr.bin/calendar/calendars/calendar.freebsd b/usr.bin/calendar/calendars/calendar.freebsd index 841896eb1d81..3a36f6eccee9 100644 --- a/usr.bin/calendar/calendars/calendar.freebsd +++ b/usr.bin/calendar/calendars/calendar.freebsd @@ -310,6 +310,7 @@ 09/20 Kevin Lo born in Taipei, Taiwan, Republic of China, 1972 09/21 Gleb Kurtsou born in Minsk, Belarus, 1984 09/22 Bryan Drewery born in San Diego, California, United States, 1984 +09/24 Larry Rosenman born in Queens, New York, United States, 1957 09/27 Neil Blakey-Milner born in Port Elizabeth, South Africa, 1978 09/27 Renato Botelho born in Araras, Sao Paulo, Brazil, 1979 09/28 Greg Lehey born in Melbourne, Victoria, Australia, 1948 diff --git a/usr.bin/tail/Makefile b/usr.bin/tail/Makefile index 672cbed008e6..6366f6f15f72 100644 --- a/usr.bin/tail/Makefile +++ b/usr.bin/tail/Makefile @@ -1,7 +1,13 @@ # $FreeBSD$ # @(#)Makefile 8.1 (Berkeley) 6/6/93 +.include + PROG= tail SRCS= forward.c misc.c read.c reverse.c tail.c +.if ${MK_TESTS} != "no" +SUBDIR+= tests +.endif + .include diff --git a/usr.bin/tail/reverse.c b/usr.bin/tail/reverse.c index 87269051ed68..88c328a3aafb 100644 --- a/usr.bin/tail/reverse.c +++ b/usr.bin/tail/reverse.c @@ -40,6 +40,7 @@ static char sccsid[] = "@(#)reverse.c 8.1 (Berkeley) 6/6/93"; __FBSDID("$FreeBSD$"); #include +#include #include #include @@ -169,12 +170,12 @@ r_reg(FILE *fp, const char *fn, enum STYLE style, off_t off, struct stat *sbp) ierr(fn); } -typedef struct bf { - struct bf *next; - struct bf *prev; - int len; - char *l; -} BF; +#define BSZ (128 * 1024) +typedef struct bfelem { + TAILQ_ENTRY(bfelem) entries; + size_t len; + char l[BSZ]; +} bfelem_t; /* * r_buf -- display a non-regular file in reverse order by line. @@ -189,64 +190,44 @@ typedef struct bf { static void r_buf(FILE *fp, const char *fn) { - BF *mark, *tl, *tr; - int ch, len, llen; + struct bfelem *tl, *first = NULL; + size_t llen; char *p; - off_t enomem; + off_t enomem = 0; + TAILQ_HEAD(bfhead, bfelem) head; + + TAILQ_INIT(&head); + + while (!feof(fp)) { + size_t len; - tl = NULL; -#define BSZ (128 * 1024) - for (mark = NULL, enomem = 0;;) { /* * Allocate a new block and link it into place in a doubly * linked list. If out of memory, toss the LRU block and * keep going. */ - if (enomem || (tl = malloc(sizeof(BF))) == NULL || - (tl->l = malloc(BSZ)) == NULL) { - if (!mark) + while ((tl = malloc(sizeof(bfelem_t))) == NULL) { + first = TAILQ_FIRST(&head); + if (TAILQ_EMPTY(&head)) err(1, "malloc"); - if (enomem) - tl = tl->next; - else { - if (tl) - free(tl); - tl = mark; - } - enomem += tl->len; - } else if (mark) { - tl->next = mark; - tl->prev = mark->prev; - mark->prev->next = tl; - mark->prev = tl; - } else { - mark = tl; - mark->next = mark->prev = mark; + enomem += first->len; + TAILQ_REMOVE(&head, first, entries); + free(first); } + TAILQ_INSERT_TAIL(&head, tl, entries); /* Fill the block with input data. */ - for (p = tl->l, len = 0; - len < BSZ && (ch = getc(fp)) != EOF; ++len) - *p++ = ch; - - if (ferror(fp)) { - ierr(fn); - return; - } - - /* - * If no input data for this block and we tossed some data, - * recover it. - */ - if (!len && enomem) { - enomem -= tl->len; - tl = tl->prev; - break; + len = 0; + while ((!feof(fp)) && len < BSZ) { + p = tl->l + len; + len += fread(p, 1, BSZ - len, fp); + if (ferror(fp)) { + ierr(fn); + return; + } } tl->len = len; - if (ch == EOF) - break; } if (enomem) { @@ -255,37 +236,46 @@ r_buf(FILE *fp, const char *fn) } /* - * Step through the blocks in the reverse order read. The last char - * is special, ignore whether newline or not. + * Now print the lines in reverse order + * Outline: + * Scan backward for "\n", + * print forward to the end of the buffers + * free any buffers that start after the "\n" just found + * Loop */ - for (mark = tl;;) { - for (p = tl->l + (len = tl->len) - 1, llen = 0; len--; - --p, ++llen) - if (*p == '\n') { - if (llen) { + tl = TAILQ_LAST(&head, bfhead); + first = TAILQ_FIRST(&head); + while (tl != NULL) { + struct bfelem *temp; + + for (p = tl->l + tl->len - 1, llen = 0; p >= tl->l; + --p, ++llen) { + int start = (tl == first && p == tl->l); + + if ((*p == '\n') || start) { + struct bfelem *tr; + + if (start && llen) + WR(p, llen + 1); + else if (llen) WR(p + 1, llen); - llen = 0; - } - if (tl == mark) - continue; - for (tr = tl->next; tr->len; tr = tr->next) { - WR(tr->l, tr->len); - tr->len = 0; - if (tr == mark) - break; + tr = TAILQ_NEXT(tl, entries); + llen = 0; + if (tr != NULL) { + TAILQ_FOREACH_FROM_SAFE(tr, &head, + entries, temp) { + if (tr->len) + WR(&tr->l, tr->len); + TAILQ_REMOVE(&head, tr, + entries); + free(tr); + } } } + } tl->len = llen; - if ((tl = tl->prev) == mark) - break; - } - tl = tl->next; - if (tl->len) { - WR(tl->l, tl->len); - tl->len = 0; - } - while ((tl = tl->next)->len) { - WR(tl->l, tl->len); - tl->len = 0; + tl = TAILQ_PREV(tl, bfhead, entries); } + TAILQ_REMOVE(&head, first, entries); + free(first); } diff --git a/usr.bin/tail/tests/Makefile b/usr.bin/tail/tests/Makefile new file mode 100644 index 000000000000..84338e292dc0 --- /dev/null +++ b/usr.bin/tail/tests/Makefile @@ -0,0 +1,7 @@ +# $FreeBSD$ + +PACKAGE= tests + +ATF_TESTS_SH= tail_test + +.include diff --git a/usr.bin/tail/tests/tail_test.sh b/usr.bin/tail/tests/tail_test.sh new file mode 100755 index 000000000000..3e407ea3ff00 --- /dev/null +++ b/usr.bin/tail/tests/tail_test.sh @@ -0,0 +1,233 @@ +# Copyright (c) 2016 Alan Somers +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ + +atf_test_case empty_r +empty_r_head() +{ + atf_set "descr" "Reverse an empty file" +} +empty_r_body() +{ + touch infile expectfile + tail -r infile > outfile + tail -r < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case file_r +file_r_head() +{ + atf_set "descr" "Reverse a file" +} +file_r_body() +{ + cat > infile < expectfile << HERE +This is the third line +This is the second line +This is the first line +HERE + tail -r infile > outfile + tail -r < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case file_rn2 +file_rn2_head() +{ + atf_set "descr" "Reverse the last two lines of a file" +} +file_rn2_body() +{ + cat > infile < expectfile << HERE +This is the third line +This is the second line +HERE + tail -rn2 infile > outfile + tail -rn2 < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case file_rc28 +file_rc28_head() +{ + atf_set "descr" "Reverse a file and display the last 28 characters" +} +file_rc28_body() +{ + cat > infile < expectfile << HERE +This is the third line +line +HERE + tail -rc28 infile > outfile + tail -rc28 < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case longfile_r +longfile_r_head() +{ + atf_set "descr" "Reverse a long file" +} +longfile_r_body() +{ + jot -w "%0511d" 1030 0 > infile + jot -w "%0511d" 1030 1029 0 -1 > expectfile + tail -r infile > outfile + tail -r < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case longfile_r_enomem +longfile_r_enomem_head() +{ + atf_set "descr" "Reverse a file that's too long to store in RAM" +} +longfile_r_enomem_body() +{ + # When we reverse a file that's too long for RAM, tail should drop the + # first part and just print what it can. We'll check that the last + # part is ok + { + ulimit -v 32768 || atf_skip "Can't adjust ulimit" + jot -w "%01023d" 32768 0 | tail -r > outfile ; + } + if [ "$?" -ne 1 ]; then + atf_skip "Didn't get ENOMEM. Adjust test parameters" + fi + # We don't know how much of the input we dropped. So just check that + # the first ten lines of tail's output are the same as the last ten of + # the input + jot -w "%01023d" 10 32767 0 -1 > expectfile + head -n 10 outfile > outtrunc + diff expectfile outtrunc + atf_check cmp expectfile outtrunc +} + +atf_test_case longfile_r_longlines +longfile_r_longlines_head() +{ + atf_set "descr" "Reverse a long file with extremely long lines" +} +longfile_r_longlines_body() +{ + jot -s " " -w "%07d" 18000 0 > infile + jot -s " " -w "%07d" 18000 18000 >> infile + jot -s " " -w "%07d" 18000 36000 >> infile + jot -s " " -w "%07d" 18000 36000 > expectfile + jot -s " " -w "%07d" 18000 18000 >> expectfile + jot -s " " -w "%07d" 18000 0 >> expectfile + tail -r infile > outfile + tail -r < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case longfile_rc135782 +longfile_rc135782_head() +{ + atf_set "descr" "Reverse a long file and print the last 135,782 bytes" +} +longfile_rc135782_body() +{ + jot -w "%063d" 9000 0 > infile + jot -w "%063d" 2121 8999 0 -1 > expectfile + echo "0000000000000000000000000000000006878" >> expectfile + tail -rc135782 infile > outfile + tail -rc135782 < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case longfile_rc145782_longlines +longfile_rc145782_longlines_head() +{ + atf_set "descr" "Reverse a long file with extremely long lines and print the last 145,782 bytes" +} +longfile_rc145782_longlines_body() +{ + jot -s " " -w "%07d" 18000 0 > infile + jot -s " " -w "%07d" 18000 18000 >> infile + jot -s " " -w "%07d" 18000 36000 >> infile + jot -s " " -w "%07d" 18000 36000 > expectfile + echo -n "35777 " >> expectfile + jot -s " " -w "%07d" 222 35778 >> expectfile + tail -rc145782 infile > outfile + tail -rc145782 < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + +atf_test_case longfile_rn2500 +longfile_rn2500_head() +{ + atf_set "descr" "Reverse a long file and print the last 2,500 lines" +} +longfile_rn2500_body() +{ + jot -w "%063d" 9000 0 > infile + jot -w "%063d" 2500 8999 0 -1 > expectfile + tail -rn2500 infile > outfile + tail -rn2500 < infile > outpipe + atf_check cmp expectfile outfile + atf_check cmp expectfile outpipe +} + + +atf_init_test_cases() +{ + atf_add_test_case empty_r + atf_add_test_case file_r + atf_add_test_case file_rc28 + atf_add_test_case file_rn2 + # The longfile tests are designed to exercise behavior in r_buf(), + # which operates on 128KB blocks + atf_add_test_case longfile_r + atf_add_test_case longfile_r_enomem + atf_add_test_case longfile_r_longlines + atf_add_test_case longfile_rc135782 + atf_add_test_case longfile_rc145782_longlines + atf_add_test_case longfile_rn2500 +} diff --git a/usr.sbin/crunch/crunchide/exec_elf32.c b/usr.sbin/crunch/crunchide/exec_elf32.c index 9aa29fd8a97b..d81775bfba71 100644 --- a/usr.sbin/crunch/crunchide/exec_elf32.c +++ b/usr.sbin/crunch/crunchide/exec_elf32.c @@ -191,6 +191,7 @@ ELFNAMEEND(check)(int fd, const char *fn) #define EM_RISCV 243 #endif case EM_RISCV: break; + case EM_S390: break; case EM_SPARCV9: break; case EM_X86_64: break; /* ELFDEFNNAME(MACHDEP_ID_CASES) */ diff --git a/usr.sbin/syslogd/syslogd.c b/usr.sbin/syslogd/syslogd.c index 81254bd564bf..278e4be57535 100644 --- a/usr.sbin/syslogd/syslogd.c +++ b/usr.sbin/syslogd/syslogd.c @@ -89,9 +89,11 @@ __FBSDID("$FreeBSD$"); #include #include +#if defined(INET) || defined(INET6) #include -#include #include +#endif +#include #include #include @@ -127,8 +129,11 @@ static const char include_ext[] = ".conf"; #define MAXUNAMES 20 /* maximum number of user names */ #define sstosa(ss) ((struct sockaddr *)(ss)) +#ifdef INET #define sstosin(ss) ((struct sockaddr_in *)(void *)(ss)) #define satosin(sa) ((struct sockaddr_in *)(void *)(sa)) +#endif +#ifdef INET6 #define sstosin6(ss) ((struct sockaddr_in6 *)(void *)(ss)) #define satosin6(sa) ((struct sockaddr_in6 *)(void *)(sa)) #define s6_addr32 __u6_addr.__u6_addr32 @@ -137,6 +142,7 @@ static const char include_ext[] = ".conf"; (((d)->s6_addr32[1] ^ (a)->s6_addr32[1]) & (m)->s6_addr32[1]) == 0 && \ (((d)->s6_addr32[2] ^ (a)->s6_addr32[2]) & (m)->s6_addr32[2]) == 0 && \ (((d)->s6_addr32[3] ^ (a)->s6_addr32[3]) & (m)->s6_addr32[3]) == 0 ) +#endif /* * List of peers and sockets for binding. */ @@ -1305,10 +1311,12 @@ fprintlog(struct filed *f, int flags, const char *msg) case F_FORW: dprintf(" %s", f->fu_forw_hname); switch (f->fu_forw_addr->ai_addr->sa_family) { +#ifdef INET case AF_INET: dprintf(":%d\n", ntohs(satosin(f->fu_forw_addr->ai_addr)->sin_port)); break; +#endif #ifdef INET6 case AF_INET6: dprintf(":%d\n", @@ -1929,7 +1937,20 @@ init(int signo) break; case F_FORW: - port = ntohs(satosin(f->fu_forw_addr->ai_addr)->sin_port); + switch (f->fu_forw_addr->ai_addr->sa_family) { +#ifdef INET + case AF_INET: + port = ntohs(satosin(f->fu_forw_addr->ai_addr)->sin_port); + break; +#endif +#ifdef INET6 + case AF_INET6: + port = ntohs(satosin6(f->fu_forw_addr->ai_addr)->sin6_port); + break; +#endif + default: + port = 0; + } if (port != 514) { printf("%s:%d", f->fu_forw_hname, port); @@ -2410,6 +2431,7 @@ timedout(int sig __unused) static int allowaddr(char *s) { +#if defined(INET) || defined(INET6) char *cp1, *cp2; struct allowedpeer *ap; struct servent *se; @@ -2571,6 +2593,7 @@ allowaddr(char *s) } printf("port = %d\n", ap->port); } +#endif return (0); }