From 21c4b2a944dce7e45a9f0c959624fe66b825fae9 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Wed, 29 Mar 2023 15:39:36 -0700 Subject: [PATCH 01/59] Linux 6.2 compat: META Update the META file to reflect compatibility with the 6.2 kernel. Reviewed-by: Brian Behlendorf Signed-off-by: Tony Hutter Closes #14689 --- META | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/META b/META index 18b3f3498eeb..8779e512f7be 100644 --- a/META +++ b/META @@ -6,5 +6,5 @@ Release: 1 Release-Tags: relext License: CDDL Author: OpenZFS -Linux-Maximum: 6.1 +Linux-Maximum: 6.2 Linux-Minimum: 3.10 From 1142362ff606ab7a1262d7d6f1f9be2205825065 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Fri, 31 Mar 2023 09:43:54 -0700 Subject: [PATCH 02/59] Use vmem_zalloc to silence allocation warning The kmem allocation in zfs_prune_aliases() will trigger a large allocation warning on systems with 64K pages. Resolve this by switching to vmem_alloc() which internally uses kvmalloc() so the right allocator will be used based on the allocation size. Reviewed-by: Richard Yao Reviewed-by: Tino Reichardt Reviewed-by: Brian Atkinson Signed-off-by: Brian Behlendorf Closes #8491 Closes #14694 --- module/os/linux/zfs/zfs_vfsops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/os/linux/zfs/zfs_vfsops.c b/module/os/linux/zfs/zfs_vfsops.c index 2d9b27a90884..48945b8af8c1 100644 --- a/module/os/linux/zfs/zfs_vfsops.c +++ b/module/os/linux/zfs/zfs_vfsops.c @@ -1194,7 +1194,7 @@ zfs_prune_aliases(zfsvfs_t *zfsvfs, unsigned long nr_to_scan) int objects = 0; int i = 0, j = 0; - zp_array = kmem_zalloc(max_array * sizeof (znode_t *), KM_SLEEP); + zp_array = vmem_zalloc(max_array * sizeof (znode_t *), KM_SLEEP); mutex_enter(&zfsvfs->z_znodes_lock); while ((zp = list_head(&zfsvfs->z_all_znodes)) != NULL) { @@ -1230,7 +1230,7 @@ zfs_prune_aliases(zfsvfs_t *zfsvfs, unsigned long nr_to_scan) zrele(zp); } - kmem_free(zp_array, max_array * sizeof (znode_t *)); + vmem_free(zp_array, max_array * sizeof (znode_t *)); return (objects); } From c5431f14655ce05d1ea99cb012806f0e5873d257 Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Fri, 31 Mar 2023 12:46:22 -0400 Subject: [PATCH 03/59] linux 6.3 compat: needs REQ_PREFLUSH | REQ_OP_WRITE Modify bio_set_flush() so if kernel version is >= 4.10, flags REQ_PREFLUSH and REQ_OP_WRITE are set together. Reviewed-by: Tony Hutter Reviewed-by: Brian Behlendorf Signed-off-by: Youzhong Yang Closes #14695 --- include/os/linux/kernel/linux/blkdev_compat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/os/linux/kernel/linux/blkdev_compat.h b/include/os/linux/kernel/linux/blkdev_compat.h index c7405ffab8ba..c5c6385be6ff 100644 --- a/include/os/linux/kernel/linux/blkdev_compat.h +++ b/include/os/linux/kernel/linux/blkdev_compat.h @@ -426,7 +426,7 @@ static inline void bio_set_flush(struct bio *bio) { #if defined(HAVE_REQ_PREFLUSH) /* >= 4.10 */ - bio_set_op_attrs(bio, 0, REQ_PREFLUSH); + bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE); #elif defined(WRITE_FLUSH_FUA) /* >= 2.6.37 and <= 4.9 */ bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA); #else From 3399a30ee02d0d31ba2d43d0ce0a2fd90d5c575d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Fri, 31 Mar 2023 18:47:48 +0200 Subject: [PATCH 04/59] contrib: dracut: fix race with root=zfs:dset when necessities required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This had always worked in my testing, but a user on hardware reported this to happen 100%, and I reproduced it once with cold VM host caches. dracut-zfs-generator runs as a systemd generator, i.e. at Some Relatively Early Time; if root= is a fixed dataset, it tries to "solve [necessities] statically at generation time". If by that point zfs-import.target hasn't popped (because the import is taking a non-negligible amount of time for whatever reason), it'll see no children for the root datase, and as such generate no mounts. This has never had any right to work. No-one caught this earlier because it's just that much more convenient to have root=zfs:AUTO, which orders itself properly. To fix this, always run zfs-nonroot-necessities.service; this additionally simplifies the implementation by: * making BOOTFS from zfs-env-bootfs.service be the real, canonical, root dataset name, not just "whatever the first bootfs is", and only set it if we're ZFS-booting * zfs-{rollback,snapshot}-bootfs.service can use this instead of re-implementing it * having zfs-env-bootfs.service also set BOOTFSFLAGS * this means the sysroot.mount drop-in can be fixed text * zfs-nonroot-necessities.service can also be constant and always enabled, because it's conditioned on BOOTFS being set There is no longer any code generated at run-time (the sysroot.mount drop-in is an unavoidable gratuitous cp). The flow of BOOTFS{,FLAGS} from zfs-env-bootfs.service to sysroot.mount is not noted explicitly in dracut.zfs(7), because (a) at some point it's just visual noise and (b) it's already ordered via d-p-m.s from z-i.t. Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #14690 --- contrib/dracut/90zfs/module-setup.sh.in | 3 + .../dracut/90zfs/zfs-env-bootfs.service.in | 15 ++++- contrib/dracut/90zfs/zfs-generator.sh.in | 67 ++----------------- contrib/dracut/90zfs/zfs-lib.sh.in | 2 +- .../90zfs/zfs-nonroot-necessities.service.in | 20 ++++++ .../90zfs/zfs-rollback-bootfs.service.in | 3 +- .../90zfs/zfs-snapshot-bootfs.service.in | 3 +- contrib/dracut/Makefile.am | 1 + man/man7/dracut.zfs.7 | 16 ++--- 9 files changed, 54 insertions(+), 76 deletions(-) create mode 100644 contrib/dracut/90zfs/zfs-nonroot-necessities.service.in diff --git a/contrib/dracut/90zfs/module-setup.sh.in b/contrib/dracut/90zfs/module-setup.sh.in index 78c74e7423bb..e55cb60e1612 100755 --- a/contrib/dracut/90zfs/module-setup.sh.in +++ b/contrib/dracut/90zfs/module-setup.sh.in @@ -81,6 +81,9 @@ install() { inst_simple "${moddir}/zfs-env-bootfs.service" "${systemdsystemunitdir}/zfs-env-bootfs.service" systemctl -q --root "${initdir}" add-wants zfs-import.target zfs-env-bootfs.service + inst_simple "${moddir}/zfs-nonroot-necessities.service" "${systemdsystemunitdir}/zfs-nonroot-necessities.service" + systemctl -q --root "${initdir}" add-requires initrd-root-fs.target zfs-nonroot-necessities.service + # Add user-provided unit overrides: # - /etc/systemd/system/${_service} # - /etc/systemd/system/${_service}.d/overrides.conf diff --git a/contrib/dracut/90zfs/zfs-env-bootfs.service.in b/contrib/dracut/90zfs/zfs-env-bootfs.service.in index 34c88037cac2..7ebab4c1a58d 100644 --- a/contrib/dracut/90zfs/zfs-env-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-env-bootfs.service.in @@ -1,6 +1,5 @@ [Unit] -Description=Set BOOTFS environment for dracut -Documentation=man:zpool(8) +Description=Set BOOTFS and BOOTFSFLAGS environment variables for dracut DefaultDependencies=no After=zfs-import-cache.service After=zfs-import-scan.service @@ -8,7 +7,17 @@ Before=zfs-import.target [Service] Type=oneshot -ExecStart=/bin/sh -c "exec systemctl set-environment BOOTFS=$(@sbindir@/zpool list -H -o bootfs | grep -m1 -vFx -)" +ExecStart=/bin/sh -c ' \ + . /lib/dracut-zfs-lib.sh; \ + decode_root_args || exit 0; \ + [ "$root" = "zfs:AUTO" ] && root="$(@sbindir@/zpool list -H -o bootfs | grep -m1 -vFx -)"; \ + rootflags="$(getarg rootflags=)"; \ + case ",$rootflags," in \ + *,zfsutil,*) ;; \ + ,,) rootflags=zfsutil ;; \ + *) rootflags="zfsutil,$rootflags" ;; \ + esac; \ + exec systemctl set-environment BOOTFS="$root" BOOTFSFLAGS="$rootflags"' [Install] WantedBy=zfs-import.target diff --git a/contrib/dracut/90zfs/zfs-generator.sh.in b/contrib/dracut/90zfs/zfs-generator.sh.in index 56f7ca9785ba..4e1eb7490e0d 100755 --- a/contrib/dracut/90zfs/zfs-generator.sh.in +++ b/contrib/dracut/90zfs/zfs-generator.sh.in @@ -14,81 +14,24 @@ GENERATOR_DIR="$1" . /lib/dracut-zfs-lib.sh decode_root_args || exit 0 -[ -z "${rootflags}" ] && rootflags=$(getarg rootflags=) -case ",${rootflags}," in - *,zfsutil,*) ;; - ,,) rootflags=zfsutil ;; - *) rootflags="zfsutil,${rootflags}" ;; -esac - [ -n "$debug" ] && echo "zfs-generator: writing extension for sysroot.mount to $GENERATOR_DIR/sysroot.mount.d/zfs-enhancement.conf" >> /dev/kmsg -mkdir -p "$GENERATOR_DIR"/sysroot.mount.d "$GENERATOR_DIR"/initrd-root-fs.target.requires "$GENERATOR_DIR"/dracut-pre-mount.service.d +mkdir -p "$GENERATOR_DIR"/sysroot.mount.d "$GENERATOR_DIR"/dracut-pre-mount.service.d + { echo "[Unit]" echo "Before=initrd-root-fs.target" echo "After=zfs-import.target" echo echo "[Mount]" - if [ "${root}" = "zfs:AUTO" ]; then - echo "PassEnvironment=BOOTFS" - echo 'What=${BOOTFS}' - else - echo "What=${root}" - fi + echo "PassEnvironment=BOOTFS BOOTFSFLAGS" + echo 'What=${BOOTFS}' echo "Type=zfs" - echo "Options=${rootflags}" + echo 'Options=${BOOTFSFLAGS}' } > "$GENERATOR_DIR"/sysroot.mount.d/zfs-enhancement.conf ln -fs ../sysroot.mount "$GENERATOR_DIR"/initrd-root-fs.target.requires/sysroot.mount - -if [ "${root}" = "zfs:AUTO" ]; then - { - echo "[Unit]" - echo "Before=initrd-root-fs.target" - echo "After=sysroot.mount" - echo "DefaultDependencies=no" - echo - echo "[Service]" - echo "Type=oneshot" - echo "PassEnvironment=BOOTFS" - echo "ExecStart=/bin/sh -c '" ' \ - . /lib/dracut-zfs-lib.sh; \ - _zfs_nonroot_necessities_cb() { \ - zfs mount | grep -m1 -q "^$1 " && return 0; \ - echo "Mounting $1 on /sysroot$2"; \ - mount -o zfsutil -t zfs "$1" "/sysroot$2"; \ - }; \ - for_relevant_root_children "${BOOTFS}" _zfs_nonroot_necessities_cb;' \ - "'" - } > "$GENERATOR_DIR"/zfs-nonroot-necessities.service - ln -fs ../zfs-nonroot-necessities.service "$GENERATOR_DIR"/initrd-root-fs.target.requires/zfs-nonroot-necessities.service -else - # We can solve this statically at generation time, so do! - _zfs_generator_cb() { - dset="${1}" - mpnt="${2}" - unit="$(systemd-escape --suffix=mount -p "/sysroot${mpnt}")" - - { - echo "[Unit]" - echo "Before=initrd-root-fs.target" - echo "After=sysroot.mount" - echo - echo "[Mount]" - echo "Where=/sysroot${mpnt}" - echo "What=${dset}" - echo "Type=zfs" - echo "Options=zfsutil" - } > "$GENERATOR_DIR/${unit}" - ln -fs ../"${unit}" "$GENERATOR_DIR"/initrd-root-fs.target.requires/"${unit}" - } - - for_relevant_root_children "${root}" _zfs_generator_cb -fi - - { echo "[Unit]" echo "After=zfs-import.target" diff --git a/contrib/dracut/90zfs/zfs-lib.sh.in b/contrib/dracut/90zfs/zfs-lib.sh.in index 3a43e514d6f9..7139e2e6fe4b 100755 --- a/contrib/dracut/90zfs/zfs-lib.sh.in +++ b/contrib/dracut/90zfs/zfs-lib.sh.in @@ -39,7 +39,7 @@ mount_dataset() { # for_relevant_root_children DATASET EXEC # Runs "EXEC dataset mountpoint" for all children of DATASET that are needed for system bringup -# Used by zfs-generator.sh and friends, too! +# Used by zfs-nonroot-necessities.service and friends, too! for_relevant_root_children() { dataset="${1}" exec="${2}" diff --git a/contrib/dracut/90zfs/zfs-nonroot-necessities.service.in b/contrib/dracut/90zfs/zfs-nonroot-necessities.service.in new file mode 100644 index 000000000000..8f420c737c72 --- /dev/null +++ b/contrib/dracut/90zfs/zfs-nonroot-necessities.service.in @@ -0,0 +1,20 @@ +[Unit] +Before=initrd-root-fs.target +After=sysroot.mount +DefaultDependencies=no +ConditionEnvironment=BOOTFS + +[Service] +Type=oneshot +PassEnvironment=BOOTFS +ExecStart=/bin/sh -c ' \ + . /lib/dracut-zfs-lib.sh; \ + _zfs_nonroot_necessities_cb() { \ + @sbindir@/zfs mount | grep -m1 -q "^$1 " && return 0; \ + echo "Mounting $1 on /sysroot$2"; \ + mount -o zfsutil -t zfs "$1" "/sysroot$2"; \ + }; \ + for_relevant_root_children "${BOOTFS}" _zfs_nonroot_necessities_cb' + +[Install] +RequiredBy=initrd-root-fs.target diff --git a/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in b/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in index a29cf3a3dd81..68fdcb1f323e 100644 --- a/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-rollback-bootfs.service.in @@ -5,8 +5,9 @@ After=zfs-import.target dracut-pre-mount.service zfs-snapshot-bootfs.service Before=dracut-mount.service DefaultDependencies=no ConditionKernelCommandLine=bootfs.rollback +ConditionEnvironment=BOOTFS [Service] Type=oneshot -ExecStart=/bin/sh -c '. /lib/dracut-zfs-lib.sh; decode_root_args || exit; [ "$root" = "zfs:AUTO" ] && root="$BOOTFS"; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "$root@${SNAPNAME:-%v}"' +ExecStart=/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.rollback)"; exec @sbindir@/zfs rollback -Rf "$BOOTFS@${SNAPNAME:-%v}"' RemainAfterExit=yes diff --git a/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in b/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in index 9e73d1a78724..a675b5b2ea98 100644 --- a/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in +++ b/contrib/dracut/90zfs/zfs-snapshot-bootfs.service.in @@ -5,8 +5,9 @@ After=zfs-import.target dracut-pre-mount.service Before=dracut-mount.service DefaultDependencies=no ConditionKernelCommandLine=bootfs.snapshot +ConditionEnvironment=BOOTFS [Service] Type=oneshot -ExecStart=-/bin/sh -c '. /lib/dracut-zfs-lib.sh; decode_root_args || exit; [ "$root" = "zfs:AUTO" ] && root="$BOOTFS"; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "$root@${SNAPNAME:-%v}"' +ExecStart=-/bin/sh -c '. /lib/dracut-lib.sh; SNAPNAME="$(getarg bootfs.snapshot)"; exec @sbindir@/zfs snapshot "$BOOTFS@${SNAPNAME:-%v}"' RemainAfterExit=yes diff --git a/contrib/dracut/Makefile.am b/contrib/dracut/Makefile.am index 73ca52b66316..b432ab76a6d8 100644 --- a/contrib/dracut/Makefile.am +++ b/contrib/dracut/Makefile.am @@ -16,6 +16,7 @@ pkgdracut_90_SCRIPTS = \ pkgdracut_90_DATA = \ %D%/90zfs/zfs-env-bootfs.service \ + %D%/90zfs/zfs-nonroot-necessities.service \ %D%/90zfs/zfs-rollback-bootfs.service \ %D%/90zfs/zfs-snapshot-bootfs.service diff --git a/man/man7/dracut.zfs.7 b/man/man7/dracut.zfs.7 index 2db2f6639eaf..c1475c695e83 100644 --- a/man/man7/dracut.zfs.7 +++ b/man/man7/dracut.zfs.7 @@ -1,6 +1,6 @@ .\" SPDX-License-Identifier: 0BSD .\" -.Dd April 4, 2022 +.Dd March 28, 2023 .Dt DRACUT.ZFS 7 .Os . @@ -28,13 +28,13 @@ zfs-import-scan.service \(da \(da | zfs-import-c zfs-import.target \(-> dracut-pre-mount.service | \(ua | | dracut-zfs-generator | - | ____________________/| + | _____________________/| |/ \(da - | sysroot.mount \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em dracut-zfs-generator - | | \(da | - | \(da sysroot-{usr,etc,lib,&c.}.mount | - | initrd-root-fs.target \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em or \(da - | | zfs-nonroot-necessities.service + | sysroot.mount \(<-\(em\(em\(em dracut-zfs-generator + | | + | \(da + | initrd-root-fs.target \(<-\(em zfs-nonroot-necessities.service + | | | | \(da | \(da dracut-mount.service | zfs-snapshot-bootfs.service | | @@ -42,7 +42,7 @@ zfs-import-scan.service \(da \(da | zfs-import-c \(da … | zfs-rollback-bootfs.service | | | \(da | - | sysroot-usr.mount \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em/ + | /sysroot/{usr,etc,lib,&c.} \(<-\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em\(em/ | | | \(da | initrd-fs.target From 6ecdd35bdbcdeae0adabefe107677620e88d5548 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Wed, 5 Apr 2023 18:57:01 +0200 Subject: [PATCH 05/59] Fix "Add colored output to zfs list" Running `zfs list -o avail rpool` resulted in a core dump. This commit will fix this. Run the needed overhead only, when `use_color()` is true. Reviewed-by: Brian Behlendorf Reviewed-by: George Wilson Signed-off-by: Tino Reichardt Closes #14712 --- cmd/zfs/zfs_main.c | 17 +++++++++++++++-- include/libzutil.h | 1 + lib/libzfs/libzfs.abi | 4 ++++ lib/libzfs/libzfs_util.c | 2 +- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index f918036cb9b7..d65b01579037 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -3556,8 +3556,21 @@ print_dataset(zfs_handle_t *zhp, list_cbdata_t *cb) right_justify = B_FALSE; } - if (pl->pl_prop == ZFS_PROP_AVAILABLE) - color_start(zfs_list_avail_color(zhp)); + /* + * zfs_list_avail_color() needs ZFS_PROP_AVAILABLE + USED + * - so we need another for() search for the USED part + * - when no colors wanted, we can skip the whole thing + */ + if (use_color() && pl->pl_prop == ZFS_PROP_AVAILABLE) { + zprop_list_t *pl2 = cb->cb_proplist; + for (; pl2 != NULL; pl2 = pl2->pl_next) { + if (pl2->pl_prop == ZFS_PROP_USED) { + color_start(zfs_list_avail_color(zhp)); + /* found it, no need for more loops */ + break; + } + } + } /* * If this is being called in scripted mode, or if this is the diff --git a/include/libzutil.h b/include/libzutil.h index 465e463f0c1f..237ff976ba62 100644 --- a/include/libzutil.h +++ b/include/libzutil.h @@ -182,6 +182,7 @@ struct zfs_cmd; #define ANSI_RESET "\033[0m" #define ANSI_BOLD "\033[1m" +_LIBZUTIL_H int use_color(void); _LIBZUTIL_H void color_start(const char *color); _LIBZUTIL_H void color_end(void); _LIBZUTIL_H int printf_color(const char *color, const char *format, ...); diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi index 99e1b8cdf695..2b61710f5592 100644 --- a/lib/libzfs/libzfs.abi +++ b/lib/libzfs/libzfs.abi @@ -259,6 +259,7 @@ + @@ -5338,6 +5339,9 @@ + + + diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index 60695f8a63f4..393971ddf13c 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -1965,7 +1965,7 @@ zfs_version_print(void) * Return 1 if the user requested ANSI color output, and our terminal supports * it. Return 0 for no color. */ -static int +int use_color(void) { static int use_color = -1; From 8eb2f2605717572d92041534a0556d6d75c47d99 Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Wed, 5 Apr 2023 13:01:38 -0400 Subject: [PATCH 06/59] Linux 6.3 compat: writepage_t first arg struct folio* The type def of writepage_t in kernel 6.3 is changed to take struct folio* as the first argument. We need to detect this change and pass correct function to write_cache_pages(). Reviewed-by: Brian Behlendorf Reviewed-by: Brian Atkinson Signed-off-by: Youzhong Yang Closes #14699 --- config/kernel-writepage_t.m4 | 26 ++++++++++++++++++++++++++ config/kernel.m4 | 2 ++ module/os/linux/zfs/zpl_file.c | 28 +++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 config/kernel-writepage_t.m4 diff --git a/config/kernel-writepage_t.m4 b/config/kernel-writepage_t.m4 new file mode 100644 index 000000000000..3a0cffd98570 --- /dev/null +++ b/config/kernel-writepage_t.m4 @@ -0,0 +1,26 @@ +AC_DEFUN([ZFS_AC_KERNEL_SRC_WRITEPAGE_T], [ + dnl # + dnl # 6.3 API change + dnl # The writepage_t function type now has its first argument as + dnl # struct folio* instead of struct page* + dnl # + ZFS_LINUX_TEST_SRC([writepage_t_folio], [ + #include + int putpage(struct folio *folio, + struct writeback_control *wbc, void *data) + { return 0; } + writepage_t func = putpage; + ],[]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_WRITEPAGE_T], [ + AC_MSG_CHECKING([whether int (*writepage_t)() takes struct folio*]) + ZFS_LINUX_TEST_RESULT([writepage_t_folio], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_WRITEPAGE_T_FOLIO, 1, + [int (*writepage_t)() takes struct folio*]) + ],[ + AC_MSG_RESULT(no) + ]) +]) + diff --git a/config/kernel.m4 b/config/kernel.m4 index 4c7569841f33..fb07f5004d3c 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -151,6 +151,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_IDMAP_MNT_API ZFS_AC_KERNEL_SRC_IATTR_VFSID ZFS_AC_KERNEL_SRC_FILEMAP + ZFS_AC_KERNEL_SRC_WRITEPAGE_T case "$host_cpu" in powerpc*) ZFS_AC_KERNEL_SRC_CPU_HAS_FEATURE @@ -281,6 +282,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_IDMAP_MNT_API ZFS_AC_KERNEL_IATTR_VFSID ZFS_AC_KERNEL_FILEMAP + ZFS_AC_KERNEL_WRITEPAGE_T case "$host_cpu" in powerpc*) ZFS_AC_KERNEL_CPU_HAS_FEATURE diff --git a/module/os/linux/zfs/zpl_file.c b/module/os/linux/zfs/zpl_file.c index 0a50f80ea68d..ce22e9a9e0e4 100644 --- a/module/os/linux/zfs/zpl_file.c +++ b/module/os/linux/zfs/zpl_file.c @@ -736,6 +736,29 @@ zpl_putpage(struct page *pp, struct writeback_control *wbc, void *data) return (0); } +#ifdef HAVE_WRITEPAGE_T_FOLIO +static int +zpl_putfolio(struct folio *pp, struct writeback_control *wbc, void *data) +{ + (void) zpl_putpage(&pp->page, wbc, data); + return (0); +} +#endif + +static inline int +zpl_write_cache_pages(struct address_space *mapping, + struct writeback_control *wbc, void *data) +{ + int result; + +#ifdef HAVE_WRITEPAGE_T_FOLIO + result = write_cache_pages(mapping, wbc, zpl_putfolio, data); +#else + result = write_cache_pages(mapping, wbc, zpl_putpage, data); +#endif + return (result); +} + static int zpl_writepages(struct address_space *mapping, struct writeback_control *wbc) { @@ -760,7 +783,7 @@ zpl_writepages(struct address_space *mapping, struct writeback_control *wbc) */ boolean_t for_sync = (sync_mode == WB_SYNC_ALL); wbc->sync_mode = WB_SYNC_NONE; - result = write_cache_pages(mapping, wbc, zpl_putpage, &for_sync); + result = zpl_write_cache_pages(mapping, wbc, &for_sync); if (sync_mode != wbc->sync_mode) { if ((result = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) return (result); @@ -776,8 +799,7 @@ zpl_writepages(struct address_space *mapping, struct writeback_control *wbc) * details). That being said, this is a no-op in most cases. */ wbc->sync_mode = sync_mode; - result = write_cache_pages(mapping, wbc, zpl_putpage, - &for_sync); + result = zpl_write_cache_pages(mapping, wbc, &for_sync); } return (result); } From 1038f87c4edcc66d7d9446efb9b0d9ed50beda19 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Thu, 6 Apr 2023 02:42:22 +0900 Subject: [PATCH 07/59] Fix some signedness issues in arc_evict() It may happen that "wanted total ARC size" (wt) is negative, that was expected. But multiplication product of it and unsigned fractions result in unsigned value, incorrectly shifted right with a sing loss. Reviewed-by: Brian Behlendorf Reviewed-by: Prakash Surya Reviewed-by: Paul Dagnelie Signed-off-by: Alexander Motin Sponsored by: iXsystems, Inc. Closes #14692 --- module/zfs/arc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module/zfs/arc.c b/module/zfs/arc.c index e32707bbe5c3..c50228a2682f 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -4465,7 +4465,7 @@ arc_evict(void) */ int64_t prune = 0; int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size); - w = wt * (arc_meta >> 16) >> 16; + w = wt * (int64_t)(arc_meta >> 16) >> 16; if (zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) + zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]) - zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) - @@ -4481,7 +4481,7 @@ arc_evict(void) arc_prune_async(prune); /* Evict MRU metadata. */ - w = wt * (arc_meta * arc_pm >> 48) >> 16; + w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16; e = MIN((int64_t)(asize - arc_c), (int64_t)(mrum - w)); bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e); total_evicted += bytes; @@ -4489,7 +4489,7 @@ arc_evict(void) asize -= bytes; /* Evict MFU metadata. */ - w = wt * (arc_meta >> 16) >> 16; + w = wt * (int64_t)(arc_meta >> 16) >> 16; e = MIN((int64_t)(asize - arc_c), (int64_t)(m - w)); bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e); total_evicted += bytes; @@ -4498,7 +4498,7 @@ arc_evict(void) /* Evict MRU data. */ wt -= m - total_evicted; - w = wt * (arc_pd >> 16) >> 16; + w = wt * (int64_t)(arc_pd >> 16) >> 16; e = MIN((int64_t)(asize - arc_c), (int64_t)(mrud - w)); bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e); total_evicted += bytes; From b66c2a0899a71fce6454ff92e77e3838eafb27c8 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Thu, 6 Apr 2023 10:29:27 -0700 Subject: [PATCH 08/59] Storage device expansion "silently" fails on degraded vdev When a vdev is degraded or faulted, we refuse to expand it when doing online -e. However, we also don't actually cause the online command to fail, even though the disk didn't expand. This is confusing and misleading, and can result in violated expectations. Reviewed-by: Brian Behlendorf Reviewed-by: Matthew Ahrens Reviewed-by: Mark Maybee Signed-off-by: Paul Dagnelie Closes 14145 --- cmd/zpool/zpool_main.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index d79c1608b09f..9475beaa2368 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -6936,6 +6936,17 @@ zpool_do_online(int argc, char **argv) return (1); for (i = 1; i < argc; i++) { + vdev_state_t oldstate; + boolean_t avail_spare, l2cache; + nvlist_t *tgt = zpool_find_vdev(zhp, argv[i], &avail_spare, + &l2cache, NULL); + if (tgt == NULL) { + ret = 1; + continue; + } + uint_t vsc; + oldstate = ((vdev_stat_t *)fnvlist_lookup_uint64_array(tgt, + ZPOOL_CONFIG_VDEV_STATS, &vsc))->vs_state; if (zpool_vdev_online(zhp, argv[i], flags, &newstate) == 0) { if (newstate != VDEV_STATE_HEALTHY) { (void) printf(gettext("warning: device '%s' " @@ -6949,6 +6960,17 @@ zpool_do_online(int argc, char **argv) (void) printf(gettext("use 'zpool " "replace' to replace devices " "that are no longer present\n")); + if ((flags & ZFS_ONLINE_EXPAND)) { + (void) printf(gettext("%s: failed " + "to expand usable space on " + "unhealthy device '%s'\n"), + (oldstate >= VDEV_STATE_DEGRADED ? + "error" : "warning"), argv[i]); + if (oldstate >= VDEV_STATE_DEGRADED) { + ret = 1; + break; + } + } } } else { ret = 1; From ece7ab7e7de81a9a51923d7baa7db3577de4aace Mon Sep 17 00:00:00 2001 From: Rob N Date: Fri, 7 Apr 2023 03:31:19 +1000 Subject: [PATCH 09/59] vdev: expose zfs_vdev_def_queue_depth as a module parameter It was previously available only to FreeBSD. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Reviewed-by: Alexander Motin Signed-off-by: Rob Norris Sponsored-by: Klara, Inc. Sponsored-by: Seagate Technology LLC Closes #14718 --- man/man4/zfs.4 | 5 +++++ module/os/freebsd/zfs/sysctl_os.c | 8 -------- module/zfs/vdev_queue.c | 3 +++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index e8e2cfec61e8..566caae7b4e9 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -1292,6 +1292,11 @@ as fuller devices will tend to be slower than empty devices. Also see .Sy zio_dva_throttle_enabled . . +.It Sy zfs_vdev_def_queue_depth Ns = Ns Sy 32 Pq uint +Default queue depth for each vdev IO allocator. +Higher values allow for better coalescing of sequential writes before sending +them to the disk, but can increase transaction commit times. +. .It Sy zfs_vdev_failfast_mask Ns = Ns Sy 1 Pq uint Defines if the driver should retire on a given error type. The following options may be bitwise-ored together: diff --git a/module/os/freebsd/zfs/sysctl_os.c b/module/os/freebsd/zfs/sysctl_os.c index eccb91deff4f..cc616f33db96 100644 --- a/module/os/freebsd/zfs/sysctl_os.c +++ b/module/os/freebsd/zfs/sysctl_os.c @@ -887,14 +887,6 @@ SYSCTL_UINT(_vfs_zfs, OID_AUTO, top_maxinflight, " (LEGACY)"); /* END CSTYLED */ -extern uint_t zfs_vdev_def_queue_depth; - -/* BEGIN CSTYLED */ -SYSCTL_UINT(_vfs_zfs_vdev, OID_AUTO, def_queue_depth, - CTLFLAG_RWTUN, &zfs_vdev_def_queue_depth, 0, - "Default queue depth for each allocator"); -/* END CSTYLED */ - /* zio.c */ /* BEGIN CSTYLED */ diff --git a/module/zfs/vdev_queue.c b/module/zfs/vdev_queue.c index ec55674393ce..1a75d68abd9e 100644 --- a/module/zfs/vdev_queue.c +++ b/module/zfs/vdev_queue.c @@ -1119,3 +1119,6 @@ ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, nia_delay, UINT, ZMOD_RW, ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, queue_depth_pct, UINT, ZMOD_RW, "Queue depth percentage for each top-level vdev"); + +ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, def_queue_depth, UINT, ZMOD_RW, + "Default queue depth for each allocator"); From a3f82aec933660558d6512a83481527ef731ff0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Matu=C5=A1ka?= Date: Thu, 6 Apr 2023 19:35:02 +0200 Subject: [PATCH 10/59] Miscellaneous FreBSD compilation bugfixes Add missing machine/md_var.h to spl/sys/simd_aarch64.h and spl/sys/simd_arm.h In spl/sys/simd_x86.h, PCB_FPUNOSAVE exists only on amd64, use PCB_NPXNOSAVE on i386 In FreeBSD sys/elf_common.h redefines AT_UID and AT_GID on FreeBSD, we need a hack in vnode.h similar to Linux. sys/simd.h needs to be included early. In zfs_freebsd_copy_file_range() we pass a (size_t *)lenp to zfs_clone_range() that expects a (uint64_t *) Allow compiling armv6 world by limiting ARM macros in sha256_impl.c and sha512_impl.c to __ARM_ARCH > 6 Reviewed-by: Alexander Motin Reviewed-by: Tino Reichardt Reviewed-by: Richard Yao Reviewed-by: Pawel Jakub Dawidek Reviewed-by: Signed-off-by: WHR Signed-off-by: Martin Matuska Closes #14674 --- include/os/freebsd/spl/sys/simd_aarch64.h | 1 + include/os/freebsd/spl/sys/simd_arm.h | 1 + include/os/freebsd/spl/sys/simd_x86.h | 4 ++++ include/os/freebsd/spl/sys/vnode.h | 4 ++++ module/icp/algs/blake3/blake3.c | 1 + module/icp/algs/blake3/blake3_generic.c | 1 + module/icp/algs/blake3/blake3_impl.c | 2 +- module/icp/algs/sha2/sha256_impl.c | 6 +++--- module/icp/algs/sha2/sha512_impl.c | 6 +++--- module/os/freebsd/zfs/zfs_vnops_os.c | 7 ++++--- module/zcommon/zfs_fletcher.c | 2 +- module/zcommon/zfs_prop.c | 6 ++++-- module/zfs/vdev_raidz_math.c | 2 +- 13 files changed, 29 insertions(+), 14 deletions(-) diff --git a/include/os/freebsd/spl/sys/simd_aarch64.h b/include/os/freebsd/spl/sys/simd_aarch64.h index 847c2ed29189..df33bdaeccf8 100644 --- a/include/os/freebsd/spl/sys/simd_aarch64.h +++ b/include/os/freebsd/spl/sys/simd_aarch64.h @@ -45,6 +45,7 @@ #include #include +#include #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) diff --git a/include/os/freebsd/spl/sys/simd_arm.h b/include/os/freebsd/spl/sys/simd_arm.h index f6362cd6bb54..178fbc3b3c6e 100644 --- a/include/os/freebsd/spl/sys/simd_arm.h +++ b/include/os/freebsd/spl/sys/simd_arm.h @@ -44,6 +44,7 @@ #include #include +#include #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) diff --git a/include/os/freebsd/spl/sys/simd_x86.h b/include/os/freebsd/spl/sys/simd_x86.h index 6512d4fcba4f..8e93b558dfe8 100644 --- a/include/os/freebsd/spl/sys/simd_x86.h +++ b/include/os/freebsd/spl/sys/simd_x86.h @@ -45,6 +45,10 @@ fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX);\ } +#ifndef PCB_FPUNOSAVE +#define PCB_FPUNOSAVE PCB_NPXNOSAVE +#endif + #define kfpu_end() { \ if (__predict_false(curpcb->pcb_flags & PCB_FPUNOSAVE)) \ fpu_kern_leave(curthread, NULL); \ diff --git a/include/os/freebsd/spl/sys/vnode.h b/include/os/freebsd/spl/sys/vnode.h index 483d12ae59a2..ab1727dca0c9 100644 --- a/include/os/freebsd/spl/sys/vnode.h +++ b/include/os/freebsd/spl/sys/vnode.h @@ -143,6 +143,10 @@ vn_flush_cached_data(vnode_t *vp, boolean_t sync) /* * Attributes of interest to the caller of setattr or getattr. */ + +#undef AT_UID +#undef AT_GID + #define AT_MODE 0x00002 #define AT_UID 0x00004 #define AT_GID 0x00008 diff --git a/module/icp/algs/blake3/blake3.c b/module/icp/algs/blake3/blake3.c index 4f93e4ff2051..0bab7a3a7593 100644 --- a/module/icp/algs/blake3/blake3.c +++ b/module/icp/algs/blake3/blake3.c @@ -25,6 +25,7 @@ * Copyright (c) 2021-2022 Tino Reichardt */ +#include #include #include diff --git a/module/icp/algs/blake3/blake3_generic.c b/module/icp/algs/blake3/blake3_generic.c index ca7197a26f39..fbe184969672 100644 --- a/module/icp/algs/blake3/blake3_generic.c +++ b/module/icp/algs/blake3/blake3_generic.c @@ -25,6 +25,7 @@ * Copyright (c) 2021-2022 Tino Reichardt */ +#include #include #include "blake3_impl.h" diff --git a/module/icp/algs/blake3/blake3_impl.c b/module/icp/algs/blake3/blake3_impl.c index b59fde1a4dd3..f3f48c2dfa1a 100644 --- a/module/icp/algs/blake3/blake3_impl.c +++ b/module/icp/algs/blake3/blake3_impl.c @@ -23,10 +23,10 @@ * Copyright (c) 2021-2022 Tino Reichardt */ +#include #include #include #include -#include #include "blake3_impl.h" diff --git a/module/icp/algs/sha2/sha256_impl.c b/module/icp/algs/sha2/sha256_impl.c index 278d7e577d72..01ce5cbd814c 100644 --- a/module/icp/algs/sha2/sha256_impl.c +++ b/module/icp/algs/sha2/sha256_impl.c @@ -23,10 +23,10 @@ * Copyright (c) 2022 Tino Reichardt */ +#include #include #include #include -#include #include #include @@ -118,7 +118,7 @@ const sha256_ops_t sha256_shani_impl = { }; #endif -#elif defined(__aarch64__) || defined(__arm__) +#elif defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH > 6) static boolean_t sha256_have_neon(void) { return (kfpu_allowed() && zfs_neon_available()); @@ -192,7 +192,7 @@ static const sha256_ops_t *const sha256_impls[] = { #if defined(__x86_64) && defined(HAVE_SSE4_1) &sha256_shani_impl, #endif -#if defined(__aarch64__) || defined(__arm__) +#if defined(__aarch64__) || (defined(__arm__) && __ARM_ARCH > 6) &sha256_armv7_impl, &sha256_neon_impl, &sha256_armv8_impl, diff --git a/module/icp/algs/sha2/sha512_impl.c b/module/icp/algs/sha2/sha512_impl.c index 991e832b15ca..27b35a639a54 100644 --- a/module/icp/algs/sha2/sha512_impl.c +++ b/module/icp/algs/sha2/sha512_impl.c @@ -23,10 +23,10 @@ * Copyright (c) 2022 Tino Reichardt */ +#include #include #include #include -#include #include #include @@ -108,7 +108,7 @@ const sha512_ops_t sha512_armv8_impl = { .name = "armv8-ce" }; -#elif defined(__arm__) +#elif defined(__arm__) && __ARM_ARCH > 6 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t); const sha512_ops_t sha512_armv7_impl = { .is_supported = sha2_is_supported, @@ -168,7 +168,7 @@ static const sha512_ops_t *const sha512_impls[] = { &sha512_armv7_impl, &sha512_armv8_impl, #endif -#if defined(__arm__) +#if defined(__arm__) && __ARM_ARCH > 6 &sha512_armv7_impl, &sha512_neon_impl, #endif diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 9169244b1a13..b3405b7593f4 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -29,12 +29,12 @@ /* Portions Copyright 2007 Jeremy Teo */ /* Portions Copyright 2010 Robert Milkowski */ - #include #include #include #include #include +#include #include #include #include @@ -85,7 +85,6 @@ #include #include #include -#include #include #include #include @@ -6241,6 +6240,7 @@ zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap) struct mount *mp; struct uio io; int error; + uint64_t len = *ap->a_lenp; /* * TODO: If offset/length is not aligned to recordsize, use @@ -6289,7 +6289,8 @@ zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap) goto unlock; error = zfs_clone_range(VTOZ(invp), ap->a_inoffp, VTOZ(outvp), - ap->a_outoffp, ap->a_lenp, ap->a_fsizetd->td_ucred); + ap->a_outoffp, &len, ap->a_fsizetd->td_ucred); + *ap->a_lenp = (size_t)len; unlock: if (invp != outvp) diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index 1d9b1cffc0b2..619ddef0243a 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -136,8 +136,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index 9c65702b8d43..3db6fd13f4ae 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -30,6 +30,10 @@ /* Portions Copyright 2010 Robert Milkowski */ +#if defined(_KERNEL) +#include +#endif + #include #include #include @@ -1037,8 +1041,6 @@ zfs_prop_align_right(zfs_prop_t prop) #if defined(_KERNEL) -#include - #if defined(HAVE_KERNEL_FPU_INTERNAL) uint8_t **zfs_kfpu_fpregs; EXPORT_SYMBOL(zfs_kfpu_fpregs); diff --git a/module/zfs/vdev_raidz_math.c b/module/zfs/vdev_raidz_math.c index 66f211c430ce..e12b96170f55 100644 --- a/module/zfs/vdev_raidz_math.c +++ b/module/zfs/vdev_raidz_math.c @@ -22,6 +22,7 @@ * Copyright (C) 2016 Gvozden Nešković. All rights reserved. */ +#include #include #include #include @@ -29,7 +30,6 @@ #include #include #include -#include /* Opaque implementation with NULL methods to represent original methods */ static const raidz_impl_ops_t vdev_raidz_original_impl = { From 0b8fdb8ade5d5f0f4ab5be4e488643c2b6c312be Mon Sep 17 00:00:00 2001 From: Andrew Innes Date: Fri, 7 Apr 2023 01:40:23 +0800 Subject: [PATCH 11/59] ZTS: Use inbuilt monotonic time Make the test runner try to use the included python monotonic time function instead of calling librt. This makes the test runner work on macos where librt wasn't available. Reviewed-by: Tino Reichardt Signed-off-by: Andrew Innes Closes #14700 --- tests/test-runner/bin/test-runner.py.in | 34 ++++++++++++------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test-runner/bin/test-runner.py.in b/tests/test-runner/bin/test-runner.py.in index 28276ebc47e3..c454bf8d7c6a 100755 --- a/tests/test-runner/bin/test-runner.py.in +++ b/tests/test-runner/bin/test-runner.py.in @@ -47,25 +47,25 @@ LOG_OUT = 'LOG_OUT' LOG_ERR = 'LOG_ERR' LOG_FILE_OBJ = None +try: + from time import monotonic as monotonic_time +except ImportError: + class timespec(ctypes.Structure): + _fields_ = [ + ('tv_sec', ctypes.c_long), + ('tv_nsec', ctypes.c_long) + ] -class timespec(ctypes.Structure): - _fields_ = [ - ('tv_sec', ctypes.c_long), - ('tv_nsec', ctypes.c_long) - ] + librt = ctypes.CDLL('librt.so.1', use_errno=True) + clock_gettime = librt.clock_gettime + clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] - -librt = ctypes.CDLL('librt.so.1', use_errno=True) -clock_gettime = librt.clock_gettime -clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] - - -def monotonic_time(): - t = timespec() - if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: - errno_ = ctypes.get_errno() - raise OSError(errno_, os.strerror(errno_)) - return t.tv_sec + t.tv_nsec * 1e-9 + def monotonic_time(): + t = timespec() + if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0: + errno_ = ctypes.get_errno() + raise OSError(errno_, os.strerror(errno_)) + return t.tv_sec + t.tv_nsec * 1e-9 class Result(object): From 8ab674ab9c8f4f73fec5582b7186fab633c324e3 Mon Sep 17 00:00:00 2001 From: Damian Szuberski Date: Fri, 7 Apr 2023 03:43:24 +1000 Subject: [PATCH 12/59] ZTS: add existing tests to runfiles Some test cases were committed to the repository but never added to runfiles. Move `zfs_unshare_008_pos` to the Linux-only runfile. Reviewed-by: Brian Behlendorf Signed-off-by: szubersk Closes #14701 --- tests/runfiles/common.run | 15 ++++++++------- tests/runfiles/linux.run | 6 +++++- tests/zfs-tests/tests/Makefile.am | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 50a9309acea5..4233c0285c4b 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -106,7 +106,7 @@ tests = ['tst.destroy_fs', 'tst.destroy_snap', 'tst.get_count_and_limit', 'tst.list_user_props', 'tst.parse_args_neg','tst.promote_conflict', 'tst.promote_multiple', 'tst.promote_simple', 'tst.rollback_mult', 'tst.rollback_one', 'tst.set_props', 'tst.snapshot_destroy', 'tst.snapshot_neg', - 'tst.snapshot_recursive', 'tst.snapshot_simple', + 'tst.snapshot_recursive', 'tst.snapshot_rename', 'tst.snapshot_simple', 'tst.bookmark.create', 'tst.bookmark.copy', 'tst.terminate_by_signal' ] @@ -151,7 +151,8 @@ tags = ['functional', 'cli_root', 'zfs_change-key'] tests = ['zfs_clone_001_neg', 'zfs_clone_002_pos', 'zfs_clone_003_pos', 'zfs_clone_004_pos', 'zfs_clone_005_pos', 'zfs_clone_006_pos', 'zfs_clone_007_pos', 'zfs_clone_008_neg', 'zfs_clone_009_neg', - 'zfs_clone_010_pos', 'zfs_clone_encrypted', 'zfs_clone_deeply_nested'] + 'zfs_clone_010_pos', 'zfs_clone_encrypted', 'zfs_clone_deeply_nested', + 'zfs_clone_rm_nested'] tags = ['functional', 'cli_root', 'zfs_clone'] [tests/functional/cli_root/zfs_copies] @@ -266,8 +267,8 @@ tags = ['functional', 'cli_root', 'zfs_rollback'] [tests/functional/cli_root/zfs_send] tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos', 'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos', - 'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_raw', - 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing'] + 'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_encrypted_unloaded', + 'zfs_send_raw', 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing'] tags = ['functional', 'cli_root', 'zfs_send'] [tests/functional/cli_root/zfs_set] @@ -310,7 +311,7 @@ tags = ['functional', 'cli_root', 'zfs_unmount'] [tests/functional/cli_root/zfs_unshare] tests = ['zfs_unshare_001_pos', 'zfs_unshare_002_pos', 'zfs_unshare_003_pos', 'zfs_unshare_004_neg', 'zfs_unshare_005_neg', 'zfs_unshare_006_pos', - 'zfs_unshare_007_pos', 'zfs_unshare_008_pos'] + 'zfs_unshare_007_pos'] tags = ['functional', 'cli_root', 'zfs_unshare'] [tests/functional/cli_root/zfs_upgrade] @@ -794,13 +795,13 @@ tests = ['removal_all_vdev', 'removal_cancel', 'removal_check_space', 'removal_nopwrite', 'removal_remap_deadlists', 'removal_resume_export', 'removal_sanity', 'removal_with_add', 'removal_with_create_fs', 'removal_with_dedup', - 'removal_with_errors', 'removal_with_export', + 'removal_with_errors', 'removal_with_export', 'removal_with_indirect', 'removal_with_ganging', 'removal_with_faulted', 'removal_with_remove', 'removal_with_scrub', 'removal_with_send', 'removal_with_send_recv', 'removal_with_snapshot', 'removal_with_write', 'removal_with_zdb', 'remove_expanded', 'remove_mirror', 'remove_mirror_sanity', 'remove_raidz', - 'remove_indirect', 'remove_attach_mirror'] + 'remove_indirect', 'remove_attach_mirror', 'removal_reservation'] tags = ['functional', 'removal'] [tests/functional/rename_dirs] diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 15755408b5ad..4df770d61f07 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -52,6 +52,10 @@ tests = ['zfs_share_005_pos', 'zfs_share_007_neg', 'zfs_share_009_neg', 'zfs_share_012_pos', 'zfs_share_013_pos'] tags = ['functional', 'cli_root', 'zfs_share'] +[tests/functional/cli_root/zfs_unshare:Linux] +tests = ['zfs_unshare_008_pos'] +tags = ['functional', 'cli_root', 'zfs_unshare'] + [tests/functional/cli_root/zfs_sysfs:Linux] tests = ['zfeature_set_unsupported', 'zfs_get_unsupported', 'zfs_set_unsupported', 'zfs_sysfs_live', 'zpool_get_unsupported', @@ -121,7 +125,7 @@ post = tags = ['functional', 'largest_pool'] [tests/functional/mmap:Linux] -tests = ['mmap_libaio_001_pos'] +tests = ['mmap_libaio_001_pos', 'mmap_sync_001_pos'] tags = ['functional', 'mmap'] [tests/functional/mmp:Linux] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 92d62b503f65..a470573616af 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1703,6 +1703,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/removal/removal_with_export.ksh \ functional/removal/removal_with_faulted.ksh \ functional/removal/removal_with_ganging.ksh \ + functional/removal/removal_with_indirect.ksh \ functional/removal/removal_with_remove.ksh \ functional/removal/removal_with_scrub.ksh \ functional/removal/removal_with_send.ksh \ From a8a127e2c9f352ba797cd6625d61840b425d6534 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Thu, 6 Apr 2023 19:46:18 +0200 Subject: [PATCH 13/59] Fix typo in check_clones() Run kmem_free() after zap_cursor_fini(). Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Reviewed-by: Adam Moss Signed-off-by: George Amanakis Closes #14702 --- module/zfs/spa_errlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/zfs/spa_errlog.c b/module/zfs/spa_errlog.c index 41cb9d01273c..af144ef16978 100644 --- a/module/zfs/spa_errlog.c +++ b/module/zfs/spa_errlog.c @@ -470,9 +470,9 @@ static int check_clones(spa_t *spa, uint64_t zap_clone, uint64_t snap_count, break; } + zap_cursor_fini(zc); kmem_free(za, sizeof (*za)); kmem_free(zc, sizeof (*zc)); - zap_cursor_fini(zc); return (error); } From ff73574cd83580e4dd5905a43695bd5d0f4911b3 Mon Sep 17 00:00:00 2001 From: Rob N Date: Fri, 7 Apr 2023 03:52:50 +1000 Subject: [PATCH 14/59] vdev: expose zfs_vdev_max_ms_shift as a module parameter Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Reviewed-by: Alexander Motin Signed-off-by: Rob Norris Sponsored-by: Klara, Inc. Sponsored-by: Seagate Technology LLC Closes #14719 --- man/man4/zfs.4 | 5 ++++- module/zfs/vdev.c | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index 566caae7b4e9..d529147464fe 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -357,7 +357,10 @@ and the allocation can't actually be satisfied When a vdev is added, target this number of metaslabs per top-level vdev. . .It Sy zfs_vdev_default_ms_shift Ns = Ns Sy 29 Po 512 MiB Pc Pq uint -Default limit for metaslab size. +Default lower limit for metaslab size. +. +.It Sy zfs_vdev_max_ms_shift Ns = Ns Sy 34 Po 16 GiB Pc Pq uint +Default upper limit for metaslab size. . .It Sy zfs_vdev_max_auto_ashift Ns = Ns Sy 14 Pq uint Maximum ashift used when optimizing for logical \[->] physical sector size on diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index ad932a7ba764..7cf858c05051 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -96,7 +96,7 @@ static uint_t zfs_vdev_ms_count_limit = 1ULL << 17; static uint_t zfs_vdev_default_ms_shift = 29; /* upper limit for metaslab size (16G) */ -static const uint_t zfs_vdev_max_ms_shift = 34; +static uint_t zfs_vdev_max_ms_shift = 34; int vdev_validate_skip = B_FALSE; @@ -6288,7 +6288,10 @@ ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_count, UINT, ZMOD_RW, "Target number of metaslabs per top-level vdev"); ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, default_ms_shift, UINT, ZMOD_RW, - "Default limit for metaslab size"); + "Default lower limit for metaslab size"); + +ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, max_ms_shift, UINT, ZMOD_RW, + "Default upper limit for metaslab size"); ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, min_ms_count, UINT, ZMOD_RW, "Minimum number of metaslabs per top-level vdev"); From baca06c258e07522165cb8e33ff2c0224ad0da57 Mon Sep 17 00:00:00 2001 From: Rob N Date: Tue, 11 Apr 2023 04:53:02 +1000 Subject: [PATCH 15/59] libzfs: add v2 iterator interfaces f6a0dac84 modified the zfs_iter_* functions to take a new "flags" parameter, and introduced a variety of flags to ask the kernel to limit the results in various ways, reducing the amount of work the caller needed to do to filter out things they didn't need. Unfortunately this change broke the ABI for existing clients (read: older versions of the `zfs` program), and was reverted 399b98198. dc95911d2 reintroduced the original patch, with the understanding that a backwards-compatible fix would be made before the 2.2 release branch was tagged. This commit is that fix. This introduces zfs_iter_*_v2 functions that have the new flags argument, and reverts the existing functions to not have the flags parameter, as they were before. The old functions are now reimplemented in terms of the new, with flags set to 0. Reviewed-by: Brian Behlendorf Reviewed-by: George Wilson Original-patch-by: George Wilson Signed-off-by: Rob Norris Sponsored-by: Klara, Inc. Closes #14597 --- cmd/zfs/zfs_iter.c | 6 +- cmd/zfs/zfs_main.c | 30 ++++----- cmd/zpool/zpool_main.c | 2 +- contrib/pam_zfs_key/pam_zfs_key.c | 4 +- include/libzfs.h | 26 ++++++-- lib/libzfs/libzfs.abi | 102 +++++++++++++++++++++++------- lib/libzfs/libzfs_changelist.c | 7 +- lib/libzfs/libzfs_crypto.c | 2 +- lib/libzfs/libzfs_dataset.c | 18 +++--- lib/libzfs/libzfs_iter.c | 75 ++++++++++++++++++---- lib/libzfs/libzfs_mount.c | 4 +- lib/libzfs/libzfs_sendrecv.c | 21 +++--- 12 files changed, 208 insertions(+), 89 deletions(-) diff --git a/cmd/zfs/zfs_iter.c b/cmd/zfs/zfs_iter.c index 6665627d43e3..202cb0e82b5a 100644 --- a/cmd/zfs/zfs_iter.c +++ b/cmd/zfs/zfs_iter.c @@ -143,19 +143,19 @@ zfs_callback(zfs_handle_t *zhp, void *data) (cb->cb_types & (ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME))) && zfs_get_type(zhp) == ZFS_TYPE_FILESYSTEM) { - (void) zfs_iter_filesystems(zhp, cb->cb_flags, + (void) zfs_iter_filesystems_v2(zhp, cb->cb_flags, zfs_callback, data); } if (((zfs_get_type(zhp) & (ZFS_TYPE_SNAPSHOT | ZFS_TYPE_BOOKMARK)) == 0) && include_snaps) { - (void) zfs_iter_snapshots(zhp, cb->cb_flags, + (void) zfs_iter_snapshots_v2(zhp, cb->cb_flags, zfs_callback, data, 0, 0); } if (((zfs_get_type(zhp) & (ZFS_TYPE_SNAPSHOT | ZFS_TYPE_BOOKMARK)) == 0) && include_bmarks) { - (void) zfs_iter_bookmarks(zhp, cb->cb_flags, + (void) zfs_iter_bookmarks_v2(zhp, cb->cb_flags, zfs_callback, data); } diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index d65b01579037..e28f1d04f350 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -1532,7 +1532,8 @@ destroy_print_snapshots(zfs_handle_t *fs_zhp, destroy_cbdata_t *cb) int err; assert(cb->cb_firstsnap == NULL); assert(cb->cb_prevsnap == NULL); - err = zfs_iter_snapshots_sorted(fs_zhp, 0, destroy_print_cb, cb, 0, 0); + err = zfs_iter_snapshots_sorted_v2(fs_zhp, 0, destroy_print_cb, cb, 0, + 0); if (cb->cb_firstsnap != NULL) { uint64_t used = 0; if (err == 0) { @@ -1558,7 +1559,7 @@ snapshot_to_nvl_cb(zfs_handle_t *zhp, void *arg) if (!cb->cb_doclones && !cb->cb_defer_destroy) { cb->cb_target = zhp; cb->cb_first = B_TRUE; - err = zfs_iter_dependents(zhp, 0, B_TRUE, + err = zfs_iter_dependents_v2(zhp, 0, B_TRUE, destroy_check_dependent, cb); } @@ -1576,7 +1577,7 @@ gather_snapshots(zfs_handle_t *zhp, void *arg) destroy_cbdata_t *cb = arg; int err = 0; - err = zfs_iter_snapspec(zhp, 0, cb->cb_snapspec, + err = zfs_iter_snapspec_v2(zhp, 0, cb->cb_snapspec, snapshot_to_nvl_cb, cb); if (err == ENOENT) err = 0; @@ -1590,7 +1591,7 @@ gather_snapshots(zfs_handle_t *zhp, void *arg) } if (cb->cb_recurse) - err = zfs_iter_filesystems(zhp, 0, gather_snapshots, cb); + err = zfs_iter_filesystems_v2(zhp, 0, gather_snapshots, cb); out: zfs_close(zhp); @@ -1615,7 +1616,7 @@ destroy_clones(destroy_cbdata_t *cb) * false while destroying the clones. */ cb->cb_defer_destroy = B_FALSE; - err = zfs_iter_dependents(zhp, 0, B_FALSE, + err = zfs_iter_dependents_v2(zhp, 0, B_FALSE, destroy_callback, cb); cb->cb_defer_destroy = defer; zfs_close(zhp); @@ -1825,9 +1826,8 @@ zfs_do_destroy(int argc, char **argv) * Check for any dependents and/or clones. */ cb.cb_first = B_TRUE; - if (!cb.cb_doclones && - zfs_iter_dependents(zhp, 0, B_TRUE, destroy_check_dependent, - &cb) != 0) { + if (!cb.cb_doclones && zfs_iter_dependents_v2(zhp, 0, B_TRUE, + destroy_check_dependent, &cb) != 0) { rv = 1; goto out; } @@ -1837,7 +1837,7 @@ zfs_do_destroy(int argc, char **argv) goto out; } cb.cb_batchedsnaps = fnvlist_alloc(); - if (zfs_iter_dependents(zhp, 0, B_FALSE, destroy_callback, + if (zfs_iter_dependents_v2(zhp, 0, B_FALSE, destroy_callback, &cb) != 0) { rv = 1; goto out; @@ -4065,7 +4065,7 @@ rollback_check(zfs_handle_t *zhp, void *data) } if (cbp->cb_recurse) { - if (zfs_iter_dependents(zhp, 0, B_TRUE, + if (zfs_iter_dependents_v2(zhp, 0, B_TRUE, rollback_check_dependent, cbp) != 0) { zfs_close(zhp); return (-1); @@ -4164,10 +4164,10 @@ zfs_do_rollback(int argc, char **argv) if (cb.cb_create > 0) min_txg = cb.cb_create; - if ((ret = zfs_iter_snapshots(zhp, 0, rollback_check, &cb, + if ((ret = zfs_iter_snapshots_v2(zhp, 0, rollback_check, &cb, min_txg, 0)) != 0) goto out; - if ((ret = zfs_iter_bookmarks(zhp, 0, rollback_check, &cb)) != 0) + if ((ret = zfs_iter_bookmarks_v2(zhp, 0, rollback_check, &cb)) != 0) goto out; if ((ret = cb.cb_error) != 0) @@ -4309,7 +4309,7 @@ zfs_snapshot_cb(zfs_handle_t *zhp, void *arg) free(name); if (sd->sd_recursive) - rv = zfs_iter_filesystems(zhp, 0, zfs_snapshot_cb, sd); + rv = zfs_iter_filesystems_v2(zhp, 0, zfs_snapshot_cb, sd); zfs_close(zhp); return (rv); } @@ -6373,7 +6373,7 @@ zfs_do_allow_unallow_impl(int argc, char **argv, boolean_t un) if (un && opts.recursive) { struct deleg_perms data = { un, update_perm_nvl }; - if (zfs_iter_filesystems(zhp, 0, set_deleg_perms, + if (zfs_iter_filesystems_v2(zhp, 0, set_deleg_perms, &data) != 0) goto cleanup0; } @@ -6751,7 +6751,7 @@ get_one_dataset(zfs_handle_t *zhp, void *data) /* * Iterate over any nested datasets. */ - if (zfs_iter_filesystems(zhp, 0, get_one_dataset, data) != 0) { + if (zfs_iter_filesystems_v2(zhp, 0, get_one_dataset, data) != 0) { zfs_close(zhp); return (1); } diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 9475beaa2368..20f9cd679534 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -8855,7 +8855,7 @@ check_unsupp_fs(zfs_handle_t *zhp, void *unsupp_fs) (*count)++; } - zfs_iter_filesystems(zhp, 0, check_unsupp_fs, unsupp_fs); + zfs_iter_filesystems_v2(zhp, 0, check_unsupp_fs, unsupp_fs); zfs_close(zhp); diff --git a/contrib/pam_zfs_key/pam_zfs_key.c b/contrib/pam_zfs_key/pam_zfs_key.c index 99cdb8d7733f..6ba5b5fba75f 100644 --- a/contrib/pam_zfs_key/pam_zfs_key.c +++ b/contrib/pam_zfs_key/pam_zfs_key.c @@ -535,8 +535,8 @@ zfs_key_config_get_dataset(zfs_key_config_t *config) return (NULL); } - (void) zfs_iter_filesystems(zhp, 0, find_dsname_by_prop_value, - config); + (void) zfs_iter_filesystems_v2(zhp, 0, + find_dsname_by_prop_value, config); zfs_close(zhp); char *dsname = config->dsname; config->dsname = NULL; diff --git a/include/libzfs.h b/include/libzfs.h index 4f8eeb72ad95..7ec9768d8e93 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -656,17 +656,29 @@ _LIBZFS_H void zprop_print_one_property(const char *, zprop_get_cbdata_t *, typedef int (*zfs_iter_f)(zfs_handle_t *, void *); _LIBZFS_H int zfs_iter_root(libzfs_handle_t *, zfs_iter_f, void *); -_LIBZFS_H int zfs_iter_children(zfs_handle_t *, int, zfs_iter_f, void *); -_LIBZFS_H int zfs_iter_dependents(zfs_handle_t *, int, boolean_t, zfs_iter_f, +_LIBZFS_H int zfs_iter_children(zfs_handle_t *, zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_dependents(zfs_handle_t *, boolean_t, zfs_iter_f, void *); -_LIBZFS_H int zfs_iter_filesystems(zfs_handle_t *, int, zfs_iter_f, void *); -_LIBZFS_H int zfs_iter_snapshots(zfs_handle_t *, int, zfs_iter_f, void *, +_LIBZFS_H int zfs_iter_filesystems(zfs_handle_t *, zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_snapshots(zfs_handle_t *, boolean_t, zfs_iter_f, void *, uint64_t, uint64_t); -_LIBZFS_H int zfs_iter_snapshots_sorted(zfs_handle_t *, int, zfs_iter_f, void *, +_LIBZFS_H int zfs_iter_snapshots_sorted(zfs_handle_t *, zfs_iter_f, void *, uint64_t, uint64_t); -_LIBZFS_H int zfs_iter_snapspec(zfs_handle_t *, int, const char *, zfs_iter_f, +_LIBZFS_H int zfs_iter_snapspec(zfs_handle_t *, const char *, zfs_iter_f, void *); -_LIBZFS_H int zfs_iter_bookmarks(zfs_handle_t *, int, zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_bookmarks(zfs_handle_t *, zfs_iter_f, void *); + +_LIBZFS_H int zfs_iter_children_v2(zfs_handle_t *, int, zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_dependents_v2(zfs_handle_t *, int, boolean_t, zfs_iter_f, + void *); +_LIBZFS_H int zfs_iter_filesystems_v2(zfs_handle_t *, int, zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_snapshots_v2(zfs_handle_t *, int, zfs_iter_f, void *, + uint64_t, uint64_t); +_LIBZFS_H int zfs_iter_snapshots_sorted_v2(zfs_handle_t *, int, zfs_iter_f, + void *, uint64_t, uint64_t); +_LIBZFS_H int zfs_iter_snapspec_v2(zfs_handle_t *, int, const char *, + zfs_iter_f, void *); +_LIBZFS_H int zfs_iter_bookmarks_v2(zfs_handle_t *, int, zfs_iter_f, void *); _LIBZFS_H int zfs_iter_mounted(zfs_handle_t *, zfs_iter_f, void *); typedef struct get_all_cb { diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi index 2b61710f5592..41e74fd8db19 100644 --- a/lib/libzfs/libzfs.abi +++ b/lib/libzfs/libzfs.abi @@ -339,14 +339,21 @@ + + + + + + + @@ -2261,18 +2268,31 @@ - - - - + + + + + + + + + + - - - - - + + + + + + + + + + + + @@ -3305,10 +3325,16 @@ - - - - + + + + + + + + + + @@ -3882,19 +3908,34 @@ - - - - - - + + + + + + + + + + + + + + + - - - - + + + + + + + + + + @@ -5091,6 +5132,14 @@ + + + + + + + + @@ -5100,6 +5149,13 @@ + + + + + + + diff --git a/lib/libzfs/libzfs_changelist.c b/lib/libzfs/libzfs_changelist.c index d7ea60822419..dd14c570ec03 100644 --- a/lib/libzfs/libzfs_changelist.c +++ b/lib/libzfs/libzfs_changelist.c @@ -552,7 +552,7 @@ change_one(zfs_handle_t *zhp, void *data) } if (!clp->cl_alldependents) - ret = zfs_iter_children(zhp, 0, change_one, data); + ret = zfs_iter_children_v2(zhp, 0, change_one, data); /* * If we added the handle to the changelist, we will re-use it @@ -721,11 +721,12 @@ changelist_gather(zfs_handle_t *zhp, zfs_prop_t prop, int gather_flags, return (NULL); } } else if (clp->cl_alldependents) { - if (zfs_iter_dependents(zhp, 0, B_TRUE, change_one, clp) != 0) { + if (zfs_iter_dependents_v2(zhp, 0, B_TRUE, change_one, + clp) != 0) { changelist_free(clp); return (NULL); } - } else if (zfs_iter_children(zhp, 0, change_one, clp) != 0) { + } else if (zfs_iter_children_v2(zhp, 0, change_one, clp) != 0) { changelist_free(clp); return (NULL); } diff --git a/lib/libzfs/libzfs_crypto.c b/lib/libzfs/libzfs_crypto.c index 40059063e21a..8f2a50d55e87 100644 --- a/lib/libzfs/libzfs_crypto.c +++ b/lib/libzfs/libzfs_crypto.c @@ -1226,7 +1226,7 @@ load_keys_cb(zfs_handle_t *zhp, void *arg) cb->cb_numfailed++; out: - (void) zfs_iter_filesystems(zhp, 0, load_keys_cb, cb); + (void) zfs_iter_filesystems_v2(zhp, 0, load_keys_cb, cb); zfs_close(zhp); /* always return 0, since this function is best effort */ diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index 8fa36fa95a17..138eca19acc3 100644 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -757,7 +757,7 @@ zfs_open(libzfs_handle_t *hdl, const char *path, int types) * Iterate bookmarks to find the right one. */ errno = 0; - if ((zfs_iter_bookmarks(pzhp, 0, zfs_open_bookmarks_cb, + if ((zfs_iter_bookmarks_v2(pzhp, 0, zfs_open_bookmarks_cb, &cb_data) == 0) && (cb_data.zhp == NULL)) { (void) zfs_error(hdl, EZFS_NOENT, errbuf); zfs_close(pzhp); @@ -2476,7 +2476,7 @@ get_clones_cb(zfs_handle_t *zhp, void *arg) } out: - (void) zfs_iter_children(zhp, 0, get_clones_cb, gca); + (void) zfs_iter_children_v2(zhp, 0, get_clones_cb, gca); zfs_close(zhp); return (0); } @@ -3925,7 +3925,7 @@ zfs_check_snap_cb(zfs_handle_t *zhp, void *arg) if (lzc_exists(name)) fnvlist_add_boolean(dd->nvl, name); - rv = zfs_iter_filesystems(zhp, 0, zfs_check_snap_cb, dd); + rv = zfs_iter_filesystems_v2(zhp, 0, zfs_check_snap_cb, dd); zfs_close(zhp); return (rv); } @@ -4163,7 +4163,7 @@ zfs_snapshot_cb(zfs_handle_t *zhp, void *arg) fnvlist_add_boolean(sd->sd_nvl, name); - rv = zfs_iter_filesystems(zhp, 0, zfs_snapshot_cb, sd); + rv = zfs_iter_filesystems_v2(zhp, 0, zfs_snapshot_cb, sd); } zfs_close(zhp); @@ -4340,7 +4340,7 @@ rollback_destroy(zfs_handle_t *zhp, void *data) rollback_data_t *cbp = data; if (zfs_prop_get_int(zhp, ZFS_PROP_CREATETXG) > cbp->cb_create) { - cbp->cb_error |= zfs_iter_dependents(zhp, 0, B_FALSE, + cbp->cb_error |= zfs_iter_dependents_v2(zhp, 0, B_FALSE, rollback_destroy_dependent, cbp); cbp->cb_error |= zfs_destroy(zhp, B_FALSE); @@ -4380,10 +4380,10 @@ zfs_rollback(zfs_handle_t *zhp, zfs_handle_t *snap, boolean_t force) if (cb.cb_create > 0) min_txg = cb.cb_create; - (void) zfs_iter_snapshots(zhp, 0, rollback_destroy, &cb, + (void) zfs_iter_snapshots_v2(zhp, 0, rollback_destroy, &cb, min_txg, 0); - (void) zfs_iter_bookmarks(zhp, 0, rollback_destroy, &cb); + (void) zfs_iter_bookmarks_v2(zhp, 0, rollback_destroy, &cb); if (cb.cb_error) return (-1); @@ -4964,7 +4964,7 @@ zfs_hold_one(zfs_handle_t *zhp, void *arg) fnvlist_add_string(ha->nvl, name, ha->tag); if (ha->recursive) - rv = zfs_iter_filesystems(zhp, 0, zfs_hold_one, ha); + rv = zfs_iter_filesystems_v2(zhp, 0, zfs_hold_one, ha); zfs_close(zhp); return (rv); } @@ -5095,7 +5095,7 @@ zfs_release_one(zfs_handle_t *zhp, void *arg) } if (ha->recursive) - rv = zfs_iter_filesystems(zhp, 0, zfs_release_one, ha); + rv = zfs_iter_filesystems_v2(zhp, 0, zfs_release_one, ha); zfs_close(zhp); return (rv); } diff --git a/lib/libzfs/libzfs_iter.c b/lib/libzfs/libzfs_iter.c index 681fe5b4748d..452d8fd6ab71 100644 --- a/lib/libzfs/libzfs_iter.c +++ b/lib/libzfs/libzfs_iter.c @@ -103,7 +103,14 @@ zfs_do_list_ioctl(zfs_handle_t *zhp, int arg, zfs_cmd_t *zc) * Iterate over all child filesystems */ int -zfs_iter_filesystems(zfs_handle_t *zhp, int flags, zfs_iter_f func, void *data) +zfs_iter_filesystems(zfs_handle_t *zhp, zfs_iter_f func, void *data) +{ + return (zfs_iter_filesystems_v2(zhp, 0, func, data)); +} + +int +zfs_iter_filesystems_v2(zfs_handle_t *zhp, int flags, zfs_iter_f func, + void *data) { zfs_cmd_t zc = {"\0"}; zfs_handle_t *nzhp; @@ -143,7 +150,15 @@ zfs_iter_filesystems(zfs_handle_t *zhp, int flags, zfs_iter_f func, void *data) * Iterate over all snapshots */ int -zfs_iter_snapshots(zfs_handle_t *zhp, int flags, zfs_iter_f func, +zfs_iter_snapshots(zfs_handle_t *zhp, boolean_t simple, zfs_iter_f func, + void *data, uint64_t min_txg, uint64_t max_txg) +{ + return (zfs_iter_snapshots_v2(zhp, simple ? ZFS_ITER_SIMPLE : 0, func, + data, min_txg, max_txg)); +} + +int +zfs_iter_snapshots_v2(zfs_handle_t *zhp, int flags, zfs_iter_f func, void *data, uint64_t min_txg, uint64_t max_txg) { zfs_cmd_t zc = {"\0"}; @@ -197,7 +212,13 @@ zfs_iter_snapshots(zfs_handle_t *zhp, int flags, zfs_iter_f func, * Iterate over all bookmarks */ int -zfs_iter_bookmarks(zfs_handle_t *zhp, int flags __maybe_unused, +zfs_iter_bookmarks(zfs_handle_t *zhp, zfs_iter_f func, void *data) +{ + return (zfs_iter_bookmarks_v2(zhp, 0, func, data)); +} + +int +zfs_iter_bookmarks_v2(zfs_handle_t *zhp, int flags __maybe_unused, zfs_iter_f func, void *data) { zfs_handle_t *nzhp; @@ -305,7 +326,15 @@ zfs_snapshot_compare(const void *larg, const void *rarg) } int -zfs_iter_snapshots_sorted(zfs_handle_t *zhp, int flags, zfs_iter_f callback, +zfs_iter_snapshots_sorted(zfs_handle_t *zhp, zfs_iter_f callback, + void *data, uint64_t min_txg, uint64_t max_txg) +{ + return (zfs_iter_snapshots_sorted_v2(zhp, 0, callback, data, + min_txg, max_txg)); +} + +int +zfs_iter_snapshots_sorted_v2(zfs_handle_t *zhp, int flags, zfs_iter_f callback, void *data, uint64_t min_txg, uint64_t max_txg) { int ret = 0; @@ -316,7 +345,7 @@ zfs_iter_snapshots_sorted(zfs_handle_t *zhp, int flags, zfs_iter_f callback, avl_create(&avl, zfs_snapshot_compare, sizeof (zfs_node_t), offsetof(zfs_node_t, zn_avlnode)); - ret = zfs_iter_snapshots(zhp, flags, zfs_sort_snaps, &avl, min_txg, + ret = zfs_iter_snapshots_v2(zhp, flags, zfs_sort_snaps, &avl, min_txg, max_txg); for (node = avl_first(&avl); node != NULL; node = AVL_NEXT(&avl, node)) @@ -379,7 +408,14 @@ snapspec_cb(zfs_handle_t *zhp, void *arg) * return ENOENT at the end. */ int -zfs_iter_snapspec(zfs_handle_t *fs_zhp, int flags, const char *spec_orig, +zfs_iter_snapspec(zfs_handle_t *fs_zhp, const char *spec_orig, + zfs_iter_f func, void *arg) +{ + return (zfs_iter_snapspec_v2(fs_zhp, 0, spec_orig, func, arg)); +} + +int +zfs_iter_snapspec_v2(zfs_handle_t *fs_zhp, int flags, const char *spec_orig, zfs_iter_f func, void *arg) { char *buf, *comma_separated, *cp; @@ -419,7 +455,7 @@ zfs_iter_snapspec(zfs_handle_t *fs_zhp, int flags, const char *spec_orig, } } - err = zfs_iter_snapshots_sorted(fs_zhp, flags, + err = zfs_iter_snapshots_sorted_v2(fs_zhp, flags, snapspec_cb, &ssa, 0, 0); if (ret == 0) ret = err; @@ -456,14 +492,20 @@ zfs_iter_snapspec(zfs_handle_t *fs_zhp, int flags, const char *spec_orig, * and as close as possible. */ int -zfs_iter_children(zfs_handle_t *zhp, int flags, zfs_iter_f func, void *data) +zfs_iter_children(zfs_handle_t *zhp, zfs_iter_f func, void *data) +{ + return (zfs_iter_children_v2(zhp, 0, func, data)); +} + +int +zfs_iter_children_v2(zfs_handle_t *zhp, int flags, zfs_iter_f func, void *data) { int ret; - if ((ret = zfs_iter_snapshots(zhp, flags, func, data, 0, 0)) != 0) + if ((ret = zfs_iter_snapshots_v2(zhp, flags, func, data, 0, 0)) != 0) return (ret); - return (zfs_iter_filesystems(zhp, flags, func, data)); + return (zfs_iter_filesystems_v2(zhp, flags, func, data)); } @@ -524,10 +566,10 @@ iter_dependents_cb(zfs_handle_t *zhp, void *arg) isf.zhp = zhp; isf.next = ida->stack; ida->stack = &isf; - err = zfs_iter_filesystems(zhp, ida->flags, + err = zfs_iter_filesystems_v2(zhp, ida->flags, iter_dependents_cb, ida); if (err == 0) - err = zfs_iter_snapshots(zhp, ida->flags, + err = zfs_iter_snapshots_v2(zhp, ida->flags, iter_dependents_cb, ida, 0, 0); ida->stack = isf.next; } @@ -541,7 +583,14 @@ iter_dependents_cb(zfs_handle_t *zhp, void *arg) } int -zfs_iter_dependents(zfs_handle_t *zhp, int flags, boolean_t allowrecursion, +zfs_iter_dependents(zfs_handle_t *zhp, boolean_t allowrecursion, + zfs_iter_f func, void *data) +{ + return (zfs_iter_dependents_v2(zhp, 0, allowrecursion, func, data)); +} + +int +zfs_iter_dependents_v2(zfs_handle_t *zhp, int flags, boolean_t allowrecursion, zfs_iter_f func, void *data) { iter_dependents_arg_t ida; diff --git a/lib/libzfs/libzfs_mount.c b/lib/libzfs/libzfs_mount.c index 024f449baa0b..5d1fe651c97e 100644 --- a/lib/libzfs/libzfs_mount.c +++ b/lib/libzfs/libzfs_mount.c @@ -940,7 +940,7 @@ zfs_iter_cb(zfs_handle_t *zhp, void *data) } libzfs_add_handle(cbp, zhp); - if (zfs_iter_filesystems(zhp, 0, zfs_iter_cb, cbp) != 0) { + if (zfs_iter_filesystems_v2(zhp, 0, zfs_iter_cb, cbp) != 0) { zfs_close(zhp); return (-1); } @@ -1289,7 +1289,7 @@ zpool_enable_datasets(zpool_handle_t *zhp, const char *mntopts, int flags) * over all child filesystems. */ libzfs_add_handle(&cb, zfsp); - if (zfs_iter_filesystems(zfsp, 0, zfs_iter_cb, &cb) != 0) + if (zfs_iter_filesystems_v2(zfsp, 0, zfs_iter_cb, &cb) != 0) goto out; /* diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 23402f86a8ae..87a30f54fea8 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -288,7 +288,7 @@ send_iterate_prop(zfs_handle_t *zhp, boolean_t received_only, nvlist_t *nv); /* * Collect guid, valid props, optionally holds, etc. of a snapshot. - * This interface is intended for use as a zfs_iter_snapshots_sorted visitor. + * This interface is intended for use as a zfs_iter_snapshots_v2_sorted visitor. */ static int send_iterate_snap(zfs_handle_t *zhp, void *arg) @@ -619,8 +619,8 @@ send_iterate_fs(zfs_handle_t *zhp, void *arg) min_txg = fromsnap_txg; if (!sd->replicate && tosnap_txg != 0) max_txg = tosnap_txg; - (void) zfs_iter_snapshots_sorted(zhp, 0, send_iterate_snap, sd, - min_txg, max_txg); + (void) zfs_iter_snapshots_sorted_v2(zhp, 0, send_iterate_snap, + sd, min_txg, max_txg); } else { char snapname[MAXPATHLEN] = { 0 }; zfs_handle_t *snap; @@ -662,7 +662,7 @@ send_iterate_fs(zfs_handle_t *zhp, void *arg) /* Iterate over children. */ if (sd->recursive) - rv = zfs_iter_filesystems(zhp, 0, send_iterate_fs, sd); + rv = zfs_iter_filesystems_v2(zhp, 0, send_iterate_fs, sd); out: /* Restore saved fields. */ @@ -1083,7 +1083,7 @@ send_print_verbose(FILE *fout, const char *tosnap, const char *fromsnap, /* * Send a single filesystem snapshot, updating the send dump data. - * This interface is intended for use as a zfs_iter_snapshots_sorted visitor. + * This interface is intended for use as a zfs_iter_snapshots_v2_sorted visitor. */ static int dump_snapshot(zfs_handle_t *zhp, void *arg) @@ -1293,7 +1293,7 @@ dump_filesystem(zfs_handle_t *zhp, send_dump_data_t *sdd) zhp->zfs_name, sdd->tosnap); } } - rv = zfs_iter_snapshots_sorted(zhp, 0, dump_snapshot, sdd, + rv = zfs_iter_snapshots_sorted_v2(zhp, 0, dump_snapshot, sdd, min_txg, max_txg); } else { char snapname[MAXPATHLEN] = { 0 }; @@ -3162,9 +3162,9 @@ guid_to_name_cb(zfs_handle_t *zhp, void *arg) return (EEXIST); } - err = zfs_iter_children(zhp, 0, guid_to_name_cb, gtnd); + err = zfs_iter_children_v2(zhp, 0, guid_to_name_cb, gtnd); if (err != EEXIST && gtnd->bookmark_ok) - err = zfs_iter_bookmarks(zhp, 0, guid_to_name_cb, gtnd); + err = zfs_iter_bookmarks_v2(zhp, 0, guid_to_name_cb, gtnd); zfs_close(zhp); return (err); } @@ -3218,9 +3218,10 @@ guid_to_name_redact_snaps(libzfs_handle_t *hdl, const char *parent, continue; int err = guid_to_name_cb(zfs_handle_dup(zhp), >nd); if (err != EEXIST) - err = zfs_iter_children(zhp, 0, guid_to_name_cb, >nd); + err = zfs_iter_children_v2(zhp, 0, guid_to_name_cb, + >nd); if (err != EEXIST && bookmark_ok) - err = zfs_iter_bookmarks(zhp, 0, guid_to_name_cb, + err = zfs_iter_bookmarks_v2(zhp, 0, guid_to_name_cb, >nd); zfs_close(zhp); if (err == EEXIST) From dee77f45d0f961de0b421f36bbde4196624a13c5 Mon Sep 17 00:00:00 2001 From: Kyle Evans Date: Fri, 7 Apr 2023 00:40:34 +0000 Subject: [PATCH 16/59] module: resync part of Makefile.bsd sha256-armv8.S and sha512-armv8.S need the same treatment as the sse bits; removal of -mgeneral-regs-only from flags. This fixes errors about requiring NEON, which is a difference in clang vs. gcc treatment of -mgeneral-regs-only being specified on asm files. Reviewed-by: Richard Yao Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Kyle Evans Closes #14715 --- module/Makefile.bsd | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/module/Makefile.bsd b/module/Makefile.bsd index 8ec094d4ad1c..365609fb8585 100644 --- a/module/Makefile.bsd +++ b/module/Makefile.bsd @@ -507,6 +507,16 @@ CFLAGS.zstd_lazy.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_ldm.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} CFLAGS.zstd_opt.c+= ${__ZFS_ZSTD_AARCH64_FLAGS} +sha256-armv8.o: sha256-armv8.S + ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ + -o ${.TARGET} + ${CTFCONVERT_CMD} + +sha512-armv8.o: sha512-armv8.S + ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ + -o ${.TARGET} + ${CTFCONVERT_CMD} + b3_aarch64_sse2.o: b3_aarch64_sse2.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC} \ -o ${.TARGET} From d0cbd9feaf5b82130f2e679256c71e0c7413aae9 Mon Sep 17 00:00:00 2001 From: Kyle Evans Date: Fri, 7 Apr 2023 00:40:34 +0000 Subject: [PATCH 17/59] module: freebsd: fix aarch64 fpu handling Just like x86, aarch64 needs to use the fpu_kern(9) API around FPU usage, otherwise we panic promptly at boot as soon as ZFS attempts to do checksum benchmarking. Reviewed-by: Richard Yao Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Kyle Evans Closes #14715 --- include/os/freebsd/spl/sys/simd_aarch64.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/include/os/freebsd/spl/sys/simd_aarch64.h b/include/os/freebsd/spl/sys/simd_aarch64.h index df33bdaeccf8..234f401db791 100644 --- a/include/os/freebsd/spl/sys/simd_aarch64.h +++ b/include/os/freebsd/spl/sys/simd_aarch64.h @@ -44,13 +44,23 @@ #define _FREEBSD_SIMD_AARCH64_H #include +#include #include +#include #include +#include #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) +#define kfpu_begin() do { \ + if (__predict_false(!is_fpu_kern_thread(0))) \ + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); \ +} while (0) + +#define kfpu_end() do { \ + if (__predict_false(curthread->td_pcb->pcb_fpflags & PCB_FP_NOSAVE)) \ + fpu_kern_leave(curthread, NULL); \ +} while (0) #define kfpu_init() (0) #define kfpu_fini() do {} while (0) From d4dc53dad2f6c3a2d107f1ba0e8d66228c845e00 Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Mon, 10 Apr 2023 17:15:36 -0400 Subject: [PATCH 18/59] Linux 6.3 compat: idmapped mount API changes Linux kernel 6.3 changed a bunch of APIs to use the dedicated idmap type for mounts (struct mnt_idmap), we need to detect these changes and make zfs work with the new APIs. Reviewed-by: Brian Behlendorf Signed-off-by: Youzhong Yang Closes #14682 --- config/kernel-acl.m4 | 34 ++++++-- config/kernel-generic_fillattr.m4 | 37 ++++++-- config/kernel-inode-create.m4 | 41 +++++++-- config/kernel-inode-getattr.m4 | 63 ++++++++++---- config/kernel-inode-permission.m4 | 35 ++++++-- config/kernel-inode-setattr.m4 | 87 +++++++++++++++++++ config/kernel-is_owner_or_cap.m4 | 25 +++++- config/kernel-mkdir.m4 | 55 +++++++++--- config/kernel-mknod.m4 | 34 +++++++- config/kernel-rename.m4 | 56 ++++++++---- config/kernel-setattr-prepare.m4 | 44 +++++++--- config/kernel-symlink.m4 | 33 +++++-- config/kernel-tmpfile.m4 | 33 +++++-- config/kernel-xattr-handler.m4 | 91 +++++++++++++------- config/kernel.m4 | 6 +- include/os/freebsd/spl/sys/types.h | 2 +- include/os/freebsd/zfs/sys/zfs_vnops_os.h | 10 +-- include/os/linux/kernel/linux/vfs_compat.h | 21 ++++- include/os/linux/kernel/linux/xattr_compat.h | 17 +++- include/os/linux/spl/sys/cred.h | 30 ++++--- include/os/linux/spl/sys/types.h | 15 +++- include/os/linux/zfs/sys/policy.h | 6 +- include/os/linux/zfs/sys/zfs_vnops_os.h | 15 ++-- include/os/linux/zfs/sys/zpl.h | 13 ++- include/sys/zfs_acl.h | 10 +-- module/os/freebsd/zfs/zfs_acl.c | 10 +-- module/os/freebsd/zfs/zfs_vnops_os.c | 10 +-- module/os/linux/spl/spl-cred.c | 12 +++ module/os/linux/zfs/policy.c | 13 +-- module/os/linux/zfs/zfs_acl.c | 33 ++++--- module/os/linux/zfs/zfs_dir.c | 4 +- module/os/linux/zfs/zfs_ioctl_os.c | 4 + module/os/linux/zfs/zfs_vnops_os.c | 35 ++++---- module/os/linux/zfs/zfs_znode.c | 2 +- module/os/linux/zfs/zpl_ctldir.c | 53 +++++++++--- module/os/linux/zfs/zpl_file.c | 10 +-- module/os/linux/zfs/zpl_inode.c | 77 ++++++++++++----- module/os/linux/zfs/zpl_xattr.c | 25 +++--- module/zfs/zfs_replay.c | 14 +-- module/zfs/zfs_vnops.c | 4 +- 40 files changed, 823 insertions(+), 296 deletions(-) create mode 100644 config/kernel-inode-setattr.m4 diff --git a/config/kernel-acl.m4 b/config/kernel-acl.m4 index 6e92da97d0fe..be08c3c60724 100644 --- a/config/kernel-acl.m4 +++ b/config/kernel-acl.m4 @@ -236,7 +236,22 @@ dnl # dnl # 6.2 API change, dnl # set_acl() second paramter changed to a struct dentry * dnl # +dnl # 6.3 API change, +dnl # set_acl() first parameter changed to struct mnt_idmap * +dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OPERATIONS_SET_ACL], [ + ZFS_LINUX_TEST_SRC([inode_operations_set_acl_mnt_idmap_dentry], [ + #include + + int set_acl_fn(struct mnt_idmap *idmap, + struct dentry *dent, struct posix_acl *acl, + int type) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .set_acl = set_acl_fn, + }; + ],[]) ZFS_LINUX_TEST_SRC([inode_operations_set_acl_userns_dentry], [ #include @@ -281,17 +296,24 @@ AC_DEFUN([ZFS_AC_KERNEL_INODE_OPERATIONS_SET_ACL], [ AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists]) AC_DEFINE(HAVE_SET_ACL_USERNS, 1, [iops->set_acl() takes 4 args]) ],[ - ZFS_LINUX_TEST_RESULT([inode_operations_set_acl_userns_dentry], [ + ZFS_LINUX_TEST_RESULT([inode_operations_set_acl_mnt_idmap_dentry], [ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists]) - AC_DEFINE(HAVE_SET_ACL_USERNS_DENTRY_ARG2, 1, - [iops->set_acl() takes 4 args, arg2 is struct dentry *]) + AC_DEFINE(HAVE_SET_ACL_IDMAP_DENTRY, 1, + [iops->set_acl() takes 4 args, arg1 is struct mnt_idmap *]) ],[ - ZFS_LINUX_TEST_RESULT([inode_operations_set_acl], [ + ZFS_LINUX_TEST_RESULT([inode_operations_set_acl_userns_dentry], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists, takes 3 args]) + AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists]) + AC_DEFINE(HAVE_SET_ACL_USERNS_DENTRY_ARG2, 1, + [iops->set_acl() takes 4 args, arg2 is struct dentry *]) ],[ - ZFS_LINUX_REQUIRE_API([i_op->set_acl()], [3.14]) + ZFS_LINUX_TEST_RESULT([inode_operations_set_acl], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_SET_ACL, 1, [iops->set_acl() exists, takes 3 args]) + ],[ + ZFS_LINUX_REQUIRE_API([i_op->set_acl()], [3.14]) + ]) ]) ]) ]) diff --git a/config/kernel-generic_fillattr.m4 b/config/kernel-generic_fillattr.m4 index 0acd5d53103f..02dee4d4c000 100644 --- a/config/kernel-generic_fillattr.m4 +++ b/config/kernel-generic_fillattr.m4 @@ -4,7 +4,10 @@ dnl # dnl # generic_fillattr in linux/fs.h now requires a struct user_namespace* dnl # as the first arg, to support idmapped mounts. dnl # -AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR_USERNS], [ +dnl # 6.3 API +dnl # generic_fillattr() now takes struct mnt_idmap* as the first argument +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR], [ ZFS_LINUX_TEST_SRC([generic_fillattr_userns], [ #include ],[ @@ -13,16 +16,32 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR_USERNS], [ struct kstat *k = NULL; generic_fillattr(userns, in, k); ]) -]) -AC_DEFUN([ZFS_AC_KERNEL_GENERIC_FILLATTR_USERNS], [ - AC_MSG_CHECKING([whether generic_fillattr requires struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([generic_fillattr_userns], [ - AC_MSG_RESULT([yes]) - AC_DEFINE(HAVE_GENERIC_FILLATTR_USERNS, 1, - [generic_fillattr requires struct user_namespace*]) + ZFS_LINUX_TEST_SRC([generic_fillattr_mnt_idmap], [ + #include ],[ - AC_MSG_RESULT([no]) + struct mnt_idmap *idmap = NULL; + struct inode *in = NULL; + struct kstat *k = NULL; + generic_fillattr(idmap, in, k); + ]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_GENERIC_FILLATTR], [ + AC_MSG_CHECKING([whether generic_fillattr requires struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([generic_fillattr_mnt_idmap], [ + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_GENERIC_FILLATTR_IDMAP, 1, + [generic_fillattr requires struct mnt_idmap*]) + ],[ + AC_MSG_CHECKING([whether generic_fillattr requires struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([generic_fillattr_userns], [ + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_GENERIC_FILLATTR_USERNS, 1, + [generic_fillattr requires struct user_namespace*]) + ],[ + AC_MSG_RESULT([no]) + ]) ]) ]) diff --git a/config/kernel-inode-create.m4 b/config/kernel-inode-create.m4 index a6ea11fb61b2..9e9e43180976 100644 --- a/config/kernel-inode-create.m4 +++ b/config/kernel-inode-create.m4 @@ -1,4 +1,22 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_CREATE], [ + dnl # + dnl # 6.3 API change + dnl # The first arg is changed to struct mnt_idmap * + dnl # + ZFS_LINUX_TEST_SRC([create_mnt_idmap], [ + #include + #include + + int inode_create(struct mnt_idmap *idmap, + struct inode *inode ,struct dentry *dentry, + umode_t umode, bool flag) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .create = inode_create, + }; + ],[]) + dnl # dnl # 5.12 API change that added the struct user_namespace* arg dnl # to the front of this function type's arg list. @@ -35,19 +53,28 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_CREATE], [ ]) AC_DEFUN([ZFS_AC_KERNEL_CREATE], [ - AC_MSG_CHECKING([whether iops->create() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([create_userns], [ + AC_MSG_CHECKING([whether iops->create() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([create_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_CREATE_USERNS, 1, - [iops->create() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_CREATE_IDMAP, 1, + [iops->create() takes struct mnt_idmap*]) ],[ AC_MSG_RESULT(no) - AC_MSG_CHECKING([whether iops->create() passes flags]) - ZFS_LINUX_TEST_RESULT([create_flags], [ + AC_MSG_CHECKING([whether iops->create() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([create_userns], [ AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IOPS_CREATE_USERNS, 1, + [iops->create() takes struct user_namespace*]) ],[ - ZFS_LINUX_TEST_ERROR([iops->create()]) + AC_MSG_RESULT(no) + + AC_MSG_CHECKING([whether iops->create() passes flags]) + ZFS_LINUX_TEST_RESULT([create_flags], [ + AC_MSG_RESULT(yes) + ],[ + ZFS_LINUX_TEST_ERROR([iops->create()]) + ]) ]) ]) ]) diff --git a/config/kernel-inode-getattr.m4 b/config/kernel-inode-getattr.m4 index f62e82f5230a..c8bfb07862ab 100644 --- a/config/kernel-inode-getattr.m4 +++ b/config/kernel-inode-getattr.m4 @@ -1,4 +1,24 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [ + dnl # + dnl # Linux 6.3 API + dnl # The first arg of getattr I/O operations handler type + dnl # is changed to struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([inode_operations_getattr_mnt_idmap], [ + #include + + int test_getattr( + struct mnt_idmap *idmap, + const struct path *p, struct kstat *k, + u32 request_mask, unsigned int query_flags) + { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .getattr = test_getattr, + }; + ],[]) + dnl # dnl # Linux 5.12 API dnl # The getattr I/O operations handler type was extended to require @@ -55,37 +75,48 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_GETATTR], [ AC_DEFUN([ZFS_AC_KERNEL_INODE_GETATTR], [ dnl # - dnl # Kernel 5.12 test + dnl # Kernel 6.3 test dnl # - AC_MSG_CHECKING([whether iops->getattr() takes user_namespace]) - ZFS_LINUX_TEST_RESULT([inode_operations_getattr_userns], [ + AC_MSG_CHECKING([whether iops->getattr() takes mnt_idmap]) + ZFS_LINUX_TEST_RESULT([inode_operations_getattr_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_USERNS_IOPS_GETATTR, 1, - [iops->getattr() takes struct user_namespace*]) + AC_DEFINE(HAVE_IDMAP_IOPS_GETATTR, 1, + [iops->getattr() takes struct mnt_idmap*]) ],[ AC_MSG_RESULT(no) - dnl # - dnl # Kernel 4.11 test + dnl # Kernel 5.12 test dnl # - AC_MSG_CHECKING([whether iops->getattr() takes a path]) - ZFS_LINUX_TEST_RESULT([inode_operations_getattr_path], [ + AC_MSG_CHECKING([whether iops->getattr() takes user_namespace]) + ZFS_LINUX_TEST_RESULT([inode_operations_getattr_userns], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_PATH_IOPS_GETATTR, 1, - [iops->getattr() takes a path]) + AC_DEFINE(HAVE_USERNS_IOPS_GETATTR, 1, + [iops->getattr() takes struct user_namespace*]) ],[ AC_MSG_RESULT(no) dnl # - dnl # Kernel < 4.11 test + dnl # Kernel 4.11 test dnl # - AC_MSG_CHECKING([whether iops->getattr() takes a vfsmount]) - ZFS_LINUX_TEST_RESULT([inode_operations_getattr_vfsmount], [ + AC_MSG_CHECKING([whether iops->getattr() takes a path]) + ZFS_LINUX_TEST_RESULT([inode_operations_getattr_path], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_VFSMOUNT_IOPS_GETATTR, 1, - [iops->getattr() takes a vfsmount]) + AC_DEFINE(HAVE_PATH_IOPS_GETATTR, 1, + [iops->getattr() takes a path]) ],[ AC_MSG_RESULT(no) + + dnl # + dnl # Kernel < 4.11 test + dnl # + AC_MSG_CHECKING([whether iops->getattr() takes a vfsmount]) + ZFS_LINUX_TEST_RESULT([inode_operations_getattr_vfsmount], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_VFSMOUNT_IOPS_GETATTR, 1, + [iops->getattr() takes a vfsmount]) + ],[ + AC_MSG_RESULT(no) + ]) ]) ]) ]) diff --git a/config/kernel-inode-permission.m4 b/config/kernel-inode-permission.m4 index ba9ff5d43d4d..01d23635b0c9 100644 --- a/config/kernel-inode-permission.m4 +++ b/config/kernel-inode-permission.m4 @@ -1,4 +1,22 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PERMISSION], [ + dnl # + dnl # 6.3 API change + dnl # iops->permission() now takes struct mnt_idmap* + dnl # as its first arg + dnl # + ZFS_LINUX_TEST_SRC([permission_mnt_idmap], [ + #include + #include + + int inode_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .permission = inode_permission, + }; + ],[]) + dnl # dnl # 5.12 API change that added the struct user_namespace* arg dnl # to the front of this function type's arg list. @@ -18,12 +36,19 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_PERMISSION], [ ]) AC_DEFUN([ZFS_AC_KERNEL_PERMISSION], [ - AC_MSG_CHECKING([whether iops->permission() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([permission_userns], [ + AC_MSG_CHECKING([whether iops->permission() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([permission_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_PERMISSION_USERNS, 1, - [iops->permission() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_PERMISSION_IDMAP, 1, + [iops->permission() takes struct mnt_idmap*]) ],[ - AC_MSG_RESULT(no) + AC_MSG_CHECKING([whether iops->permission() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([permission_userns], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IOPS_PERMISSION_USERNS, 1, + [iops->permission() takes struct user_namespace*]) + ],[ + AC_MSG_RESULT(no) + ]) ]) ]) diff --git a/config/kernel-inode-setattr.m4 b/config/kernel-inode-setattr.m4 new file mode 100644 index 000000000000..45755b4eb273 --- /dev/null +++ b/config/kernel-inode-setattr.m4 @@ -0,0 +1,87 @@ +AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_SETATTR], [ + dnl # + dnl # Linux 6.3 API + dnl # The first arg of setattr I/O operations handler type + dnl # is changed to struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([inode_operations_setattr_mnt_idmap], [ + #include + + int test_setattr( + struct mnt_idmap *idmap, + struct dentry *de, struct iattr *ia) + { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .setattr = test_setattr, + }; + ],[]) + + dnl # + dnl # Linux 5.12 API + dnl # The setattr I/O operations handler type was extended to require + dnl # a struct user_namespace* as its first arg, to support idmapped + dnl # mounts. + dnl # + ZFS_LINUX_TEST_SRC([inode_operations_setattr_userns], [ + #include + + int test_setattr( + struct user_namespace *userns, + struct dentry *de, struct iattr *ia) + { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .setattr = test_setattr, + }; + ],[]) + + ZFS_LINUX_TEST_SRC([inode_operations_setattr], [ + #include + + int test_setattr( + struct dentry *de, struct iattr *ia) + { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .setattr = test_setattr, + }; + ],[]) +]) + +AC_DEFUN([ZFS_AC_KERNEL_INODE_SETATTR], [ + dnl # + dnl # Kernel 6.3 test + dnl # + AC_MSG_CHECKING([whether iops->setattr() takes mnt_idmap]) + ZFS_LINUX_TEST_RESULT([inode_operations_setattr_mnt_idmap], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IDMAP_IOPS_SETATTR, 1, + [iops->setattr() takes struct mnt_idmap*]) + ],[ + AC_MSG_RESULT(no) + dnl # + dnl # Kernel 5.12 test + dnl # + AC_MSG_CHECKING([whether iops->setattr() takes user_namespace]) + ZFS_LINUX_TEST_RESULT([inode_operations_setattr_userns], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_USERNS_IOPS_SETATTR, 1, + [iops->setattr() takes struct user_namespace*]) + ],[ + AC_MSG_RESULT(no) + + AC_MSG_CHECKING([whether iops->setattr() exists]) + ZFS_LINUX_TEST_RESULT([inode_operations_setattr], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IOPS_SETATTR, 1, + [iops->setattr() exists]) + ],[ + AC_MSG_RESULT(no) + ]) + ]) + ]) +]) diff --git a/config/kernel-is_owner_or_cap.m4 b/config/kernel-is_owner_or_cap.m4 index a90cf3da641d..4e9c002b77f2 100644 --- a/config/kernel-is_owner_or_cap.m4 +++ b/config/kernel-is_owner_or_cap.m4 @@ -16,12 +16,20 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_INODE_OWNER_OR_CAPABLE], [ (void) inode_owner_or_capable(ip); ]) - ZFS_LINUX_TEST_SRC([inode_owner_or_capable_idmapped], [ + ZFS_LINUX_TEST_SRC([inode_owner_or_capable_userns], [ #include ],[ struct inode *ip = NULL; (void) inode_owner_or_capable(&init_user_ns, ip); ]) + + ZFS_LINUX_TEST_SRC([inode_owner_or_capable_mnt_idmap], [ + #include + #include + ],[ + struct inode *ip = NULL; + (void) inode_owner_or_capable(&nop_mnt_idmap, ip); + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_INODE_OWNER_OR_CAPABLE], [ @@ -35,12 +43,21 @@ AC_DEFUN([ZFS_AC_KERNEL_INODE_OWNER_OR_CAPABLE], [ AC_MSG_CHECKING( [whether inode_owner_or_capable() takes user_ns]) - ZFS_LINUX_TEST_RESULT([inode_owner_or_capable_idmapped], [ + ZFS_LINUX_TEST_RESULT([inode_owner_or_capable_userns], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_INODE_OWNER_OR_CAPABLE_IDMAPPED, 1, + AC_DEFINE(HAVE_INODE_OWNER_OR_CAPABLE_USERNS, 1, [inode_owner_or_capable() takes user_ns]) ],[ - ZFS_LINUX_TEST_ERROR([capability]) + AC_MSG_RESULT(no) + AC_MSG_CHECKING( + [whether inode_owner_or_capable() takes mnt_idmap]) + ZFS_LINUX_TEST_RESULT([inode_owner_or_capable_mnt_idmap], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_INODE_OWNER_OR_CAPABLE_IDMAP, 1, + [inode_owner_or_capable() takes mnt_idmap]) + ], [ + ZFS_LINUX_TEST_ERROR([capability]) + ]) ]) ]) ]) diff --git a/config/kernel-mkdir.m4 b/config/kernel-mkdir.m4 index 6667ed04fa4c..7407a791b846 100644 --- a/config/kernel-mkdir.m4 +++ b/config/kernel-mkdir.m4 @@ -2,6 +2,22 @@ dnl # dnl # Supported mkdir() interfaces checked newest to oldest. dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [ + dnl # + dnl # 6.3 API change + dnl # mkdir() takes struct mnt_idmap * as the first arg + dnl # + ZFS_LINUX_TEST_SRC([mkdir_mnt_idmap], [ + #include + + int mkdir(struct mnt_idmap *idmap, + struct inode *inode, struct dentry *dentry, + umode_t umode) { return 0; } + static const struct inode_operations + iops __attribute__ ((unused)) = { + .mkdir = mkdir, + }; + ],[]) + dnl # dnl # 5.12 API change dnl # The struct user_namespace arg was added as the first argument to @@ -43,25 +59,36 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [ AC_DEFUN([ZFS_AC_KERNEL_MKDIR], [ dnl # - dnl # 5.12 API change - dnl # The struct user_namespace arg was added as the first argument to - dnl # mkdir() of the iops structure. + dnl # 6.3 API change + dnl # mkdir() takes struct mnt_idmap * as the first arg dnl # - AC_MSG_CHECKING([whether iops->mkdir() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([mkdir_user_namespace], [ + AC_MSG_CHECKING([whether iops->mkdir() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([mkdir_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_MKDIR_USERNS, 1, - [iops->mkdir() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_MKDIR_IDMAP, 1, + [iops->mkdir() takes struct mnt_idmap*]) ],[ - AC_MSG_RESULT(no) - - AC_MSG_CHECKING([whether iops->mkdir() takes umode_t]) - ZFS_LINUX_TEST_RESULT([inode_operations_mkdir], [ + dnl # + dnl # 5.12 API change + dnl # The struct user_namespace arg was added as the first argument to + dnl # mkdir() of the iops structure. + dnl # + AC_MSG_CHECKING([whether iops->mkdir() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([mkdir_user_namespace], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_MKDIR_UMODE_T, 1, - [iops->mkdir() takes umode_t]) + AC_DEFINE(HAVE_IOPS_MKDIR_USERNS, 1, + [iops->mkdir() takes struct user_namespace*]) ],[ - ZFS_LINUX_TEST_ERROR([mkdir()]) + AC_MSG_RESULT(no) + + AC_MSG_CHECKING([whether iops->mkdir() takes umode_t]) + ZFS_LINUX_TEST_RESULT([inode_operations_mkdir], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_MKDIR_UMODE_T, 1, + [iops->mkdir() takes umode_t]) + ],[ + ZFS_LINUX_TEST_ERROR([mkdir()]) + ]) ]) ]) ]) diff --git a/config/kernel-mknod.m4 b/config/kernel-mknod.m4 index ffe45106003a..1494ec1ae4d4 100644 --- a/config/kernel-mknod.m4 +++ b/config/kernel-mknod.m4 @@ -1,4 +1,22 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKNOD], [ + dnl # + dnl # 6.3 API change + dnl # The first arg is now struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([mknod_mnt_idmap], [ + #include + #include + + int tmp_mknod(struct mnt_idmap *idmap, + struct inode *inode ,struct dentry *dentry, + umode_t u, dev_t d) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .mknod = tmp_mknod, + }; + ],[]) + dnl # dnl # 5.12 API change that added the struct user_namespace* arg dnl # to the front of this function type's arg list. @@ -19,12 +37,20 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_MKNOD], [ ]) AC_DEFUN([ZFS_AC_KERNEL_MKNOD], [ - AC_MSG_CHECKING([whether iops->mknod() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([mknod_userns], [ + AC_MSG_CHECKING([whether iops->mknod() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([mknod_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_MKNOD_USERNS, 1, - [iops->mknod() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_MKNOD_IDMAP, 1, + [iops->mknod() takes struct mnt_idmap*]) ],[ AC_MSG_RESULT(no) + AC_MSG_CHECKING([whether iops->mknod() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([mknod_userns], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IOPS_MKNOD_USERNS, 1, + [iops->mknod() takes struct user_namespace*]) + ],[ + AC_MSG_RESULT(no) + ]) ]) ]) diff --git a/config/kernel-rename.m4 b/config/kernel-rename.m4 index a2b0800ab4d2..57c3eed78974 100644 --- a/config/kernel-rename.m4 +++ b/config/kernel-rename.m4 @@ -71,39 +71,61 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_RENAME], [ .rename = rename_fn, }; ],[]) + + dnl # + dnl # 6.3 API change - the first arg is now struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([inode_operations_rename_mnt_idmap], [ + #include + int rename_fn(struct mnt_idmap *idmap, struct inode *sip, + struct dentry *sdp, struct inode *tip, struct dentry *tdp, + unsigned int flags) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .rename = rename_fn, + }; + ],[]) ]) AC_DEFUN([ZFS_AC_KERNEL_RENAME], [ - AC_MSG_CHECKING([whether iops->rename() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([inode_operations_rename_userns], [ + AC_MSG_CHECKING([whether iops->rename() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([inode_operations_rename_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_RENAME_USERNS, 1, - [iops->rename() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_RENAME_IDMAP, 1, + [iops->rename() takes struct mnt_idmap*]) ],[ - AC_MSG_RESULT(no) - - AC_MSG_CHECKING([whether iops->rename2() exists]) - ZFS_LINUX_TEST_RESULT([inode_operations_rename2], [ + AC_MSG_CHECKING([whether iops->rename() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([inode_operations_rename_userns], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_RENAME2, 1, [iops->rename2() exists]) + AC_DEFINE(HAVE_IOPS_RENAME_USERNS, 1, + [iops->rename() takes struct user_namespace*]) ],[ AC_MSG_RESULT(no) - AC_MSG_CHECKING([whether iops->rename() wants flags]) - ZFS_LINUX_TEST_RESULT([inode_operations_rename_flags], [ + AC_MSG_CHECKING([whether iops->rename2() exists]) + ZFS_LINUX_TEST_RESULT([inode_operations_rename2], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_RENAME_WANTS_FLAGS, 1, - [iops->rename() wants flags]) + AC_DEFINE(HAVE_RENAME2, 1, [iops->rename2() exists]) ],[ AC_MSG_RESULT(no) - AC_MSG_CHECKING([whether struct inode_operations_wrapper takes .rename2()]) - ZFS_LINUX_TEST_RESULT([dir_inode_operations_wrapper_rename2], [ + AC_MSG_CHECKING([whether iops->rename() wants flags]) + ZFS_LINUX_TEST_RESULT([inode_operations_rename_flags], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_RENAME2_OPERATIONS_WRAPPER, 1, - [struct inode_operations_wrapper takes .rename2()]) + AC_DEFINE(HAVE_RENAME_WANTS_FLAGS, 1, + [iops->rename() wants flags]) ],[ AC_MSG_RESULT(no) + + AC_MSG_CHECKING([whether struct inode_operations_wrapper takes .rename2()]) + ZFS_LINUX_TEST_RESULT([dir_inode_operations_wrapper_rename2], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_RENAME2_OPERATIONS_WRAPPER, 1, + [struct inode_operations_wrapper takes .rename2()]) + ],[ + AC_MSG_RESULT(no) + ]) ]) ]) ]) diff --git a/config/kernel-setattr-prepare.m4 b/config/kernel-setattr-prepare.m4 index 24245aa53448..e02d6263e9c9 100644 --- a/config/kernel-setattr-prepare.m4 +++ b/config/kernel-setattr-prepare.m4 @@ -27,26 +27,48 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SETATTR_PREPARE], [ int error __attribute__ ((unused)) = setattr_prepare(userns, dentry, attr); ]) + + dnl # + dnl # 6.3 API change + dnl # The first arg of setattr_prepare() is changed to struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([setattr_prepare_mnt_idmap], [ + #include + ], [ + struct dentry *dentry = NULL; + struct iattr *attr = NULL; + struct mnt_idmap *idmap = NULL; + int error __attribute__ ((unused)) = + setattr_prepare(idmap, dentry, attr); + ]) ]) AC_DEFUN([ZFS_AC_KERNEL_SETATTR_PREPARE], [ - AC_MSG_CHECKING([whether setattr_prepare() is available and accepts struct user_namespace*]) - ZFS_LINUX_TEST_RESULT_SYMBOL([setattr_prepare_userns], + AC_MSG_CHECKING([whether setattr_prepare() is available and accepts struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT_SYMBOL([setattr_prepare_mnt_idmap], [setattr_prepare], [fs/attr.c], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_SETATTR_PREPARE_USERNS, 1, - [setattr_prepare() accepts user_namespace]) + AC_DEFINE(HAVE_SETATTR_PREPARE_IDMAP, 1, + [setattr_prepare() accepts mnt_idmap]) ], [ - AC_MSG_RESULT(no) - - AC_MSG_CHECKING([whether setattr_prepare() is available, doesn't accept user_namespace]) - ZFS_LINUX_TEST_RESULT_SYMBOL([setattr_prepare], - [setattr_prepare], [fs/attr.c], [ + AC_MSG_CHECKING([whether setattr_prepare() is available and accepts struct user_namespace*]) + ZFS_LINUX_TEST_RESULT_SYMBOL([setattr_prepare_userns], + [setattr_prepare], [fs/attr.c], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_SETATTR_PREPARE_NO_USERNS, 1, - [setattr_prepare() is available, doesn't accept user_namespace]) + AC_DEFINE(HAVE_SETATTR_PREPARE_USERNS, 1, + [setattr_prepare() accepts user_namespace]) ], [ AC_MSG_RESULT(no) + + AC_MSG_CHECKING([whether setattr_prepare() is available, doesn't accept user_namespace]) + ZFS_LINUX_TEST_RESULT_SYMBOL([setattr_prepare], + [setattr_prepare], [fs/attr.c], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_SETATTR_PREPARE_NO_USERNS, 1, + [setattr_prepare() is available, doesn't accept user_namespace]) + ], [ + AC_MSG_RESULT(no) + ]) ]) ]) ]) diff --git a/config/kernel-symlink.m4 b/config/kernel-symlink.m4 index d90366d04b72..a0333ed66a7c 100644 --- a/config/kernel-symlink.m4 +++ b/config/kernel-symlink.m4 @@ -1,4 +1,20 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SYMLINK], [ + dnl # + dnl # 6.3 API change that changed the first arg + dnl # to struct mnt_idmap* + dnl # + ZFS_LINUX_TEST_SRC([symlink_mnt_idmap], [ + #include + #include + int tmp_symlink(struct mnt_idmap *idmap, + struct inode *inode ,struct dentry *dentry, + const char *path) { return 0; } + + static const struct inode_operations + iops __attribute__ ((unused)) = { + .symlink = tmp_symlink, + }; + ],[]) dnl # dnl # 5.12 API change that added the struct user_namespace* arg dnl # to the front of this function type's arg list. @@ -19,12 +35,19 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_SYMLINK], [ ]) AC_DEFUN([ZFS_AC_KERNEL_SYMLINK], [ - AC_MSG_CHECKING([whether iops->symlink() takes struct user_namespace*]) - ZFS_LINUX_TEST_RESULT([symlink_userns], [ + AC_MSG_CHECKING([whether iops->symlink() takes struct mnt_idmap*]) + ZFS_LINUX_TEST_RESULT([symlink_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_IOPS_SYMLINK_USERNS, 1, - [iops->symlink() takes struct user_namespace*]) + AC_DEFINE(HAVE_IOPS_SYMLINK_IDMAP, 1, + [iops->symlink() takes struct mnt_idmap*]) ],[ - AC_MSG_RESULT(no) + AC_MSG_CHECKING([whether iops->symlink() takes struct user_namespace*]) + ZFS_LINUX_TEST_RESULT([symlink_userns], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_IOPS_SYMLINK_USERNS, 1, + [iops->symlink() takes struct user_namespace*]) + ],[ + AC_MSG_RESULT(no) + ]) ]) ]) diff --git a/config/kernel-tmpfile.m4 b/config/kernel-tmpfile.m4 index 0e1deb3612f3..cc18b8f65a88 100644 --- a/config/kernel-tmpfile.m4 +++ b/config/kernel-tmpfile.m4 @@ -4,6 +4,19 @@ dnl # Add support for i_op->tmpfile dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [ dnl # + dnl # 6.3 API change + dnl # The first arg is now struct mnt_idmap * + dnl # + ZFS_LINUX_TEST_SRC([inode_operations_tmpfile_mnt_idmap], [ + #include + int tmpfile(struct mnt_idmap *idmap, + struct inode *inode, struct file *file, + umode_t mode) { return 0; } + static struct inode_operations + iops __attribute__ ((unused)) = { + .tmpfile = tmpfile, + }; + ],[]) dnl # 6.1 API change dnl # use struct file instead of struct dentry dnl # @@ -44,23 +57,29 @@ AC_DEFUN([ZFS_AC_KERNEL_SRC_TMPFILE], [ AC_DEFUN([ZFS_AC_KERNEL_TMPFILE], [ AC_MSG_CHECKING([whether i_op->tmpfile() exists]) - ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile], [ + ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_mnt_idmap], [ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists]) - AC_DEFINE(HAVE_TMPFILE_USERNS, 1, [i_op->tmpfile() has userns]) - ],[ - ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_dentry_userns], [ + AC_DEFINE(HAVE_TMPFILE_IDMAP, 1, [i_op->tmpfile() has mnt_idmap]) + ], [ + ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile], [ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists]) AC_DEFINE(HAVE_TMPFILE_USERNS, 1, [i_op->tmpfile() has userns]) - AC_DEFINE(HAVE_TMPFILE_DENTRY, 1, [i_op->tmpfile() uses old dentry signature]) ],[ - ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_dentry], [ + ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_dentry_userns], [ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists]) + AC_DEFINE(HAVE_TMPFILE_USERNS, 1, [i_op->tmpfile() has userns]) AC_DEFINE(HAVE_TMPFILE_DENTRY, 1, [i_op->tmpfile() uses old dentry signature]) ],[ - ZFS_LINUX_REQUIRE_API([i_op->tmpfile()], [3.11]) + ZFS_LINUX_TEST_RESULT([inode_operations_tmpfile_dentry], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_TMPFILE, 1, [i_op->tmpfile() exists]) + AC_DEFINE(HAVE_TMPFILE_DENTRY, 1, [i_op->tmpfile() uses old dentry signature]) + ],[ + ZFS_LINUX_REQUIRE_API([i_op->tmpfile()], [3.11]) + ]) ]) ]) ]) diff --git a/config/kernel-xattr-handler.m4 b/config/kernel-xattr-handler.m4 index b6cbfa155007..6b8a08dbcc80 100644 --- a/config/kernel-xattr-handler.m4 +++ b/config/kernel-xattr-handler.m4 @@ -179,6 +179,21 @@ dnl # dnl # Supported xattr handler set() interfaces checked newest to oldest. dnl # AC_DEFUN([ZFS_AC_KERNEL_SRC_XATTR_HANDLER_SET], [ + ZFS_LINUX_TEST_SRC([xattr_handler_set_mnt_idmap], [ + #include + + int set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) + { return 0; } + static const struct xattr_handler + xops __attribute__ ((unused)) = { + .set = set, + }; + ],[]) + ZFS_LINUX_TEST_SRC([xattr_handler_set_userns], [ #include @@ -240,53 +255,63 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_SET], [ dnl # The xattr_handler->set() callback was changed to 8 arguments, and dnl # struct user_namespace* was inserted as arg #2 dnl # - AC_MSG_CHECKING([whether xattr_handler->set() wants dentry, inode, and user_namespace]) - ZFS_LINUX_TEST_RESULT([xattr_handler_set_userns], [ + dnl # 6.3 API change, + dnl # The xattr_handler->set() callback 2nd arg is now struct mnt_idmap * + dnl # + AC_MSG_CHECKING([whether xattr_handler->set() wants dentry, inode, and mnt_idmap]) + ZFS_LINUX_TEST_RESULT([xattr_handler_set_mnt_idmap], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_USERNS, 1, - [xattr_handler->set() takes user_namespace]) - ],[ - dnl # - dnl # 4.7 API change, - dnl # The xattr_handler->set() callback was changed to take both - dnl # dentry and inode. - dnl # - AC_MSG_RESULT(no) - AC_MSG_CHECKING([whether xattr_handler->set() wants dentry and inode]) - ZFS_LINUX_TEST_RESULT([xattr_handler_set_dentry_inode], [ + AC_DEFINE(HAVE_XATTR_SET_IDMAP, 1, + [xattr_handler->set() takes mnt_idmap]) + ], [ + AC_MSG_CHECKING([whether xattr_handler->set() wants dentry, inode, and user_namespace]) + ZFS_LINUX_TEST_RESULT([xattr_handler_set_userns], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_DENTRY_INODE, 1, - [xattr_handler->set() wants both dentry and inode]) + AC_DEFINE(HAVE_XATTR_SET_USERNS, 1, + [xattr_handler->set() takes user_namespace]) ],[ dnl # - dnl # 4.4 API change, - dnl # The xattr_handler->set() callback was changed to take a - dnl # xattr_handler, and handler_flags argument was removed and - dnl # should be accessed by handler->flags. + dnl # 4.7 API change, + dnl # The xattr_handler->set() callback was changed to take both + dnl # dentry and inode. dnl # AC_MSG_RESULT(no) - AC_MSG_CHECKING( - [whether xattr_handler->set() wants xattr_handler]) - ZFS_LINUX_TEST_RESULT([xattr_handler_set_xattr_handler], [ + AC_MSG_CHECKING([whether xattr_handler->set() wants dentry and inode]) + ZFS_LINUX_TEST_RESULT([xattr_handler_set_dentry_inode], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_HANDLER, 1, - [xattr_handler->set() wants xattr_handler]) + AC_DEFINE(HAVE_XATTR_SET_DENTRY_INODE, 1, + [xattr_handler->set() wants both dentry and inode]) ],[ dnl # - dnl # 2.6.33 API change, - dnl # The xattr_handler->set() callback was changed - dnl # to take a dentry instead of an inode, and a - dnl # handler_flags argument was added. + dnl # 4.4 API change, + dnl # The xattr_handler->set() callback was changed to take a + dnl # xattr_handler, and handler_flags argument was removed and + dnl # should be accessed by handler->flags. dnl # AC_MSG_RESULT(no) AC_MSG_CHECKING( - [whether xattr_handler->set() wants dentry]) - ZFS_LINUX_TEST_RESULT([xattr_handler_set_dentry], [ + [whether xattr_handler->set() wants xattr_handler]) + ZFS_LINUX_TEST_RESULT([xattr_handler_set_xattr_handler], [ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_DENTRY, 1, - [xattr_handler->set() wants dentry]) + AC_DEFINE(HAVE_XATTR_SET_HANDLER, 1, + [xattr_handler->set() wants xattr_handler]) ],[ - ZFS_LINUX_TEST_ERROR([xattr set()]) + dnl # + dnl # 2.6.33 API change, + dnl # The xattr_handler->set() callback was changed + dnl # to take a dentry instead of an inode, and a + dnl # handler_flags argument was added. + dnl # + AC_MSG_RESULT(no) + AC_MSG_CHECKING( + [whether xattr_handler->set() wants dentry]) + ZFS_LINUX_TEST_RESULT([xattr_handler_set_dentry], [ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_XATTR_SET_DENTRY, 1, + [xattr_handler->set() wants dentry]) + ],[ + ZFS_LINUX_TEST_ERROR([xattr set()]) + ]) ]) ]) ]) diff --git a/config/kernel.m4 b/config/kernel.m4 index fb07f5004d3c..439ffdf5a898 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -71,6 +71,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_INODE_OWNER_OR_CAPABLE ZFS_AC_KERNEL_SRC_XATTR ZFS_AC_KERNEL_SRC_ACL + ZFS_AC_KERNEL_SRC_INODE_SETATTR ZFS_AC_KERNEL_SRC_INODE_GETATTR ZFS_AC_KERNEL_SRC_INODE_SET_FLAGS ZFS_AC_KERNEL_SRC_INODE_SET_IVERSION @@ -133,7 +134,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [ ZFS_AC_KERNEL_SRC_KSTRTOUL ZFS_AC_KERNEL_SRC_PERCPU ZFS_AC_KERNEL_SRC_CPU_HOTPLUG - ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR_USERNS + ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR ZFS_AC_KERNEL_SRC_MKNOD ZFS_AC_KERNEL_SRC_SYMLINK ZFS_AC_KERNEL_SRC_BIO_MAX_SEGS @@ -202,6 +203,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_INODE_OWNER_OR_CAPABLE ZFS_AC_KERNEL_XATTR ZFS_AC_KERNEL_ACL + ZFS_AC_KERNEL_INODE_SETATTR ZFS_AC_KERNEL_INODE_GETATTR ZFS_AC_KERNEL_INODE_SET_FLAGS ZFS_AC_KERNEL_INODE_SET_IVERSION @@ -264,7 +266,7 @@ AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [ ZFS_AC_KERNEL_KSTRTOUL ZFS_AC_KERNEL_PERCPU ZFS_AC_KERNEL_CPU_HOTPLUG - ZFS_AC_KERNEL_GENERIC_FILLATTR_USERNS + ZFS_AC_KERNEL_GENERIC_FILLATTR ZFS_AC_KERNEL_MKNOD ZFS_AC_KERNEL_SYMLINK ZFS_AC_KERNEL_BIO_MAX_SEGS diff --git a/include/os/freebsd/spl/sys/types.h b/include/os/freebsd/spl/sys/types.h index 558843dcaa74..ebc93f4f4485 100644 --- a/include/os/freebsd/spl/sys/types.h +++ b/include/os/freebsd/spl/sys/types.h @@ -105,7 +105,7 @@ typedef u_longlong_t len_t; typedef longlong_t diskaddr_t; -typedef void zuserns_t; +typedef void zidmap_t; #include #endif /* !_OPENSOLARIS_SYS_TYPES_H_ */ diff --git a/include/os/freebsd/zfs/sys/zfs_vnops_os.h b/include/os/freebsd/zfs/sys/zfs_vnops_os.h index 839ee629a5ab..eddcab575b91 100644 --- a/include/os/freebsd/zfs/sys/zfs_vnops_os.h +++ b/include/os/freebsd/zfs/sys/zfs_vnops_os.h @@ -35,23 +35,23 @@ int dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, int *rbehind, int *rahead, int last_size); extern int zfs_remove(znode_t *dzp, const char *name, cred_t *cr, int flags); extern int zfs_mkdir(znode_t *dzp, const char *dirname, vattr_t *vap, - znode_t **zpp, cred_t *cr, int flags, vsecattr_t *vsecp, zuserns_t *mnt_ns); + znode_t **zpp, cred_t *cr, int flags, vsecattr_t *vsecp, zidmap_t *mnt_ns); extern int zfs_rmdir(znode_t *dzp, const char *name, znode_t *cwd, cred_t *cr, int flags); extern int zfs_setattr(znode_t *zp, vattr_t *vap, int flag, cred_t *cr, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_rename(znode_t *sdzp, const char *snm, znode_t *tdzp, const char *tnm, cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap, - const char *link, znode_t **zpp, cred_t *cr, int flags, zuserns_t *mnt_ns); + const char *link, znode_t **zpp, cred_t *cr, int flags, zidmap_t *mnt_ns); extern int zfs_link(znode_t *tdzp, znode_t *sp, const char *name, cred_t *cr, int flags); extern int zfs_space(znode_t *zp, int cmd, struct flock *bfp, int flag, offset_t offset, cred_t *cr); extern int zfs_create(znode_t *dzp, const char *name, vattr_t *vap, int excl, int mode, znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr); extern int zfs_write_simple(znode_t *zp, const void *data, size_t len, diff --git a/include/os/linux/kernel/linux/vfs_compat.h b/include/os/linux/kernel/linux/vfs_compat.h index fd0b9e8e1068..e156ed41c28c 100644 --- a/include/os/linux/kernel/linux/vfs_compat.h +++ b/include/os/linux/kernel/linux/vfs_compat.h @@ -341,7 +341,8 @@ static inline void zfs_gid_write(struct inode *ip, gid_t gid) * 4.9 API change */ #if !(defined(HAVE_SETATTR_PREPARE_NO_USERNS) || \ - defined(HAVE_SETATTR_PREPARE_USERNS)) + defined(HAVE_SETATTR_PREPARE_USERNS) || \ + defined(HAVE_SETATTR_PREPARE_IDMAP)) static inline int setattr_prepare(struct dentry *dentry, struct iattr *ia) { @@ -396,6 +397,15 @@ func(struct user_namespace *user_ns, const struct path *path, \ return (func##_impl(user_ns, path, stat, request_mask, \ query_flags)); \ } +#elif defined(HAVE_IDMAP_IOPS_GETATTR) +#define ZPL_GETATTR_WRAPPER(func) \ +static int \ +func(struct mnt_idmap *user_ns, const struct path *path, \ + struct kstat *stat, u32 request_mask, unsigned int query_flags) \ +{ \ + return (func##_impl(user_ns, path, stat, request_mask, \ + query_flags)); \ +} #else #error #endif @@ -447,8 +457,15 @@ zpl_is_32bit_api(void) * 5.12 API change * To support id-mapped mounts, generic_fillattr() was modified to * accept a new struct user_namespace* as its first arg. + * + * 6.3 API change + * generic_fillattr() first arg is changed to struct mnt_idmap * + * */ -#ifdef HAVE_GENERIC_FILLATTR_USERNS +#ifdef HAVE_GENERIC_FILLATTR_IDMAP +#define zpl_generic_fillattr(idmap, ip, sp) \ + generic_fillattr(idmap, ip, sp) +#elif defined(HAVE_GENERIC_FILLATTR_USERNS) #define zpl_generic_fillattr(user_ns, ip, sp) \ generic_fillattr(user_ns, ip, sp) #else diff --git a/include/os/linux/kernel/linux/xattr_compat.h b/include/os/linux/kernel/linux/xattr_compat.h index ff80fbb06413..bcc7289ad857 100644 --- a/include/os/linux/kernel/linux/xattr_compat.h +++ b/include/os/linux/kernel/linux/xattr_compat.h @@ -133,13 +133,28 @@ fn(const struct xattr_handler *handler, struct dentry *dentry, \ #error "Unsupported kernel" #endif +/* + * 6.3 API change, + * The xattr_handler->set() callback was changed to take the + * struct mnt_idmap* as the first arg, to support idmapped + * mounts. + */ +#if defined(HAVE_XATTR_SET_IDMAP) +#define ZPL_XATTR_SET_WRAPPER(fn) \ +static int \ +fn(const struct xattr_handler *handler, struct mnt_idmap *user_ns, \ + struct dentry *dentry, struct inode *inode, const char *name, \ + const void *buffer, size_t size, int flags) \ +{ \ + return (__ ## fn(user_ns, inode, name, buffer, size, flags)); \ +} /* * 5.12 API change, * The xattr_handler->set() callback was changed to take the * struct user_namespace* as the first arg, to support idmapped * mounts. */ -#if defined(HAVE_XATTR_SET_USERNS) +#elif defined(HAVE_XATTR_SET_USERNS) #define ZPL_XATTR_SET_WRAPPER(fn) \ static int \ fn(const struct xattr_handler *handler, struct user_namespace *user_ns, \ diff --git a/include/os/linux/spl/sys/cred.h b/include/os/linux/spl/sys/cred.h index 75ad400d312d..7fd5f644863f 100644 --- a/include/os/linux/spl/sys/cred.h +++ b/include/os/linux/spl/sys/cred.h @@ -48,6 +48,8 @@ extern struct task_struct init_task; #define SGID_TO_KGID(x) (KGIDT_INIT(x)) #define KGIDP_TO_SGIDP(x) (&(x)->val) +extern zidmap_t *zfs_get_init_idmap(void); + /* Check if the user ns is the initial one */ static inline boolean_t zfs_is_init_userns(struct user_namespace *user_ns) @@ -74,36 +76,39 @@ static inline boolean_t zfs_no_idmapping(struct user_namespace *mnt_userns, return (zfs_is_init_userns(mnt_userns) || mnt_userns == fs_userns); } -static inline uid_t zfs_uid_to_vfsuid(struct user_namespace *mnt_userns, +static inline uid_t zfs_uid_to_vfsuid(zidmap_t *mnt_userns, struct user_namespace *fs_userns, uid_t uid) { - if (zfs_no_idmapping(mnt_userns, fs_userns)) + struct user_namespace *owner = idmap_owner(mnt_userns); + if (zfs_no_idmapping(owner, fs_userns)) return (uid); if (!zfs_is_init_userns(fs_userns)) uid = from_kuid(fs_userns, KUIDT_INIT(uid)); if (uid == (uid_t)-1) return (uid); - return (__kuid_val(make_kuid(mnt_userns, uid))); + return (__kuid_val(make_kuid(owner, uid))); } -static inline gid_t zfs_gid_to_vfsgid(struct user_namespace *mnt_userns, +static inline gid_t zfs_gid_to_vfsgid(zidmap_t *mnt_userns, struct user_namespace *fs_userns, gid_t gid) { - if (zfs_no_idmapping(mnt_userns, fs_userns)) + struct user_namespace *owner = idmap_owner(mnt_userns); + if (zfs_no_idmapping(owner, fs_userns)) return (gid); if (!zfs_is_init_userns(fs_userns)) gid = from_kgid(fs_userns, KGIDT_INIT(gid)); if (gid == (gid_t)-1) return (gid); - return (__kgid_val(make_kgid(mnt_userns, gid))); + return (__kgid_val(make_kgid(owner, gid))); } -static inline uid_t zfs_vfsuid_to_uid(struct user_namespace *mnt_userns, +static inline uid_t zfs_vfsuid_to_uid(zidmap_t *mnt_userns, struct user_namespace *fs_userns, uid_t uid) { - if (zfs_no_idmapping(mnt_userns, fs_userns)) + struct user_namespace *owner = idmap_owner(mnt_userns); + if (zfs_no_idmapping(owner, fs_userns)) return (uid); - uid = from_kuid(mnt_userns, KUIDT_INIT(uid)); + uid = from_kuid(owner, KUIDT_INIT(uid)); if (uid == (uid_t)-1) return (uid); if (zfs_is_init_userns(fs_userns)) @@ -111,12 +116,13 @@ static inline uid_t zfs_vfsuid_to_uid(struct user_namespace *mnt_userns, return (__kuid_val(make_kuid(fs_userns, uid))); } -static inline gid_t zfs_vfsgid_to_gid(struct user_namespace *mnt_userns, +static inline gid_t zfs_vfsgid_to_gid(zidmap_t *mnt_userns, struct user_namespace *fs_userns, gid_t gid) { - if (zfs_no_idmapping(mnt_userns, fs_userns)) + struct user_namespace *owner = idmap_owner(mnt_userns); + if (zfs_no_idmapping(owner, fs_userns)) return (gid); - gid = from_kgid(mnt_userns, KGIDT_INIT(gid)); + gid = from_kgid(owner, KGIDT_INIT(gid)); if (gid == (gid_t)-1) return (gid); if (zfs_is_init_userns(fs_userns)) diff --git a/include/os/linux/spl/sys/types.h b/include/os/linux/spl/sys/types.h index cae1bbddf105..a7666187ec23 100644 --- a/include/os/linux/spl/sys/types.h +++ b/include/os/linux/spl/sys/types.h @@ -55,6 +55,19 @@ typedef int major_t; typedef int minor_t; struct user_namespace; -typedef struct user_namespace zuserns_t; +#ifdef HAVE_IOPS_CREATE_IDMAP +#include +struct mnt_idmap { + struct user_namespace *owner; + refcount_t count; +}; +typedef struct mnt_idmap zidmap_t; +#define idmap_owner(p) (((struct mnt_idmap *)p)->owner) +#else +typedef struct user_namespace zidmap_t; +#define idmap_owner(p) ((struct user_namespace *)p) +#endif + +extern zidmap_t *zfs_init_idmap; #endif /* _SPL_TYPES_H */ diff --git a/include/os/linux/zfs/sys/policy.h b/include/os/linux/zfs/sys/policy.h index b182da95b8e0..0c265db78591 100644 --- a/include/os/linux/zfs/sys/policy.h +++ b/include/os/linux/zfs/sys/policy.h @@ -47,14 +47,14 @@ int secpolicy_vnode_create_gid(const cred_t *); int secpolicy_vnode_remove(const cred_t *); int secpolicy_vnode_setdac(const cred_t *, uid_t); int secpolicy_vnode_setid_retain(struct znode *, const cred_t *, boolean_t); -int secpolicy_vnode_setids_setgids(const cred_t *, gid_t, zuserns_t *, - zuserns_t *); +int secpolicy_vnode_setids_setgids(const cred_t *, gid_t, zidmap_t *, + struct user_namespace *); int secpolicy_zinject(const cred_t *); int secpolicy_zfs(const cred_t *); int secpolicy_zfs_proc(const cred_t *, proc_t *); void secpolicy_setid_clear(vattr_t *, cred_t *); int secpolicy_setid_setsticky_clear(struct inode *, vattr_t *, - const vattr_t *, cred_t *, zuserns_t *, zuserns_t *); + const vattr_t *, cred_t *, zidmap_t *, struct user_namespace *); int secpolicy_xvattr(xvattr_t *, uid_t, cred_t *, mode_t); int secpolicy_vnode_setattr(cred_t *, struct inode *, struct vattr *, const struct vattr *, int, int (void *, int, cred_t *), void *); diff --git a/include/os/linux/zfs/sys/zfs_vnops_os.h b/include/os/linux/zfs/sys/zfs_vnops_os.h index 1caec0ef4f1d..7a1db7deeec8 100644 --- a/include/os/linux/zfs/sys/zfs_vnops_os.h +++ b/include/os/linux/zfs/sys/zfs_vnops_os.h @@ -46,25 +46,24 @@ extern int zfs_lookup(znode_t *dzp, char *nm, znode_t **zpp, int flags, cred_t *cr, int *direntflags, pathname_t *realpnp); extern int zfs_create(znode_t *dzp, char *name, vattr_t *vap, int excl, int mode, znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_tmpfile(struct inode *dip, vattr_t *vapzfs, int excl, int mode, struct inode **ipp, cred_t *cr, int flag, vsecattr_t *vsecp, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_remove(znode_t *dzp, char *name, cred_t *cr, int flags); extern int zfs_mkdir(znode_t *dzp, char *dirname, vattr_t *vap, - znode_t **zpp, cred_t *cr, int flags, vsecattr_t *vsecp, zuserns_t *mnt_ns); + znode_t **zpp, cred_t *cr, int flags, vsecattr_t *vsecp, zidmap_t *mnt_ns); extern int zfs_rmdir(znode_t *dzp, char *name, znode_t *cwd, cred_t *cr, int flags); extern int zfs_readdir(struct inode *ip, zpl_dir_context_t *ctx, cred_t *cr); -extern int zfs_getattr_fast(struct user_namespace *, struct inode *ip, - struct kstat *sp); +extern int zfs_getattr_fast(zidmap_t *, struct inode *ip, struct kstat *sp); extern int zfs_setattr(znode_t *zp, vattr_t *vap, int flag, cred_t *cr, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_rename(znode_t *sdzp, char *snm, znode_t *tdzp, char *tnm, cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, - zuserns_t *mnt_ns); + zidmap_t *mnt_ns); extern int zfs_symlink(znode_t *dzp, char *name, vattr_t *vap, - char *link, znode_t **zpp, cred_t *cr, int flags, zuserns_t *mnt_ns); + char *link, znode_t **zpp, cred_t *cr, int flags, zidmap_t *mnt_ns); extern int zfs_readlink(struct inode *ip, zfs_uio_t *uio, cred_t *cr); extern int zfs_link(znode_t *tdzp, znode_t *szp, char *name, cred_t *cr, int flags); diff --git a/include/os/linux/zfs/sys/zpl.h b/include/os/linux/zfs/sys/zpl.h index ac1f01a86c41..2b302e9dab07 100644 --- a/include/os/linux/zfs/sys/zpl.h +++ b/include/os/linux/zfs/sys/zpl.h @@ -39,7 +39,7 @@ /* zpl_inode.c */ extern void zpl_vap_init(vattr_t *vap, struct inode *dir, - umode_t mode, cred_t *cr, zuserns_t *mnt_ns); + umode_t mode, cred_t *cr, zidmap_t *mnt_ns); extern const struct inode_operations zpl_inode_operations; #ifdef HAVE_RENAME2_OPERATIONS_WRAPPER @@ -68,7 +68,10 @@ extern int zpl_xattr_security_init(struct inode *ip, struct inode *dip, const struct qstr *qstr); #if defined(CONFIG_FS_POSIX_ACL) #if defined(HAVE_SET_ACL) -#if defined(HAVE_SET_ACL_USERNS) +#if defined(HAVE_SET_ACL_IDMAP_DENTRY) +extern int zpl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +#elif defined(HAVE_SET_ACL_USERNS) extern int zpl_set_acl(struct user_namespace *userns, struct inode *ip, struct posix_acl *acl, int type); #elif defined(HAVE_SET_ACL_USERNS_DENTRY_ARG2) @@ -189,13 +192,15 @@ zpl_dir_emit_dots(struct file *file, zpl_dir_context_t *ctx) #if defined(HAVE_INODE_OWNER_OR_CAPABLE) #define zpl_inode_owner_or_capable(ns, ip) inode_owner_or_capable(ip) -#elif defined(HAVE_INODE_OWNER_OR_CAPABLE_IDMAPPED) +#elif defined(HAVE_INODE_OWNER_OR_CAPABLE_USERNS) #define zpl_inode_owner_or_capable(ns, ip) inode_owner_or_capable(ns, ip) +#elif defined(HAVE_INODE_OWNER_OR_CAPABLE_IDMAP) +#define zpl_inode_owner_or_capable(idmap, ip) inode_owner_or_capable(idmap, ip) #else #error "Unsupported kernel" #endif -#ifdef HAVE_SETATTR_PREPARE_USERNS +#if defined(HAVE_SETATTR_PREPARE_USERNS) || defined(HAVE_SETATTR_PREPARE_IDMAP) #define zpl_setattr_prepare(ns, dentry, ia) setattr_prepare(ns, dentry, ia) #else /* diff --git a/include/sys/zfs_acl.h b/include/sys/zfs_acl.h index e5c570c474a3..e19288528849 100644 --- a/include/sys/zfs_acl.h +++ b/include/sys/zfs_acl.h @@ -206,7 +206,7 @@ struct zfsvfs; #ifdef _KERNEL int zfs_acl_ids_create(struct znode *, int, vattr_t *, - cred_t *, vsecattr_t *, zfs_acl_ids_t *, zuserns_t *); + cred_t *, vsecattr_t *, zfs_acl_ids_t *, zidmap_t *); void zfs_acl_ids_free(zfs_acl_ids_t *); boolean_t zfs_acl_ids_overquota(struct zfsvfs *, zfs_acl_ids_t *, uint64_t); int zfs_getacl(struct znode *, vsecattr_t *, boolean_t, cred_t *); @@ -216,15 +216,15 @@ void zfs_oldace_byteswap(ace_t *, int); void zfs_ace_byteswap(void *, size_t, boolean_t); extern boolean_t zfs_has_access(struct znode *zp, cred_t *cr); extern int zfs_zaccess(struct znode *, int, int, boolean_t, cred_t *, - zuserns_t *); + zidmap_t *); int zfs_fastaccesschk_execute(struct znode *, cred_t *); -extern int zfs_zaccess_rwx(struct znode *, mode_t, int, cred_t *, zuserns_t *); +extern int zfs_zaccess_rwx(struct znode *, mode_t, int, cred_t *, zidmap_t *); extern int zfs_zaccess_unix(void *, int, cred_t *); extern int zfs_acl_access(struct znode *, int, cred_t *); int zfs_acl_chmod_setattr(struct znode *, zfs_acl_t **, uint64_t); -int zfs_zaccess_delete(struct znode *, struct znode *, cred_t *, zuserns_t *); +int zfs_zaccess_delete(struct znode *, struct znode *, cred_t *, zidmap_t *); int zfs_zaccess_rename(struct znode *, struct znode *, - struct znode *, struct znode *, cred_t *cr, zuserns_t *mnt_ns); + struct znode *, struct znode *, cred_t *cr, zidmap_t *mnt_ns); void zfs_acl_free(zfs_acl_t *); int zfs_vsec_2_aclp(struct zfsvfs *, umode_t, vsecattr_t *, cred_t *, struct zfs_fuid_info **, zfs_acl_t **); diff --git a/module/os/freebsd/zfs/zfs_acl.c b/module/os/freebsd/zfs/zfs_acl.c index 9f735dbb558c..a077076927a1 100644 --- a/module/os/freebsd/zfs/zfs_acl.c +++ b/module/os/freebsd/zfs/zfs_acl.c @@ -1619,7 +1619,7 @@ zfs_acl_inherit(zfsvfs_t *zfsvfs, vtype_t vtype, zfs_acl_t *paclp, */ int zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, - vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids, zuserns_t *mnt_ns) + vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids, zidmap_t *mnt_ns) { int error; zfsvfs_t *zfsvfs = dzp->z_zfsvfs; @@ -2341,7 +2341,7 @@ zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) */ int zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { uint32_t working_mode; int error; @@ -2471,7 +2471,7 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr, */ int zfs_zaccess_rwx(znode_t *zp, mode_t mode, int flags, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { return (zfs_zaccess(zp, zfs_unix_to_v4(mode >> 6), flags, B_FALSE, cr, mnt_ns)); @@ -2541,7 +2541,7 @@ zfs_delete_final_check(znode_t *zp, znode_t *dzp, * */ int -zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zuserns_t *mnt_ns) +zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zidmap_t *mnt_ns) { uint32_t dzp_working_mode = 0; uint32_t zp_working_mode = 0; @@ -2628,7 +2628,7 @@ zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zuserns_t *mnt_ns) int zfs_zaccess_rename(znode_t *sdzp, znode_t *szp, znode_t *tdzp, - znode_t *tzp, cred_t *cr, zuserns_t *mnt_ns) + znode_t *tzp, cred_t *cr, zidmap_t *mnt_ns) { int add_perm; int error; diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index b3405b7593f4..8abd7239ad2e 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -1053,7 +1053,7 @@ zfs_lookup(vnode_t *dvp, const char *nm, vnode_t **vpp, */ int zfs_create(znode_t *dzp, const char *name, vattr_t *vap, int excl, int mode, - znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp, zuserns_t *mnt_ns) + znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp, zidmap_t *mnt_ns) { (void) excl, (void) mode, (void) flag; znode_t *zp; @@ -1405,7 +1405,7 @@ zfs_remove(znode_t *dzp, const char *name, cred_t *cr, int flags) */ int zfs_mkdir(znode_t *dzp, const char *dirname, vattr_t *vap, znode_t **zpp, - cred_t *cr, int flags, vsecattr_t *vsecp, zuserns_t *mnt_ns) + cred_t *cr, int flags, vsecattr_t *vsecp, zidmap_t *mnt_ns) { (void) flags, (void) vsecp; znode_t *zp; @@ -2159,7 +2159,7 @@ zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr) * vp - ctime updated, mtime updated if size changed. */ int -zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zuserns_t *mnt_ns) +zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) { vnode_t *vp = ZTOV(zp); zfsvfs_t *zfsvfs = zp->z_zfsvfs; @@ -3420,7 +3420,7 @@ zfs_do_rename_impl(vnode_t *sdvp, vnode_t **svpp, struct componentname *scnp, int zfs_rename(znode_t *sdzp, const char *sname, znode_t *tdzp, const char *tname, - cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, zuserns_t *mnt_ns) + cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, zidmap_t *mnt_ns) { struct componentname scn, tcn; vnode_t *sdvp, *tdvp; @@ -3477,7 +3477,7 @@ zfs_rename(znode_t *sdzp, const char *sname, znode_t *tdzp, const char *tname, */ int zfs_symlink(znode_t *dzp, const char *name, vattr_t *vap, - const char *link, znode_t **zpp, cred_t *cr, int flags, zuserns_t *mnt_ns) + const char *link, znode_t **zpp, cred_t *cr, int flags, zidmap_t *mnt_ns) { (void) flags; znode_t *zp; diff --git a/module/os/linux/spl/spl-cred.c b/module/os/linux/spl/spl-cred.c index f81b9540a639..d407fc66b2de 100644 --- a/module/os/linux/spl/spl-cred.c +++ b/module/os/linux/spl/spl-cred.c @@ -145,6 +145,18 @@ crgetgid(const cred_t *cr) return (KGID_TO_SGID(cr->fsgid)); } +/* Return the initial user ns or nop_mnt_idmap */ +zidmap_t * +zfs_get_init_idmap(void) +{ +#ifdef HAVE_IOPS_CREATE_IDMAP + return ((zidmap_t *)&nop_mnt_idmap); +#else + return ((zidmap_t *)&init_user_ns); +#endif +} + +EXPORT_SYMBOL(zfs_get_init_idmap); EXPORT_SYMBOL(crhold); EXPORT_SYMBOL(crfree); EXPORT_SYMBOL(crgetuid); diff --git a/module/os/linux/zfs/policy.c b/module/os/linux/zfs/policy.c index eaf38df864d3..5d1b4383412a 100644 --- a/module/os/linux/zfs/policy.c +++ b/module/os/linux/zfs/policy.c @@ -124,7 +124,7 @@ secpolicy_vnode_any_access(const cred_t *cr, struct inode *ip, uid_t owner) if (crgetuid(cr) == owner) return (0); - if (zpl_inode_owner_or_capable(kcred->user_ns, ip)) + if (zpl_inode_owner_or_capable(zfs_init_idmap, ip)) return (0); #if defined(CONFIG_USER_NS) @@ -214,8 +214,8 @@ secpolicy_vnode_setid_retain(struct znode *zp __maybe_unused, const cred_t *cr, * Determine that subject can set the file setgid flag. */ int -secpolicy_vnode_setids_setgids(const cred_t *cr, gid_t gid, zuserns_t *mnt_ns, - zuserns_t *fs_ns) +secpolicy_vnode_setids_setgids(const cred_t *cr, gid_t gid, zidmap_t *mnt_ns, + struct user_namespace *fs_ns) { gid = zfs_gid_to_vfsgid(mnt_ns, fs_ns, gid); #if defined(CONFIG_USER_NS) @@ -286,8 +286,8 @@ secpolicy_setid_clear(vattr_t *vap, cred_t *cr) * Determine that subject can set the file setid flags. */ static int -secpolicy_vnode_setid_modify(const cred_t *cr, uid_t owner, zuserns_t *mnt_ns, - zuserns_t *fs_ns) +secpolicy_vnode_setid_modify(const cred_t *cr, uid_t owner, zidmap_t *mnt_ns, + struct user_namespace *fs_ns) { owner = zfs_uid_to_vfsuid(mnt_ns, fs_ns, owner); @@ -315,7 +315,8 @@ secpolicy_vnode_stky_modify(const cred_t *cr) int secpolicy_setid_setsticky_clear(struct inode *ip, vattr_t *vap, - const vattr_t *ovap, cred_t *cr, zuserns_t *mnt_ns, zuserns_t *fs_ns) + const vattr_t *ovap, cred_t *cr, zidmap_t *mnt_ns, + struct user_namespace *fs_ns) { int error; diff --git a/module/os/linux/zfs/zfs_acl.c b/module/os/linux/zfs/zfs_acl.c index db1bb9577197..df4ebc3870be 100644 --- a/module/os/linux/zfs/zfs_acl.c +++ b/module/os/linux/zfs/zfs_acl.c @@ -1802,7 +1802,7 @@ zfs_acl_inherit(zfsvfs_t *zfsvfs, umode_t va_mode, zfs_acl_t *paclp, */ int zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, - vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids, zuserns_t *mnt_ns) + vsecattr_t *vsecp, zfs_acl_ids_t *acl_ids, zidmap_t *mnt_ns) { int error; zfsvfs_t *zfsvfs = ZTOZSB(dzp); @@ -1981,7 +1981,7 @@ zfs_getacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr) return (SET_ERROR(ENOSYS)); if ((error = zfs_zaccess(zp, ACE_READ_ACL, 0, skipaclchk, cr, - kcred->user_ns))) + zfs_init_idmap))) return (error); mutex_enter(&zp->z_acl_lock); @@ -2141,7 +2141,7 @@ zfs_setacl(znode_t *zp, vsecattr_t *vsecp, boolean_t skipaclchk, cred_t *cr) return (SET_ERROR(EPERM)); if ((error = zfs_zaccess(zp, ACE_WRITE_ACL, 0, skipaclchk, cr, - kcred->user_ns))) + zfs_init_idmap))) return (error); error = zfs_vsec_2_aclp(zfsvfs, ZTOI(zp)->i_mode, vsecp, cr, &fuidp, @@ -2286,7 +2286,7 @@ zfs_zaccess_dataset_check(znode_t *zp, uint32_t v4_mode) */ static int zfs_zaccess_aces_check(znode_t *zp, uint32_t *working_mode, - boolean_t anyaccess, cred_t *cr, zuserns_t *mnt_ns) + boolean_t anyaccess, cred_t *cr, zidmap_t *mnt_ns) { zfsvfs_t *zfsvfs = ZTOZSB(zp); zfs_acl_t *aclp; @@ -2420,7 +2420,7 @@ zfs_has_access(znode_t *zp, cred_t *cr) uint32_t have = ACE_ALL_PERMS; if (zfs_zaccess_aces_check(zp, &have, B_TRUE, cr, - kcred->user_ns) != 0) { + zfs_init_idmap) != 0) { uid_t owner; owner = zfs_fuid_map_id(ZTOZSB(zp), @@ -2451,7 +2451,7 @@ zfs_has_access(znode_t *zp, cred_t *cr) */ static int zfs_zaccess_trivial(znode_t *zp, uint32_t *working_mode, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { int err, mask; int unmapped = 0; @@ -2464,11 +2464,10 @@ zfs_zaccess_trivial(znode_t *zp, uint32_t *working_mode, cred_t *cr, return (unmapped ? SET_ERROR(EPERM) : 0); } -#if defined(HAVE_IOPS_PERMISSION_USERNS) +#if (defined(HAVE_IOPS_PERMISSION_USERNS) || \ + defined(HAVE_IOPS_PERMISSION_IDMAP)) if (mnt_ns) err = generic_permission(mnt_ns, ZTOI(zp), mask); - else - err = generic_permission(cr->user_ns, ZTOI(zp), mask); #else err = generic_permission(ZTOI(zp), mask); #endif @@ -2483,7 +2482,7 @@ zfs_zaccess_trivial(znode_t *zp, uint32_t *working_mode, cred_t *cr, static int zfs_zaccess_common(znode_t *zp, uint32_t v4_mode, uint32_t *working_mode, - boolean_t *check_privs, boolean_t skipaclchk, cred_t *cr, zuserns_t *mnt_ns) + boolean_t *check_privs, boolean_t skipaclchk, cred_t *cr, zidmap_t *mnt_ns) { zfsvfs_t *zfsvfs = ZTOZSB(zp); int err; @@ -2540,7 +2539,7 @@ zfs_zaccess_common(znode_t *zp, uint32_t v4_mode, uint32_t *working_mode, static int zfs_zaccess_append(znode_t *zp, uint32_t *working_mode, boolean_t *check_privs, - cred_t *cr, zuserns_t *mnt_ns) + cred_t *cr, zidmap_t *mnt_ns) { if (*working_mode != ACE_WRITE_DATA) return (SET_ERROR(EACCES)); @@ -2612,7 +2611,7 @@ zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) if ((error = zfs_enter(ZTOZSB(zdp), FTAG)) != 0) return (error); error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr, - kcred->user_ns); + zfs_init_idmap); zfs_exit(ZTOZSB(zdp), FTAG); return (error); } @@ -2625,7 +2624,7 @@ zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) */ int zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { uint32_t working_mode; int error; @@ -2774,7 +2773,7 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr, */ int zfs_zaccess_rwx(znode_t *zp, mode_t mode, int flags, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { return (zfs_zaccess(zp, zfs_unix_to_v4(mode >> 6), flags, B_FALSE, cr, mnt_ns)); @@ -2788,7 +2787,7 @@ zfs_zaccess_unix(void *zp, int mode, cred_t *cr) { int v4_mode = zfs_unix_to_v4(mode >> 6); - return (zfs_zaccess(zp, v4_mode, 0, B_FALSE, cr, kcred->user_ns)); + return (zfs_zaccess(zp, v4_mode, 0, B_FALSE, cr, zfs_init_idmap)); } /* See zfs_zaccess_delete() */ @@ -2865,7 +2864,7 @@ static const boolean_t zfs_write_implies_delete_child = B_TRUE; * zfs_write_implies_delete_child */ int -zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zuserns_t *mnt_ns) +zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zidmap_t *mnt_ns) { uint32_t wanted_dirperms; uint32_t dzp_working_mode = 0; @@ -2996,7 +2995,7 @@ zfs_zaccess_delete(znode_t *dzp, znode_t *zp, cred_t *cr, zuserns_t *mnt_ns) int zfs_zaccess_rename(znode_t *sdzp, znode_t *szp, znode_t *tdzp, - znode_t *tzp, cred_t *cr, zuserns_t *mnt_ns) + znode_t *tzp, cred_t *cr, zidmap_t *mnt_ns) { int add_perm; int error; diff --git a/module/os/linux/zfs/zfs_dir.c b/module/os/linux/zfs/zfs_dir.c index 1fec4ea09317..1eeabe53d23c 100644 --- a/module/os/linux/zfs/zfs_dir.c +++ b/module/os/linux/zfs/zfs_dir.c @@ -1120,7 +1120,7 @@ zfs_make_xattrdir(znode_t *zp, vattr_t *vap, znode_t **xzpp, cred_t *cr) *xzpp = NULL; if ((error = zfs_acl_ids_create(zp, IS_XATTR, vap, cr, NULL, - &acl_ids, kcred->user_ns)) != 0) + &acl_ids, zfs_init_idmap)) != 0) return (error); if (zfs_acl_ids_overquota(zfsvfs, &acl_ids, zp->z_projid)) { zfs_acl_ids_free(&acl_ids); @@ -1269,7 +1269,7 @@ zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr) if ((uid = crgetuid(cr)) == downer || uid == fowner || zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr, - kcred->user_ns) == 0) + zfs_init_idmap) == 0) return (0); else return (secpolicy_vnode_remove(cr)); diff --git a/module/os/linux/zfs/zfs_ioctl_os.c b/module/os/linux/zfs/zfs_ioctl_os.c index f068f544f0ec..663474ea49ab 100644 --- a/module/os/linux/zfs/zfs_ioctl_os.c +++ b/module/os/linux/zfs/zfs_ioctl_os.c @@ -282,6 +282,8 @@ zfsdev_detach(void) #define ZFS_DEBUG_STR "" #endif +zidmap_t *zfs_init_idmap; + static int openzfs_init_os(void) { @@ -305,6 +307,8 @@ openzfs_init_os(void) printk(KERN_NOTICE "ZFS: Posix ACLs disabled by kernel\n"); #endif /* CONFIG_FS_POSIX_ACL */ + zfs_init_idmap = (zidmap_t *)zfs_get_init_idmap(); + return (0); } diff --git a/module/os/linux/zfs/zfs_vnops_os.c b/module/os/linux/zfs/zfs_vnops_os.c index b8af3e3b058d..234c4d5ef0e0 100644 --- a/module/os/linux/zfs/zfs_vnops_os.c +++ b/module/os/linux/zfs/zfs_vnops_os.c @@ -487,7 +487,7 @@ zfs_lookup(znode_t *zdp, char *nm, znode_t **zpp, int flags, cred_t *cr, */ if ((error = zfs_zaccess(*zpp, ACE_EXECUTE, 0, - B_TRUE, cr, kcred->user_ns))) { + B_TRUE, cr, zfs_init_idmap))) { zrele(*zpp); *zpp = NULL; } @@ -506,7 +506,7 @@ zfs_lookup(znode_t *zdp, char *nm, znode_t **zpp, int flags, cred_t *cr, */ if ((error = zfs_zaccess(zdp, ACE_EXECUTE, 0, B_FALSE, cr, - kcred->user_ns))) { + zfs_init_idmap))) { zfs_exit(zfsvfs, FTAG); return (error); } @@ -551,7 +551,7 @@ zfs_lookup(znode_t *zdp, char *nm, znode_t **zpp, int flags, cred_t *cr, int zfs_create(znode_t *dzp, char *name, vattr_t *vap, int excl, int mode, znode_t **zpp, cred_t *cr, int flag, vsecattr_t *vsecp, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { znode_t *zp; zfsvfs_t *zfsvfs = ZTOZSB(dzp); @@ -799,7 +799,7 @@ zfs_create(znode_t *dzp, char *name, vattr_t *vap, int excl, int zfs_tmpfile(struct inode *dip, vattr_t *vap, int excl, int mode, struct inode **ipp, cred_t *cr, int flag, vsecattr_t *vsecp, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { (void) excl, (void) mode, (void) flag; znode_t *zp = NULL, *dzp = ITOZ(dip); @@ -984,7 +984,7 @@ zfs_remove(znode_t *dzp, char *name, cred_t *cr, int flags) return (error); } - if ((error = zfs_zaccess_delete(dzp, zp, cr, kcred->user_ns))) { + if ((error = zfs_zaccess_delete(dzp, zp, cr, zfs_init_idmap))) { goto out; } @@ -1179,7 +1179,7 @@ zfs_remove(znode_t *dzp, char *name, cred_t *cr, int flags) */ int zfs_mkdir(znode_t *dzp, char *dirname, vattr_t *vap, znode_t **zpp, - cred_t *cr, int flags, vsecattr_t *vsecp, zuserns_t *mnt_ns) + cred_t *cr, int flags, vsecattr_t *vsecp, zidmap_t *mnt_ns) { znode_t *zp; zfsvfs_t *zfsvfs = ZTOZSB(dzp); @@ -1400,7 +1400,7 @@ zfs_rmdir(znode_t *dzp, char *name, znode_t *cwd, cred_t *cr, return (error); } - if ((error = zfs_zaccess_delete(dzp, zp, cr, kcred->user_ns))) { + if ((error = zfs_zaccess_delete(dzp, zp, cr, zfs_init_idmap))) { goto out; } @@ -1652,8 +1652,7 @@ zfs_readdir(struct inode *ip, zpl_dir_context_t *ctx, cred_t *cr) * RETURN: 0 (always succeeds) */ int -zfs_getattr_fast(struct user_namespace *user_ns, struct inode *ip, - struct kstat *sp) +zfs_getattr_fast(zidmap_t *user_ns, struct inode *ip, struct kstat *sp) { znode_t *zp = ITOZ(ip); zfsvfs_t *zfsvfs = ITOZSB(ip); @@ -1841,7 +1840,7 @@ zfs_setattr_dir(znode_t *dzp) * ip - ctime updated, mtime updated if size changed. */ int -zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zuserns_t *mnt_ns) +zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zidmap_t *mnt_ns) { struct inode *ip; zfsvfs_t *zfsvfs = ZTOZSB(zp); @@ -2038,10 +2037,10 @@ zfs_setattr(znode_t *zp, vattr_t *vap, int flags, cred_t *cr, zuserns_t *mnt_ns) * Take ownership or chgrp to group we are a member of */ - uid = zfs_uid_to_vfsuid((struct user_namespace *)mnt_ns, - zfs_i_user_ns(ip), vap->va_uid); - gid = zfs_gid_to_vfsgid((struct user_namespace *)mnt_ns, - zfs_i_user_ns(ip), vap->va_gid); + uid = zfs_uid_to_vfsuid(mnt_ns, zfs_i_user_ns(ip), + vap->va_uid); + gid = zfs_gid_to_vfsgid(mnt_ns, zfs_i_user_ns(ip), + vap->va_gid); take_owner = (mask & ATTR_UID) && (uid == crgetuid(cr)); take_group = (mask & ATTR_GID) && zfs_groupmember(zfsvfs, gid, cr); @@ -2680,7 +2679,7 @@ zfs_rename_lock(znode_t *szp, znode_t *tdzp, znode_t *sdzp, zfs_zlock_t **zlpp) */ int zfs_rename(znode_t *sdzp, char *snm, znode_t *tdzp, char *tnm, - cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, zuserns_t *mnt_ns) + cred_t *cr, int flags, uint64_t rflags, vattr_t *wo_vap, zidmap_t *mnt_ns) { znode_t *szp, *tzp; zfsvfs_t *zfsvfs = ZTOZSB(sdzp); @@ -3213,7 +3212,7 @@ zfs_rename(znode_t *sdzp, char *snm, znode_t *tdzp, char *tnm, */ int zfs_symlink(znode_t *dzp, char *name, vattr_t *vap, char *link, - znode_t **zpp, cred_t *cr, int flags, zuserns_t *mnt_ns) + znode_t **zpp, cred_t *cr, int flags, zidmap_t *mnt_ns) { znode_t *zp; zfs_dirlock_t *dl; @@ -3521,7 +3520,7 @@ zfs_link(znode_t *tdzp, znode_t *szp, char *name, cred_t *cr, } if ((error = zfs_zaccess(tdzp, ACE_ADD_FILE, 0, B_FALSE, cr, - kcred->user_ns))) { + zfs_init_idmap))) { zfs_exit(zfsvfs, FTAG); return (error); } @@ -4136,7 +4135,7 @@ zfs_space(znode_t *zp, int cmd, flock64_t *bfp, int flag, * operates directly on inodes, so we need to check access rights. */ if ((error = zfs_zaccess(zp, ACE_WRITE_DATA, 0, B_FALSE, cr, - kcred->user_ns))) { + zfs_init_idmap))) { zfs_exit(zfsvfs, FTAG); return (error); } diff --git a/module/os/linux/zfs/zfs_znode.c b/module/os/linux/zfs/zfs_znode.c index 38cdccfd8084..c104cd661bf5 100644 --- a/module/os/linux/zfs/zfs_znode.c +++ b/module/os/linux/zfs/zfs_znode.c @@ -1963,7 +1963,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) } VERIFY(0 == zfs_acl_ids_create(rootzp, IS_ROOT_NODE, &vattr, - cr, NULL, &acl_ids, kcred->user_ns)); + cr, NULL, &acl_ids, zfs_init_idmap)); zfs_mknode(rootzp, &vattr, tx, cr, IS_ROOT_NODE, &zp, &acl_ids); ASSERT3P(zp, ==, rootzp); error = zap_add(os, moid, ZFS_ROOT_OBJ, 8, 1, &rootzp->z_id, tx); diff --git a/module/os/linux/zfs/zpl_ctldir.c b/module/os/linux/zfs/zpl_ctldir.c index f0779c81dc75..68a7de78f471 100644 --- a/module/os/linux/zfs/zpl_ctldir.c +++ b/module/os/linux/zfs/zpl_ctldir.c @@ -103,7 +103,11 @@ zpl_root_readdir(struct file *filp, void *dirent, filldir_t filldir) * Get root directory attributes. */ static int -#ifdef HAVE_USERNS_IOPS_GETATTR +#ifdef HAVE_IDMAP_IOPS_GETATTR +zpl_root_getattr_impl(struct mnt_idmap *user_ns, + const struct path *path, struct kstat *stat, u32 request_mask, + unsigned int query_flags) +#elif defined(HAVE_USERNS_IOPS_GETATTR) zpl_root_getattr_impl(struct user_namespace *user_ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) @@ -115,9 +119,11 @@ zpl_root_getattr_impl(const struct path *path, struct kstat *stat, (void) request_mask, (void) query_flags; struct inode *ip = path->dentry->d_inode; -#ifdef HAVE_USERNS_IOPS_GETATTR +#if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR)) #ifdef HAVE_GENERIC_FILLATTR_USERNS generic_fillattr(user_ns, ip, stat); +#elif defined(HAVE_GENERIC_FILLATTR_IDMAP) + generic_fillattr(user_ns, ip, stat); #else (void) user_ns; #endif @@ -312,6 +318,10 @@ static int zpl_snapdir_rename2(struct user_namespace *user_ns, struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, unsigned int flags) +#elif defined(HAVE_IOPS_RENAME_IDMAP) +zpl_snapdir_rename2(struct mnt_idmap *user_ns, struct inode *sdip, + struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, + unsigned int flags) #else zpl_snapdir_rename2(struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, unsigned int flags) @@ -333,7 +343,9 @@ zpl_snapdir_rename2(struct inode *sdip, struct dentry *sdentry, return (error); } -#if !defined(HAVE_RENAME_WANTS_FLAGS) && !defined(HAVE_IOPS_RENAME_USERNS) +#if (!defined(HAVE_RENAME_WANTS_FLAGS) && \ + !defined(HAVE_IOPS_RENAME_USERNS) && \ + !defined(HAVE_IOPS_RENAME_IDMAP)) static int zpl_snapdir_rename(struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry) @@ -360,6 +372,9 @@ static int #ifdef HAVE_IOPS_MKDIR_USERNS zpl_snapdir_mkdir(struct user_namespace *user_ns, struct inode *dip, struct dentry *dentry, umode_t mode) +#elif defined(HAVE_IOPS_MKDIR_IDMAP) +zpl_snapdir_mkdir(struct mnt_idmap *user_ns, struct inode *dip, + struct dentry *dentry, umode_t mode) #else zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode) #endif @@ -371,10 +386,10 @@ zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode) crhold(cr); vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP); -#ifdef HAVE_IOPS_MKDIR_USERNS +#if (defined(HAVE_IOPS_MKDIR_USERNS) || defined(HAVE_IOPS_MKDIR_IDMAP)) zpl_vap_init(vap, dip, mode | S_IFDIR, cr, user_ns); #else - zpl_vap_init(vap, dip, mode | S_IFDIR, cr, kcred->user_ns); + zpl_vap_init(vap, dip, mode | S_IFDIR, cr, zfs_init_idmap); #endif error = -zfsctl_snapdir_mkdir(dip, dname(dentry), vap, &ip, cr, 0); @@ -395,7 +410,11 @@ zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode) * Get snapshot directory attributes. */ static int -#ifdef HAVE_USERNS_IOPS_GETATTR +#ifdef HAVE_IDMAP_IOPS_GETATTR +zpl_snapdir_getattr_impl(struct mnt_idmap *user_ns, + const struct path *path, struct kstat *stat, u32 request_mask, + unsigned int query_flags) +#elif defined(HAVE_USERNS_IOPS_GETATTR) zpl_snapdir_getattr_impl(struct user_namespace *user_ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) @@ -411,9 +430,11 @@ zpl_snapdir_getattr_impl(const struct path *path, struct kstat *stat, if ((error = zpl_enter(zfsvfs, FTAG)) != 0) return (error); -#ifdef HAVE_USERNS_IOPS_GETATTR +#if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR)) #ifdef HAVE_GENERIC_FILLATTR_USERNS generic_fillattr(user_ns, ip, stat); +#elif defined(HAVE_GENERIC_FILLATTR_IDMAP) + generic_fillattr(user_ns, ip, stat); #else (void) user_ns; #endif @@ -471,7 +492,9 @@ const struct file_operations zpl_fops_snapdir = { const struct inode_operations zpl_ops_snapdir = { .lookup = zpl_snapdir_lookup, .getattr = zpl_snapdir_getattr, -#if defined(HAVE_RENAME_WANTS_FLAGS) || defined(HAVE_IOPS_RENAME_USERNS) +#if (defined(HAVE_RENAME_WANTS_FLAGS) || \ + defined(HAVE_IOPS_RENAME_USERNS) || \ + defined(HAVE_IOPS_RENAME_IDMAP)) .rename = zpl_snapdir_rename2, #else .rename = zpl_snapdir_rename, @@ -562,6 +585,10 @@ static int zpl_shares_getattr_impl(struct user_namespace *user_ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) +#elif defined(HAVE_IDMAP_IOPS_GETATTR) +zpl_shares_getattr_impl(struct mnt_idmap *user_ns, + const struct path *path, struct kstat *stat, u32 request_mask, + unsigned int query_flags) #else zpl_shares_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) @@ -577,9 +604,11 @@ zpl_shares_getattr_impl(const struct path *path, struct kstat *stat, return (error); if (zfsvfs->z_shares_dir == 0) { -#ifdef HAVE_USERNS_IOPS_GETATTR +#if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR)) #ifdef HAVE_GENERIC_FILLATTR_USERNS generic_fillattr(user_ns, path->dentry->d_inode, stat); +#elif defined(HAVE_GENERIC_FILLATTR_IDMAP) + generic_fillattr(user_ns, path->dentry->d_inode, stat); #else (void) user_ns; #endif @@ -594,12 +623,8 @@ zpl_shares_getattr_impl(const struct path *path, struct kstat *stat, error = -zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp); if (error == 0) { -#ifdef HAVE_USERNS_IOPS_GETATTR -#ifdef HAVE_GENERIC_FILLATTR_USERNS +#if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR)) error = -zfs_getattr_fast(user_ns, ZTOI(dzp), stat); -#else - (void) user_ns; -#endif #else error = -zfs_getattr_fast(kcred->user_ns, ZTOI(dzp), stat); #endif diff --git a/module/os/linux/zfs/zpl_file.c b/module/os/linux/zfs/zpl_file.c index ce22e9a9e0e4..e690525d3cd4 100644 --- a/module/os/linux/zfs/zpl_file.c +++ b/module/os/linux/zfs/zpl_file.c @@ -1049,7 +1049,7 @@ __zpl_ioctl_setflags(struct inode *ip, uint32_t ioctl_flags, xvattr_t *xva) !capable(CAP_LINUX_IMMUTABLE)) return (-EPERM); - if (!zpl_inode_owner_or_capable(kcred->user_ns, ip)) + if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip)) return (-EACCES); xva_init(xva); @@ -1096,7 +1096,7 @@ zpl_ioctl_setflags(struct file *filp, void __user *arg) crhold(cr); cookie = spl_fstrans_mark(); - err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, kcred->user_ns); + err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap); spl_fstrans_unmark(cookie); crfree(cr); @@ -1144,7 +1144,7 @@ zpl_ioctl_setxattr(struct file *filp, void __user *arg) crhold(cr); cookie = spl_fstrans_mark(); - err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, kcred->user_ns); + err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap); spl_fstrans_unmark(cookie); crfree(cr); @@ -1179,7 +1179,7 @@ __zpl_ioctl_setdosflags(struct inode *ip, uint64_t ioctl_flags, xvattr_t *xva) !capable(CAP_LINUX_IMMUTABLE)) return (-EPERM); - if (!zpl_inode_owner_or_capable(kcred->user_ns, ip)) + if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip)) return (-EACCES); xva_init(xva); @@ -1232,7 +1232,7 @@ zpl_ioctl_setdosflags(struct file *filp, void __user *arg) crhold(cr); cookie = spl_fstrans_mark(); - err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, kcred->user_ns); + err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap); spl_fstrans_unmark(cookie); crfree(cr); diff --git a/module/os/linux/zfs/zpl_inode.c b/module/os/linux/zfs/zpl_inode.c index 993447e54683..5f5ad186a61c 100644 --- a/module/os/linux/zfs/zpl_inode.c +++ b/module/os/linux/zfs/zpl_inode.c @@ -113,12 +113,12 @@ zpl_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) void zpl_vap_init(vattr_t *vap, struct inode *dir, umode_t mode, cred_t *cr, - zuserns_t *mnt_ns) + zidmap_t *mnt_ns) { vap->va_mask = ATTR_MODE; vap->va_mode = mode; - vap->va_uid = zfs_vfsuid_to_uid((struct user_namespace *)mnt_ns, + vap->va_uid = zfs_vfsuid_to_uid(mnt_ns, zfs_i_user_ns(dir), crgetuid(cr)); if (dir->i_mode & S_ISGID) { @@ -126,7 +126,7 @@ zpl_vap_init(vattr_t *vap, struct inode *dir, umode_t mode, cred_t *cr, if (S_ISDIR(mode)) vap->va_mode |= S_ISGID; } else { - vap->va_gid = zfs_vfsgid_to_gid((struct user_namespace *)mnt_ns, + vap->va_gid = zfs_vfsgid_to_gid(mnt_ns, zfs_i_user_ns(dir), crgetgid(cr)); } } @@ -135,6 +135,9 @@ static int #ifdef HAVE_IOPS_CREATE_USERNS zpl_create(struct user_namespace *user_ns, struct inode *dir, struct dentry *dentry, umode_t mode, bool flag) +#elif defined(HAVE_IOPS_CREATE_IDMAP) +zpl_create(struct mnt_idmap *user_ns, struct inode *dir, + struct dentry *dentry, umode_t mode, bool flag) #else zpl_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool flag) #endif @@ -144,8 +147,8 @@ zpl_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool flag) vattr_t *vap; int error; fstrans_cookie_t cookie; -#ifndef HAVE_IOPS_CREATE_USERNS - zuserns_t *user_ns = kcred->user_ns; +#if !(defined(HAVE_IOPS_CREATE_USERNS) || defined(HAVE_IOPS_CREATE_IDMAP)) + zidmap_t *user_ns = kcred->user_ns; #endif crhold(cr); @@ -181,6 +184,9 @@ static int #ifdef HAVE_IOPS_MKNOD_USERNS zpl_mknod(struct user_namespace *user_ns, struct inode *dir, struct dentry *dentry, umode_t mode, +#elif defined(HAVE_IOPS_MKNOD_IDMAP) +zpl_mknod(struct mnt_idmap *user_ns, struct inode *dir, + struct dentry *dentry, umode_t mode, #else zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, #endif @@ -191,8 +197,8 @@ zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, vattr_t *vap; int error; fstrans_cookie_t cookie; -#ifndef HAVE_IOPS_MKNOD_USERNS - zuserns_t *user_ns = kcred->user_ns; +#if !(defined(HAVE_IOPS_MKNOD_USERNS) || defined(HAVE_IOPS_MKNOD_IDMAP)) + zidmap_t *user_ns = kcred->user_ns; #endif /* @@ -234,7 +240,10 @@ zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, #ifdef HAVE_TMPFILE static int -#ifndef HAVE_TMPFILE_DENTRY +#ifdef HAVE_TMPFILE_IDMAP +zpl_tmpfile(struct mnt_idmap *userns, struct inode *dir, + struct file *file, umode_t mode) +#elif !defined(HAVE_TMPFILE_DENTRY) zpl_tmpfile(struct user_namespace *userns, struct inode *dir, struct file *file, umode_t mode) #else @@ -251,8 +260,8 @@ zpl_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) vattr_t *vap; int error; fstrans_cookie_t cookie; -#ifndef HAVE_TMPFILE_USERNS - zuserns_t *userns = kcred->user_ns; +#if !(defined(HAVE_TMPFILE_USERNS) || defined(HAVE_TMPFILE_IDMAP)) + zidmap_t *userns = kcred->user_ns; #endif crhold(cr); @@ -330,6 +339,9 @@ static int #ifdef HAVE_IOPS_MKDIR_USERNS zpl_mkdir(struct user_namespace *user_ns, struct inode *dir, struct dentry *dentry, umode_t mode) +#elif defined(HAVE_IOPS_MKDIR_IDMAP) +zpl_mkdir(struct mnt_idmap *user_ns, struct inode *dir, + struct dentry *dentry, umode_t mode) #else zpl_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) #endif @@ -339,8 +351,8 @@ zpl_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) znode_t *zp; int error; fstrans_cookie_t cookie; -#ifndef HAVE_IOPS_MKDIR_USERNS - zuserns_t *user_ns = kcred->user_ns; +#if !(defined(HAVE_IOPS_MKDIR_USERNS) || defined(HAVE_IOPS_MKDIR_IDMAP)) + zidmap_t *user_ns = kcred->user_ns; #endif crhold(cr); @@ -403,6 +415,10 @@ static int zpl_getattr_impl(struct user_namespace *user_ns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) +#elif defined(HAVE_IDMAP_IOPS_GETATTR) +zpl_getattr_impl(struct mnt_idmap *user_ns, + const struct path *path, struct kstat *stat, u32 request_mask, + unsigned int query_flags) #else zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) @@ -419,7 +435,7 @@ zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask, * XXX query_flags currently ignored. */ -#ifdef HAVE_USERNS_IOPS_GETATTR +#if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR)) error = -zfs_getattr_fast(user_ns, ip, stat); #else error = -zfs_getattr_fast(kcred->user_ns, ip, stat); @@ -458,9 +474,12 @@ zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask, ZPL_GETATTR_WRAPPER(zpl_getattr); static int -#ifdef HAVE_SETATTR_PREPARE_USERNS +#ifdef HAVE_USERNS_IOPS_SETATTR zpl_setattr(struct user_namespace *user_ns, struct dentry *dentry, struct iattr *ia) +#elif defined(HAVE_IDMAP_IOPS_SETATTR) +zpl_setattr(struct mnt_idmap *user_ns, struct dentry *dentry, + struct iattr *ia) #else zpl_setattr(struct dentry *dentry, struct iattr *ia) #endif @@ -473,8 +492,10 @@ zpl_setattr(struct dentry *dentry, struct iattr *ia) #ifdef HAVE_SETATTR_PREPARE_USERNS error = zpl_setattr_prepare(user_ns, dentry, ia); +#elif defined(HAVE_SETATTR_PREPARE_IDMAP) + error = zpl_setattr_prepare(user_ns, dentry, ia); #else - error = zpl_setattr_prepare(kcred->user_ns, dentry, ia); + error = zpl_setattr_prepare(zfs_init_idmap, dentry, ia); #endif if (error) return (error); @@ -506,10 +527,12 @@ zpl_setattr(struct dentry *dentry, struct iattr *ia) ip->i_atime = zpl_inode_timestamp_truncate(ia->ia_atime, ip); cookie = spl_fstrans_mark(); -#ifdef HAVE_SETATTR_PREPARE_USERNS +#ifdef HAVE_USERNS_IOPS_SETATTR + error = -zfs_setattr(ITOZ(ip), vap, 0, cr, user_ns); +#elif defined(HAVE_IDMAP_IOPS_SETATTR) error = -zfs_setattr(ITOZ(ip), vap, 0, cr, user_ns); #else - error = -zfs_setattr(ITOZ(ip), vap, 0, cr, kcred->user_ns); + error = -zfs_setattr(ITOZ(ip), vap, 0, cr, zfs_init_idmap); #endif if (!error && (ia->ia_valid & ATTR_MODE)) error = zpl_chmod_acl(ip); @@ -527,6 +550,10 @@ static int zpl_rename2(struct user_namespace *user_ns, struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, unsigned int rflags) +#elif defined(HAVE_IOPS_RENAME_IDMAP) +zpl_rename2(struct mnt_idmap *user_ns, struct inode *sdip, + struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, + unsigned int rflags) #else zpl_rename2(struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry, unsigned int rflags) @@ -536,8 +563,8 @@ zpl_rename2(struct inode *sdip, struct dentry *sdentry, vattr_t *wo_vap = NULL; int error; fstrans_cookie_t cookie; -#ifndef HAVE_IOPS_RENAME_USERNS - zuserns_t *user_ns = kcred->user_ns; +#if !(defined(HAVE_IOPS_RENAME_USERNS) || defined(HAVE_IOPS_RENAME_IDMAP)) + zidmap_t *user_ns = kcred->user_ns; #endif crhold(cr); @@ -561,7 +588,8 @@ zpl_rename2(struct inode *sdip, struct dentry *sdentry, #if !defined(HAVE_IOPS_RENAME_USERNS) && \ !defined(HAVE_RENAME_WANTS_FLAGS) && \ - !defined(HAVE_RENAME2) + !defined(HAVE_RENAME2) && \ + !defined(HAVE_IOPS_RENAME_IDMAP) static int zpl_rename(struct inode *sdip, struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry) @@ -574,6 +602,9 @@ static int #ifdef HAVE_IOPS_SYMLINK_USERNS zpl_symlink(struct user_namespace *user_ns, struct inode *dir, struct dentry *dentry, const char *name) +#elif defined(HAVE_IOPS_SYMLINK_IDMAP) +zpl_symlink(struct mnt_idmap *user_ns, struct inode *dir, + struct dentry *dentry, const char *name) #else zpl_symlink(struct inode *dir, struct dentry *dentry, const char *name) #endif @@ -583,8 +614,8 @@ zpl_symlink(struct inode *dir, struct dentry *dentry, const char *name) znode_t *zp; int error; fstrans_cookie_t cookie; -#ifndef HAVE_IOPS_SYMLINK_USERNS - zuserns_t *user_ns = kcred->user_ns; +#if !(defined(HAVE_IOPS_SYMLINK_USERNS) || defined(HAVE_IOPS_SYMLINK_IDMAP)) + zidmap_t *user_ns = kcred->user_ns; #endif crhold(cr); @@ -802,6 +833,8 @@ const struct inode_operations zpl_dir_inode_operations = { .rename2 = zpl_rename2, #elif defined(HAVE_RENAME_WANTS_FLAGS) || defined(HAVE_IOPS_RENAME_USERNS) .rename = zpl_rename2, +#elif defined(HAVE_IOPS_RENAME_IDMAP) + .rename = zpl_rename2, #else .rename = zpl_rename, #endif diff --git a/module/os/linux/zfs/zpl_xattr.c b/module/os/linux/zfs/zpl_xattr.c index 4156d686732a..96d85991811e 100644 --- a/module/os/linux/zfs/zpl_xattr.c +++ b/module/os/linux/zfs/zpl_xattr.c @@ -499,7 +499,7 @@ zpl_xattr_set_dir(struct inode *ip, const char *name, const void *value, vap->va_gid = crgetgid(cr); error = -zfs_create(dxzp, (char *)name, vap, 0, 0644, &xzp, - cr, ATTR_NOACLCHECK, NULL, kcred->user_ns); + cr, ATTR_NOACLCHECK, NULL, zfs_init_idmap); if (error) goto out; } @@ -738,7 +738,7 @@ __zpl_xattr_user_get(struct inode *ip, const char *name, ZPL_XATTR_GET_WRAPPER(zpl_xattr_user_get); static int -__zpl_xattr_user_set(struct user_namespace *user_ns, +__zpl_xattr_user_set(zidmap_t *user_ns, struct inode *ip, const char *name, const void *value, size_t size, int flags) { @@ -848,7 +848,7 @@ __zpl_xattr_trusted_get(struct inode *ip, const char *name, ZPL_XATTR_GET_WRAPPER(zpl_xattr_trusted_get); static int -__zpl_xattr_trusted_set(struct user_namespace *user_ns, +__zpl_xattr_trusted_set(zidmap_t *user_ns, struct inode *ip, const char *name, const void *value, size_t size, int flags) { @@ -918,7 +918,7 @@ __zpl_xattr_security_get(struct inode *ip, const char *name, ZPL_XATTR_GET_WRAPPER(zpl_xattr_security_get); static int -__zpl_xattr_security_set(struct user_namespace *user_ns, +__zpl_xattr_security_set(zidmap_t *user_ns, struct inode *ip, const char *name, const void *value, size_t size, int flags) { @@ -1061,6 +1061,9 @@ int #ifdef HAVE_SET_ACL_USERNS zpl_set_acl(struct user_namespace *userns, struct inode *ip, struct posix_acl *acl, int type) +#elif defined(HAVE_SET_ACL_IDMAP_DENTRY) +zpl_set_acl(struct mnt_idmap *userns, struct dentry *dentry, + struct posix_acl *acl, int type) #elif defined(HAVE_SET_ACL_USERNS_DENTRY_ARG2) zpl_set_acl(struct user_namespace *userns, struct dentry *dentry, struct posix_acl *acl, int type) @@ -1070,6 +1073,8 @@ zpl_set_acl(struct inode *ip, struct posix_acl *acl, int type) { #ifdef HAVE_SET_ACL_USERNS_DENTRY_ARG2 return (zpl_set_acl_impl(d_inode(dentry), acl, type)); +#elif defined(HAVE_SET_ACL_IDMAP_DENTRY) + return (zpl_set_acl_impl(d_inode(dentry), acl, type)); #else return (zpl_set_acl_impl(ip, acl, type)); #endif /* HAVE_SET_ACL_USERNS_DENTRY_ARG2 */ @@ -1313,7 +1318,7 @@ __zpl_xattr_acl_get_default(struct inode *ip, const char *name, ZPL_XATTR_GET_WRAPPER(zpl_xattr_acl_get_default); static int -__zpl_xattr_acl_set_access(struct user_namespace *mnt_ns, +__zpl_xattr_acl_set_access(zidmap_t *mnt_ns, struct inode *ip, const char *name, const void *value, size_t size, int flags) { @@ -1328,12 +1333,12 @@ __zpl_xattr_acl_set_access(struct user_namespace *mnt_ns, if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIX) return (-EOPNOTSUPP); -#if defined(HAVE_XATTR_SET_USERNS) +#if defined(HAVE_XATTR_SET_USERNS) || defined(HAVE_XATTR_SET_IDMAP) if (!zpl_inode_owner_or_capable(mnt_ns, ip)) return (-EPERM); #else (void) mnt_ns; - if (!zpl_inode_owner_or_capable(kcred->user_ns, ip)) + if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip)) return (-EPERM); #endif @@ -1359,7 +1364,7 @@ __zpl_xattr_acl_set_access(struct user_namespace *mnt_ns, ZPL_XATTR_SET_WRAPPER(zpl_xattr_acl_set_access); static int -__zpl_xattr_acl_set_default(struct user_namespace *mnt_ns, +__zpl_xattr_acl_set_default(zidmap_t *mnt_ns, struct inode *ip, const char *name, const void *value, size_t size, int flags) { @@ -1374,12 +1379,12 @@ __zpl_xattr_acl_set_default(struct user_namespace *mnt_ns, if (ITOZSB(ip)->z_acl_type != ZFS_ACLTYPE_POSIX) return (-EOPNOTSUPP); -#if defined(HAVE_XATTR_SET_USERNS) +#if defined(HAVE_XATTR_SET_USERNS) || defined(HAVE_XATTR_SET_IDMAP) if (!zpl_inode_owner_or_capable(mnt_ns, ip)) return (-EPERM); #else (void) mnt_ns; - if (!zpl_inode_owner_or_capable(kcred->user_ns, ip)) + if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip)) return (-EPERM); #endif diff --git a/module/zfs/zfs_replay.c b/module/zfs/zfs_replay.c index 04dfda56b3f1..09c7be853bf9 100644 --- a/module/zfs/zfs_replay.c +++ b/module/zfs/zfs_replay.c @@ -389,7 +389,7 @@ zfs_replay_create_acl(void *arg1, void *arg2, boolean_t byteswap) #if defined(__linux__) error = zfs_create(dzp, name, &xva.xva_vattr, - 0, 0, &zp, kcred, vflg, &vsec, kcred->user_ns); + 0, 0, &zp, kcred, vflg, &vsec, zfs_init_idmap); #else error = zfs_create(dzp, name, &xva.xva_vattr, 0, 0, &zp, kcred, vflg, &vsec, NULL); @@ -424,7 +424,7 @@ zfs_replay_create_acl(void *arg1, void *arg2, boolean_t byteswap) } #if defined(__linux__) error = zfs_mkdir(dzp, name, &xva.xva_vattr, - &zp, kcred, vflg, &vsec, kcred->user_ns); + &zp, kcred, vflg, &vsec, zfs_init_idmap); #else error = zfs_mkdir(dzp, name, &xva.xva_vattr, &zp, kcred, vflg, &vsec, NULL); @@ -540,7 +540,7 @@ zfs_replay_create(void *arg1, void *arg2, boolean_t byteswap) #if defined(__linux__) error = zfs_create(dzp, name, &xva.xva_vattr, - 0, 0, &zp, kcred, vflg, NULL, kcred->user_ns); + 0, 0, &zp, kcred, vflg, NULL, zfs_init_idmap); #else error = zfs_create(dzp, name, &xva.xva_vattr, 0, 0, &zp, kcred, vflg, NULL, NULL); @@ -563,7 +563,7 @@ zfs_replay_create(void *arg1, void *arg2, boolean_t byteswap) #if defined(__linux__) error = zfs_mkdir(dzp, name, &xva.xva_vattr, - &zp, kcred, vflg, NULL, kcred->user_ns); + &zp, kcred, vflg, NULL, zfs_init_idmap); #else error = zfs_mkdir(dzp, name, &xva.xva_vattr, &zp, kcred, vflg, NULL, NULL); @@ -578,7 +578,7 @@ zfs_replay_create(void *arg1, void *arg2, boolean_t byteswap) link = name + strlen(name) + 1; #if defined(__linux__) error = zfs_symlink(dzp, name, &xva.xva_vattr, - link, &zp, kcred, vflg, kcred->user_ns); + link, &zp, kcred, vflg, zfs_init_idmap); #else error = zfs_symlink(dzp, name, &xva.xva_vattr, link, &zp, kcred, vflg, NULL); @@ -699,7 +699,7 @@ do_zfs_replay_rename(zfsvfs_t *zfsvfs, lr_rename_t *lr, char *sname, #if defined(__linux__) error = zfs_rename(sdzp, sname, tdzp, tname, kcred, vflg, rflags, - wo_vap, kcred->user_ns); + wo_vap, zfs_init_idmap); #else error = zfs_rename(sdzp, sname, tdzp, tname, kcred, vflg, rflags, wo_vap, NULL); @@ -977,7 +977,7 @@ zfs_replay_setattr(void *arg1, void *arg2, boolean_t byteswap) lr->lr_uid, lr->lr_gid); #if defined(__linux__) - error = zfs_setattr(zp, vap, 0, kcred, kcred->user_ns); + error = zfs_setattr(zp, vap, 0, kcred, zfs_init_idmap); #else error = zfs_setattr(zp, vap, 0, kcred, NULL); #endif diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c index 91b594e41cda..a6a27222bf4c 100644 --- a/module/zfs/zfs_vnops.c +++ b/module/zfs/zfs_vnops.c @@ -172,14 +172,14 @@ zfs_access(znode_t *zp, int mode, int flag, cred_t *cr) if (flag & V_ACE_MASK) #if defined(__linux__) error = zfs_zaccess(zp, mode, flag, B_FALSE, cr, - kcred->user_ns); + zfs_init_idmap); #else error = zfs_zaccess(zp, mode, flag, B_FALSE, cr, NULL); #endif else #if defined(__linux__) - error = zfs_zaccess_rwx(zp, mode, flag, cr, kcred->user_ns); + error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap); #else error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL); #endif From 678a3b8f999b9571691187f5224164c8bc58ec34 Mon Sep 17 00:00:00 2001 From: Alan Somers Date: Mon, 10 Apr 2023 15:24:27 -0600 Subject: [PATCH 19/59] Trim needless zeroes from checksum events The ereport.fs.zfs.checksum event contains histograms of the bits that were wrongly set or cleared according to their bit position in a 64-bit word. So the maximum value that any histogram bucket could have would be 64. But ZFS currently uses a uint32_t to hold each bucket. As a result, the event report is full of needless zeroes. Change the bucket size to uint8_t, stripping 768 needless zeros from each event. Original event format: ``` class=ereport.fs.zfs.checksum ena=639460469834258433 pool=testpool.1933 pool_guid=4979719877084416563 pool_state=0 pool_context=0 pool_failmode=wait vdev_guid=4136721804819128578 vdev_type=file vdev_path=/tmp/kyua.1TxP3A/2/work/file1.1933 vdev_ashift=9 vdev_complete_ts=609837019678 vdev_delta_ts=33450 vdev_read_errors=0 vdev_write_errors=0 vdev_cksum_errors=20 vdev_delays=0 parent_guid=2751977006639883417 parent_type=raidz vdev_spare_guids= zio_err=0 zio_flags=1048752 zio_stage=4194304 zio_pipeline=65011712 zio_delay=0 zio_timestamp=0 zio_delta=0 zio_priority=4 zio_offset=702976 zio_size=1024 zio_objset=24 zio_object=0 zio_level=3 zio_blkid=0 bad_ranges=0000000000000400 bad_ranges_min_gap=8 bad_range_sets=0000079e bad_range_clears=00000854 bad_set_histogram=000000210000001a000000150000001d000000240000001b000000220000001b000000210000002100000018000000260000002300000025000000210000001e000000250000001b0000001d0000001e0000001600000025000000180000001b000000240000001b000000240000001b0000001c000000210000001b0000001e000000210000001a0000001e000000220000001d0000001b000000200000001f0000001a000000250000001f0000001d0000001b0000001d000000240000001d0000001b0000001b0000001f00000024000000190000001a0000001f0000001e000000240000001e0000002400000021000000200000001d0000001d00000021 bad_cleared_histogram=000000220000002700000021000000210000001b0000001a000000250000001f0000001c0000001e0000002400000022000000220000002400000022000000240000002200000021000000220000001b0000002100000021000000190000001b000000240000002400000020000000290000002a00000028000000250000002400000020000000270000002500000016000000270000001c000000210000001f000000240000001c0000002100000022000000240000002100000023000000210000002700000022000000240000001b00000022000000210000001c00000023000000150000002600000020000000270000001e0000001d0000002400000026 time=00000016806457270000000323406839 eid=458 ``` New format: ``` class=ereport.fs.zfs.checksum ena=96599319807790081 pool=testpool.1933 pool_guid=1236902063710799041 pool_state=0 pool_context=0 pool_failmode=wait vdev_guid=2774253874431514999 vdev_type=file vdev_path=/tmp/kyua.6Temlq/2/work/file1.1933 vdev_ashift=9 vdev_complete_ts=92124283803 vdev_delta_ts=46670 vdev_read_errors=0 vdev_write_errors=0 vdev_cksum_errors=20 vdev_delays=0 parent_guid=8090931855087882905 parent_type=raidz vdev_spare_guids= zio_err=0 zio_flags=1048752 zio_stage=4194304 zio_pipeline=65011712 zio_delay=0 zio_timestamp=0 zio_delta=0 zio_priority=4 zio_offset=1028608 zio_size=512 zio_objset=0 zio_object=0 zio_level=0 zio_blkid=4 bad_ranges=0000000000000200 bad_ranges_min_gap=8 bad_range_sets=0000061f bad_range_clears=000001f4 bad_set_histogram=1719161c1c1c101618171a151a1a19161e1c171d1816161c191f1a18192117191c131d171b1613151a171419161a1b1319101b14171b18151e191a1b141a1c17 bad_cleared_histogram=06090a0808070a0b020609060506090a01090a050a0a0509070609080d050d0607080d060507080c04070807070a0608020c080c080908040808090a05090a07 time=00000016806477050000000604157480 eid=62 ``` Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Alan Somers Sponsored-by: Axcient Closes #14716 --- module/zfs/zfs_fm.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/module/zfs/zfs_fm.c b/module/zfs/zfs_fm.c index 7169e49ac46a..bdd0e96c327a 100644 --- a/module/zfs/zfs_fm.c +++ b/module/zfs/zfs_fm.c @@ -755,8 +755,8 @@ zfs_ereport_start(nvlist_t **ereport_out, nvlist_t **detector_out, typedef struct zfs_ecksum_info { /* histograms of set and cleared bits by bit number in a 64-bit word */ - uint32_t zei_histogram_set[sizeof (uint64_t) * NBBY]; - uint32_t zei_histogram_cleared[sizeof (uint64_t) * NBBY]; + uint8_t zei_histogram_set[sizeof (uint64_t) * NBBY]; + uint8_t zei_histogram_cleared[sizeof (uint64_t) * NBBY]; /* inline arrays of bits set and cleared. */ uint64_t zei_bits_set[ZFM_MAX_INLINE]; @@ -781,7 +781,7 @@ typedef struct zfs_ecksum_info { } zfs_ecksum_info_t; static void -update_histogram(uint64_t value_arg, uint32_t *hist, uint32_t *count) +update_histogram(uint64_t value_arg, uint8_t *hist, uint32_t *count) { size_t i; size_t bits = 0; @@ -1052,10 +1052,10 @@ annotate_ecksum(nvlist_t *ereport, zio_bad_cksum_t *info, } else { fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ZFS_BAD_SET_HISTOGRAM, - DATA_TYPE_UINT32_ARRAY, + DATA_TYPE_UINT8_ARRAY, NBBY * sizeof (uint64_t), eip->zei_histogram_set, FM_EREPORT_PAYLOAD_ZFS_BAD_CLEARED_HISTOGRAM, - DATA_TYPE_UINT32_ARRAY, + DATA_TYPE_UINT8_ARRAY, NBBY * sizeof (uint64_t), eip->zei_histogram_cleared, NULL); } From 574e09d8c6b49f223417e9aadb14367ac9db18f6 Mon Sep 17 00:00:00 2001 From: George Amanakis Date: Wed, 12 Apr 2023 17:53:53 +0200 Subject: [PATCH 20/59] Fix in check_filesystem() Fix the code in case of missing snapshots. Previously the check was in a conditional that would be executed if the filesystem had snapshots. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: George Amanakis Closes #14735 --- module/zfs/spa_errlog.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/module/zfs/spa_errlog.c b/module/zfs/spa_errlog.c index af144ef16978..3bc8619b51a8 100644 --- a/module/zfs/spa_errlog.c +++ b/module/zfs/spa_errlog.c @@ -354,12 +354,12 @@ check_filesystem(spa_t *spa, uint64_t head_ds, zbookmark_err_phys_t *zep, dsl_dataset_rele(ds, FTAG); return (error); } + } - if (snap_count == 0) { - /* Filesystem without snapshots. */ - dsl_dataset_rele(ds, FTAG); - return (0); - } + if (snap_count == 0) { + /* Filesystem without snapshots. */ + dsl_dataset_rele(ds, FTAG); + return (0); } uint64_t *snap_obj_array = kmem_zalloc(snap_count * sizeof (uint64_t), From 6e015933f88fe7ba5de45cf263028de1ee04460a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Wed, 12 Apr 2023 19:08:49 +0200 Subject: [PATCH 21/59] initramfs: source user scripts from /e/z/initramfs-tools-load-key{,.d/*} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By dropping in a file in a directory (for packages) or by making a file (for local administrators), custom key loading methods may be provided for the rootfs and necessities. Reviewed-by: Brian Behlendorf Signed-off-by: Nicholas Morris Signed-off-by: Ahelenia Ziemiańska Co-authored-by: Nicholas Morris Supersedes: #14704 Closes: #13757 Closes #14733 --- contrib/initramfs/README.md | 25 ++++++++++++++++++++++++- contrib/initramfs/hooks/zfs.in | 3 +++ contrib/initramfs/scripts/zfs | 10 ++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/contrib/initramfs/README.md b/contrib/initramfs/README.md index 34e9bab3c756..68647fa9fc3d 100644 --- a/contrib/initramfs/README.md +++ b/contrib/initramfs/README.md @@ -78,7 +78,30 @@ To use this feature: 1. Install the `dropbear-initramfs` package. You may wish to uninstall the `cryptsetup-initramfs` package to avoid warnings. 2. Add your SSH key(s) to `/etc/dropbear-initramfs/authorized_keys`. Note - that Dropbear does not support ed25519 keys before version 2020.79; + that Dropbear does not support ed25519 keys before version 2020.79; in that case, use RSA (2048-bit or more) instead. 3. Rebuild the initramfs with your keys: `update-initramfs -u` 4. During the system boot, login via SSH and run: `zfsunlock` + +### Unlocking a ZFS encrypted root via alternate means + +If present, a shell program at `/etc/zfs/initramfs-tools-load-key` +and files matching `/etc/zfs/initramfs-tools-load-key.d/*` +will be copied to the initramfs during generation +and sourced to load the key, if required. + +The `$ENCRYPTIONROOT` to load the key for and `$KEYLOCATION` variables are set, +and all initramfs-tools functions are available; +use unquoted `$ZPOOL` and `$ZFS` to run `zpool` and `zfs`. + +A successful return (and loaded key) stops the search. +A failure return is non-fatal, +and loading keys proceeds as normal if no hook succeeds. + +A trivial example of a key-loading drop-in that uses the BLAKE2 checksum +of the file at the `keylocation` as the key follows. + +```sh +key="$(b2sum "${KEYLOCATION#file://}")" || return +printf '%s\n' "${key%% *}" | $ZFS load-key -L prompt "$ENCRYPTIONROOT" +``` diff --git a/contrib/initramfs/hooks/zfs.in b/contrib/initramfs/hooks/zfs.in index 28dd252eea52..6cd7e6f1ea3b 100755 --- a/contrib/initramfs/hooks/zfs.in +++ b/contrib/initramfs/hooks/zfs.in @@ -41,6 +41,9 @@ copy_file cache "@sysconfdir@/zfs/zpool.cache" copy_file config "@initconfdir@/zfs" copy_file config "@sysconfdir@/zfs/zfs-functions" copy_file config "@sysconfdir@/zfs/vdev_id.conf" +for f in "@sysconfdir@/zfs/initramfs-tools-load-key" "@sysconfdir@/zfs/initramfs-tools-load-key.d/"*; do + copy_file config "$f" +done copy_file rule "@udevruledir@/60-zvol.rules" copy_file rule "@udevruledir@/69-vdev.rules" diff --git a/contrib/initramfs/scripts/zfs b/contrib/initramfs/scripts/zfs index 23aa95efc8f4..7f977a30f75b 100644 --- a/contrib/initramfs/scripts/zfs +++ b/contrib/initramfs/scripts/zfs @@ -420,6 +420,16 @@ decrypt_fs() # Continue only if the key needs to be loaded [ "$KEYSTATUS" = "unavailable" ] || return 0 + # Try extensions first + for f in "/etc/zfs/initramfs-tools-load-key" "/etc/zfs/initramfs-tools-load-key.d/"*; do + [ -r "$f" ] || continue + (. "$f") && { + # Successful return and actually-loaded key: we're done + KEYSTATUS="$(get_fs_value "${ENCRYPTIONROOT}" keystatus)" + [ "$KEYSTATUS" = "unavailable" ] || return 0 + } + done + # Do not prompt if key is stored noninteractively, if ! [ "${KEYLOCATION}" = "prompt" ]; then $ZFS load-key "${ENCRYPTIONROOT}" From c71fe716401f6919068f84b389dcd1b7ec2b8b0e Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Thu, 13 Apr 2023 08:15:05 +0900 Subject: [PATCH 22/59] Fix data corruption when cloning embedded blocks Don't overwrite blk_phys_birth, as for embedded blocks it is part of the payload. Reviewed-by: Richard Yao Reviewed-by: Brian Behlendorf Signed-off-by: Pawel Jakub Dawidek Issue #13392 Closes #14739 --- module/zfs/dmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index ce985d833f58..cda1472a77aa 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -2312,8 +2312,10 @@ dmu_brt_clone(objset_t *os, uint64_t object, uint64_t offset, uint64_t length, dl->dr_overridden_by.blk_phys_birth = 0; } else { dl->dr_overridden_by.blk_birth = dr->dr_txg; - dl->dr_overridden_by.blk_phys_birth = - BP_PHYSICAL_BIRTH(bp); + if (!BP_IS_EMBEDDED(bp)) { + dl->dr_overridden_by.blk_phys_birth = + BP_PHYSICAL_BIRTH(bp); + } } mutex_exit(&db->db_mtx); From 27a82cbb3ef2e30a54860b955fb257fb7f8307cd Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Thu, 13 Apr 2023 12:12:03 -0400 Subject: [PATCH 23/59] Linux 6.3 compat: Fix memcpy "detected field-spanning write" error Add a new union member of flexible array to dnode_phys_t and use it in the macro so we can silence the memcpy() fortify error. Reviewed-by: Brian Behlendorf Signed-off-by: Youzhong Yang Closes #14737 --- include/sys/dnode.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/include/sys/dnode.h b/include/sys/dnode.h index 2d741ea36bd0..dbe7350d4da7 100644 --- a/include/sys/dnode.h +++ b/include/sys/dnode.h @@ -120,7 +120,11 @@ extern "C" { #define DN_MAX_LEVELS (DIV_ROUND_UP(DN_MAX_OFFSET_SHIFT - SPA_MINBLOCKSHIFT, \ DN_MIN_INDBLKSHIFT - SPA_BLKPTRSHIFT) + 1) -#define DN_BONUS(dnp) ((void*)((dnp)->dn_bonus + \ +/* + * Use the flexible array instead of the fixed length one dn_bonus + * to address memcpy/memmove fortify error + */ +#define DN_BONUS(dnp) ((void*)((dnp)->dn_bonus_flexible + \ (((dnp)->dn_nblkptr - 1) * sizeof (blkptr_t)))) #define DN_MAX_BONUS_LEN(dnp) \ ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) ? \ @@ -266,6 +270,10 @@ typedef struct dnode_phys { sizeof (blkptr_t)]; blkptr_t dn_spill; }; + struct { + blkptr_t __dn_ignore4; + uint8_t dn_bonus_flexible[]; + }; }; } dnode_phys_t; From ac18dc77f3703940682aecb442f4e58aa2c14f1a Mon Sep 17 00:00:00 2001 From: dodexahedron Date: Thu, 13 Apr 2023 09:15:34 -0700 Subject: [PATCH 24/59] Minor improvements to zpoolconcepts.7 * Fixed one typo (effects -> affects) * Re-worded raidz description to make it clearer that it is not quite the same as RAID5, though similar * Clarified that data is not necessarily written in a static stripe width * Minor grammar consistency improvement * Noted that "volumes" means zvols * Fixed a couple of split infinitives * Clarified that hot spares come from the same pool they were assigned to * "we" -> ZFS * Fixed warnings thrown by mandoc, and removed unnecessary wordiness in one fixed line. Reviewed-by: Brian Behlendorf Signed-off-by: Brandon Thetford Closes #14726 --- man/man7/zpoolconcepts.7 | 64 +++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/man/man7/zpoolconcepts.7 b/man/man7/zpoolconcepts.7 index 4ef96b157564..db3fd4926236 100644 --- a/man/man7/zpoolconcepts.7 +++ b/man/man7/zpoolconcepts.7 @@ -26,7 +26,7 @@ .\" Copyright 2017 Nexenta Systems, Inc. .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved. .\" -.Dd June 2, 2021 +.Dd April 7, 2023 .Dt ZPOOLCONCEPTS 7 .Os . @@ -36,7 +36,7 @@ . .Sh DESCRIPTION .Ss Virtual Devices (vdevs) -A "virtual device" describes a single device or a collection of devices +A "virtual device" describes a single device or a collection of devices, organized according to certain performance and fault characteristics. The following virtual devices are supported: .Bl -tag -width "special" @@ -66,13 +66,14 @@ A mirror of two or more devices. Data is replicated in an identical fashion across all components of a mirror. A mirror with .Em N No disks of size Em X No can hold Em X No bytes and can withstand Em N-1 -devices failing without losing data. +devices failing, without losing data. .It Sy raidz , raidz1 , raidz2 , raidz3 -A variation on RAID-5 that allows for better distribution of parity and -eliminates the RAID-5 -.Qq write hole +A distributed-parity layout, similar to RAID-5/6, with improved distribution of +parity, and which does not suffer from the RAID-5/6 +.Qq write hole , .Pq in which data and parity become inconsistent after a power loss . -Data and parity is striped across all disks within a raidz group. +Data and parity is striped across all disks within a raidz group, though not +necessarily in a consistent stripe width. .Pp A raidz group can have single, double, or triple parity, meaning that the raidz group can sustain one, two, or three failures, respectively, without @@ -96,8 +97,8 @@ The minimum number of devices in a raidz group is one more than the number of parity disks. The recommended number is between 3 and 9 to help increase performance. .It Sy draid , draid1 , draid2 , draid3 -A variant of raidz that provides integrated distributed hot spares which -allows for faster resilvering while retaining the benefits of raidz. +A variant of raidz that provides integrated distributed hot spares, allowing +for faster resilvering, while retaining the benefits of raidz. A dRAID vdev is constructed from multiple internal raidz groups, each with .Em D No data devices and Em P No parity devices . These groups are distributed over all of the children in order to fully @@ -105,12 +106,12 @@ utilize the available disk performance. .Pp Unlike raidz, dRAID uses a fixed stripe width (padding as necessary with zeros) to allow fully sequential resilvering. -This fixed stripe width significantly effects both usable capacity and IOPS. +This fixed stripe width significantly affects both usable capacity and IOPS. For example, with the default .Em D=8 No and Em 4 KiB No disk sectors the minimum allocation size is Em 32 KiB . If using compression, this relatively large allocation size can reduce the effective compression ratio. -When using ZFS volumes and dRAID, the default of the +When using ZFS volumes (zvols) and dRAID, the default of the .Sy volblocksize property is increased to account for the allocation size. If a dRAID pool will hold a significant amount of small blocks, it is @@ -118,7 +119,7 @@ recommended to also add a mirrored .Sy special vdev to store those blocks. .Pp -In regards to I/O, performance is similar to raidz since for any read all +In regards to I/O, performance is similar to raidz since, for any read, all .Em D No data disks must be accessed . Delivered random IOPS can be reasonably approximated as .Sy floor((N-S)/(D+P))*single_drive_IOPS . @@ -178,7 +179,7 @@ For more information, see the .Sx Intent Log section. .It Sy dedup -A device dedicated solely for deduplication tables. +A device solely dedicated for deduplication tables. The redundancy of this device should match the redundancy of the other normal devices in the pool. If more than one dedup device is specified, then @@ -230,7 +231,7 @@ each a mirror of two disks: ZFS supports a rich set of mechanisms for handling device failure and data corruption. All metadata and data is checksummed, and ZFS automatically repairs bad data -from a good copy when corruption is detected. +from a good copy, when corruption is detected. .Pp In order to take advantage of these features, a pool must make use of some form of redundancy, using either mirrored or raidz groups. @@ -247,7 +248,7 @@ A faulted pool has corrupted metadata, or one or more faulted devices, and insufficient replicas to continue functioning. .Pp The health of the top-level vdev, such as a mirror or raidz device, -is potentially impacted by the state of its associated vdevs, +is potentially impacted by the state of its associated vdevs or component devices. A top-level vdev or component device is in one of the following states: .Bl -tag -width "DEGRADED" @@ -319,14 +320,15 @@ In this case, checksum errors are reported for all disks on which the block is stored. .Pp If a device is removed and later re-attached to the system, -ZFS attempts online the device automatically. +ZFS attempts to bring the device online automatically. Device attachment detection is hardware-dependent and might not be supported on all platforms. . .Ss Hot Spares ZFS allows devices to be associated with pools as .Qq hot spares . -These devices are not actively used in the pool, but when an active device +These devices are not actively used in the pool. +But, when an active device fails, it is automatically replaced by a hot spare. To create a pool with hot spares, specify a .Sy spare @@ -343,10 +345,10 @@ Once a spare replacement is initiated, a new .Sy spare vdev is created within the configuration that will remain there until the original device is replaced. -At this point, the hot spare becomes available again if another device fails. +At this point, the hot spare becomes available again, if another device fails. .Pp -If a pool has a shared spare that is currently being used, the pool can not be -exported since other pools may use this shared spare, which may lead to +If a pool has a shared spare that is currently being used, the pool cannot be +exported, since other pools may use this shared spare, which may lead to potential data corruption. .Pp Shared spares add some risk. @@ -390,7 +392,7 @@ See the .Sx EXAMPLES section for an example of mirroring multiple log devices. .Pp -Log devices can be added, replaced, attached, detached and removed. +Log devices can be added, replaced, attached, detached, and removed. In addition, log devices are imported and exported as part of the pool that contains them. Mirrored devices can be removed by specifying the top-level mirror vdev. @@ -423,8 +425,8 @@ This can be disabled by setting .Sy l2arc_rebuild_enabled Ns = Ns Sy 0 . For cache devices smaller than .Em 1 GiB , -we do not write the metadata structures -required for rebuilding the L2ARC in order not to waste space. +ZFS does not write the metadata structures +required for rebuilding the L2ARC, to conserve space. This can be changed with .Sy l2arc_rebuild_blocks_min_l2size . The cache device header @@ -435,21 +437,21 @@ Setting will result in scanning the full-length ARC lists for cacheable content to be written in L2ARC (persistent ARC). If a cache device is added with -.Nm zpool Cm add -its label and header will be overwritten and its contents are not going to be +.Nm zpool Cm add , +its label and header will be overwritten and its contents will not be restored in L2ARC, even if the device was previously part of the pool. If a cache device is onlined with -.Nm zpool Cm online +.Nm zpool Cm online , its contents will be restored in L2ARC. -This is useful in case of memory pressure +This is useful in case of memory pressure, where the contents of the cache device are not fully restored in L2ARC. -The user can off- and online the cache device when there is less memory pressure -in order to fully restore its contents to L2ARC. +The user can off- and online the cache device when there is less memory +pressure, to fully restore its contents to L2ARC. . .Ss Pool checkpoint Before starting critical procedures that include destructive actions .Pq like Nm zfs Cm destroy , -an administrator can checkpoint the pool's state and in the case of a +an administrator can checkpoint the pool's state and, in the case of a mistake or failure, rewind the entire pool back to the checkpoint. Otherwise, the checkpoint can be discarded when the procedure has completed successfully. @@ -485,7 +487,7 @@ current state of the pool won't be scanned during a scrub. . .Ss Special Allocation Class Allocations in the special class are dedicated to specific block types. -By default this includes all metadata, the indirect blocks of user data, and +By default, this includes all metadata, the indirect blocks of user data, and any deduplication tables. The class can also be provisioned to accept small file blocks. .Pp From 3b5af2013992231645e0a462eef4171d1c48de17 Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Tue, 18 Apr 2023 08:42:09 +0900 Subject: [PATCH 25/59] Fix VERIFY(!zil_replaying(zilog, tx)) panic The zfs_log_clone_range() function is never called from the zfs_clone_range_replay() function, so I assumed it is safe to assert that zil_replaying() is never TRUE here. It turns out zil_replaying() also returns TRUE when the sync property is set to disabled. Fix the problem by just returning if zil_replaying() returns TRUE. Reviewed-by: Richard Yao Reviewed-by: Brian Behlendorf Reported by: Florian Smeets Signed-off-by: Pawel Jakub Dawidek Closes #14758 --- module/zfs/zfs_log.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/module/zfs/zfs_log.c b/module/zfs/zfs_log.c index d009c58d8644..50325907b0d1 100644 --- a/module/zfs/zfs_log.c +++ b/module/zfs/zfs_log.c @@ -905,9 +905,7 @@ zfs_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, znode_t *zp, uint64_t partlen, max_log_data; size_t i, partnbps; - VERIFY(!zil_replaying(zilog, tx)); - - if (zp->z_unlinked) + if (zil_replaying(zilog, tx) || zp->z_unlinked) return; max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); From 8ed62440eff5a3ea93b0a9b6f0b7e5e7290b667a Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Mon, 17 Apr 2023 20:38:09 -0400 Subject: [PATCH 26/59] Work around Raspberry Pi kernel packaging oddities On Debian and Ubuntu and friends, you get something like "linux-image-$(uname -r)" and "linux-headers-$(uname -r)" you can put a Depends on. On Raspberry Pi OS, you get "raspberrypi-kernel" and "raspberrypi-kernel-headers", with version numbers like 20230411. There is not, as far as I can tell, a reasonable way to map that to a kernel version short of reaching out and digging around in the changelogs or Makefile, so just special-case it so the packages don't fail to install at install time. They still might not build if the versions don't match, but I don't see a way to do anything about that... Reviewed-by: Brian Behlendorf Signed-off-by: Rich Ercolani Closes #14745 Closes #14747 --- contrib/debian/control.modules.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/debian/control.modules.in b/contrib/debian/control.modules.in index 70a165266d16..34eb7fafba7c 100644 --- a/contrib/debian/control.modules.in +++ b/contrib/debian/control.modules.in @@ -5,7 +5,7 @@ Maintainer: ZFS on Linux specific mailing list Build-Depends: debhelper-compat (= 10), dkms (>> 2.1.1.2-5), libtool, - linux-headers-_KVERS_ + linux-headers-_KVERS_ | raspberrypi-kernel-headers Standards-Version: 4.3.0 Homepage: http://www.openzfs.org/ Vcs-Git: https://github.com/openzfs/zfs.git @@ -14,7 +14,7 @@ Vcs-Browser: https://github.com/openzfs/zfs Package: openzfs-zfs-modules-_KVERS_ Architecture: _ARCH_ Provides: openzfs-zfs-modules -Depends: linux-image-_KVERS_ +Depends: linux-image-_KVERS_ | raspberrypi-kernel Recommends: openzfsutils Replaces: zfs-modules-_KVERS_ Conflicts: zfs-modules-_KVERS_ From accfdeb948dab87a9576084a582f5bfeda0bd2f2 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 18 Apr 2023 08:41:52 -0700 Subject: [PATCH 27/59] Revert "ZFS_IOC_COUNT_FILLED does unnecessary txg_wait_synced()" This reverts commit 4b3133e671b958fa2c915a4faf57812820124a7b. Users identified this commit as a possible source of data corruption: https://github.com/openzfs/zfs/issues/14753 Reviewed-by: Brian Behlendorf Signed-off-by: Tony Hutter Issue #14753 Closes #14761 --- module/zfs/dnode.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 367bfaa80726..d15268cd7bc7 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -1764,29 +1764,20 @@ dnode_try_claim(objset_t *os, uint64_t object, int slots) } /* - * Checks if the dnode might contain any uncommitted changes to data blocks. - * Dirty metadata (e.g. bonus buffer) does not count. + * Checks if the dnode contains any uncommitted dirty records. */ boolean_t dnode_is_dirty(dnode_t *dn) { mutex_enter(&dn->dn_mtx); + for (int i = 0; i < TXG_SIZE; i++) { - list_t *list = &dn->dn_dirty_records[i]; - for (dbuf_dirty_record_t *dr = list_head(list); - dr != NULL; dr = list_next(list, dr)) { - if (dr->dr_dbuf == NULL || - (dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID && - dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID)) { - mutex_exit(&dn->dn_mtx); - return (B_TRUE); - } - } - if (dn->dn_free_ranges[i] != NULL) { + if (multilist_link_active(&dn->dn_dirty_link[i])) { mutex_exit(&dn->dn_mtx); return (B_TRUE); } } + mutex_exit(&dn->dn_mtx); return (B_FALSE); @@ -2650,9 +2641,7 @@ dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, rw_enter(&dn->dn_struct_rwlock, RW_READER); if (dn->dn_phys->dn_nlevels == 0) { - if (!(flags & DNODE_FIND_HOLE)) { - error = SET_ERROR(ESRCH); - } + error = SET_ERROR(ESRCH); goto out; } From f9e1c63f8c32141bb18c0270d565e3bfc1bbd233 Mon Sep 17 00:00:00 2001 From: Low-power Date: Wed, 19 Apr 2023 02:34:41 +0800 Subject: [PATCH 28/59] Values printed by zpool-iostat(8) should be right-aligned This inappropriate left-alignment was introduced in 7bb7b1f. Reviewed-by: Tony Hutter Reviewed-by: Matthew Ahrens Reviewed-by: Tino Reichardt Signed-off-by: WHR Closes #14751 --- cmd/zpool/zpool_main.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 20f9cd679534..27e805943443 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -4272,13 +4272,17 @@ print_iostat_header(iostat_cbdata_t *cb) * by order of magnitude. Uses column_size to add padding. */ static void -print_stat_color(char *statbuf, unsigned int column_size) +print_stat_color(const char *statbuf, unsigned int column_size) { fputs(" ", stdout); + size_t len = strlen(statbuf); + while (len < column_size) { + fputc(' ', stdout); + column_size--; + } if (*statbuf == '0') { color_start(ANSI_GRAY); fputc('0', stdout); - column_size--; } else { for (; *statbuf; statbuf++) { if (*statbuf == 'K') color_start(ANSI_GREEN); @@ -4293,8 +4297,6 @@ print_stat_color(char *statbuf, unsigned int column_size) } } color_end(); - for (; column_size > 0; column_size--) - fputc(' ', stdout); } /* From 23f84d161ed0ef91854be6da301ccca5ede11eec Mon Sep 17 00:00:00 2001 From: youzhongyang Date: Tue, 18 Apr 2023 21:10:40 -0400 Subject: [PATCH 29/59] Silence clang warning of flexible array not at end Reviewed-by: Brian Behlendorf Reviewed-by: Jorgen Lundman Signed-off-by: Youzhong Yang Closes #14764 --- include/sys/dmu_objset.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/sys/dmu_objset.h b/include/sys/dmu_objset.h index d22c682875d8..9f6e0fdd601b 100644 --- a/include/sys/dmu_objset.h +++ b/include/sys/dmu_objset.h @@ -72,6 +72,10 @@ struct dmu_tx; */ #define OBJSET_CRYPT_PORTABLE_FLAGS_MASK (0) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wgnu-variable-sized-type-not-at-end" +#endif typedef struct objset_phys { dnode_phys_t os_meta_dnode; zil_header_t os_zil_header; @@ -88,6 +92,9 @@ typedef struct objset_phys { char os_pad1[OBJSET_PHYS_SIZE_V3 - OBJSET_PHYS_SIZE_V2 - sizeof (dnode_phys_t)]; } objset_phys_t; +#if defined(__clang__) +#pragma clang diagnostic pop +#endif typedef int (*dmu_objset_upgrade_cb_t)(objset_t *); From 3d37e7e5f540f513ab1d8fa61d9208c43b889401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=BD=D0=B0=D0=B1?= Date: Wed, 19 Apr 2023 18:03:42 +0200 Subject: [PATCH 30/59] zfsprops.7: update mandlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?h=f7e33bdbd6d1bdf9c3df8bba5abcf3399f957ac3 https://git.kernel.org/pub/scm/docs/man-pages/man-pages.git/commit/?id=7e59106e9c34458540f7d382d5b49071d1b7104f Fixes: commit fb9baa9b2045a193a3caf0a46b5cac5ef7a84b61 ("zfsprops.8: remove nbmand-not-used-on-Linux and pointer to mount(8)") Reviewed-by: Brian Behlendorf Signed-off-by: Ahelenia Ziemiańska Closes #14765 --- man/man7/zfsprops.7 | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/man/man7/zfsprops.7 b/man/man7/zfsprops.7 index 1f6ffb59e32f..8f6b919cfc0b 100644 --- a/man/man7/zfsprops.7 +++ b/man/man7/zfsprops.7 @@ -38,7 +38,7 @@ .\" Copyright (c) 2019, Kjeld Schouten-Lebbing .\" Copyright (c) 2022 Hewlett Packard Enterprise Development LP. .\" -.Dd July 21, 2022 +.Dd April 18, 2023 .Dt ZFSPROPS 7 .Os . @@ -80,7 +80,9 @@ for zettabyte The following are all valid .Pq and equal specifications: -.Li 1536M, 1.5g, 1.50GB . +.Li 1536M , +.Li 1.5g , +.Li 1.50GB . .Pp The values of non-numeric properties are case sensitive and must be lowercase, except for @@ -1254,10 +1256,12 @@ location. Controls whether the file system should be mounted with .Sy nbmand .Pq Non-blocking mandatory locks . -This is used for SMB clients. Changes to this property only take effect when the file system is umounted and remounted. -Support for these locks is scarce and not described by POSIX. +This was only supported by Linux prior to 5.15, and was buggy there, +and is not supported by +.Fx . +On Solaris it's used for SMB clients. .It Sy overlay Ns = Ns Sy on Ns | Ns Sy off Allow mounting on a busy directory or a directory which already contains files or directories. From 719534ca8e8d39e94aef35d753c1c41179791ef5 Mon Sep 17 00:00:00 2001 From: Ameer Hamza <106930537+ixhamza@users.noreply.github.com> Date: Wed, 19 Apr 2023 21:04:32 +0500 Subject: [PATCH 31/59] Fix "Detach spare vdev in case if resilvering does not happen" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spare vdev should detach from the pool when a disk is reinserted. However, spare detachment depends on the completion of resilvering, and if resilver does not schedule, the spare vdev keeps attached to the pool until the next resilvering. When a zfs pool contains several disks (25+ mirror), resilvering does not always happen when a disk is reinserted. In this patch, spare vdev is manually detached from the pool when resilvering does not occur and it has been tested on both Linux and FreeBSD. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Ameer Hamza Closes #14722 --- include/sys/spa.h | 1 + module/zfs/spa.c | 5 +++-- module/zfs/vdev.c | 12 +++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/include/sys/spa.h b/include/sys/spa.h index 79c46aa07709..a1c102020223 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -787,6 +787,7 @@ extern int bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx); #define SPA_ASYNC_L2CACHE_REBUILD 0x800 #define SPA_ASYNC_L2CACHE_TRIM 0x1000 #define SPA_ASYNC_REBUILD_DONE 0x2000 +#define SPA_ASYNC_DETACH_SPARE 0x4000 /* device manipulation */ extern int spa_vdev_add(spa_t *spa, nvlist_t *nvroot); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index dc202978c0f6..e76700a9caa3 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -7058,7 +7058,7 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, * Detach a device from a mirror or replacing vdev. * * If 'replace_done' is specified, only detach if the parent - * is a replacing vdev. + * is a replacing or a spare vdev. */ int spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) @@ -8281,7 +8281,8 @@ spa_async_thread(void *arg) * If any devices are done replacing, detach them. */ if (tasks & SPA_ASYNC_RESILVER_DONE || - tasks & SPA_ASYNC_REBUILD_DONE) { + tasks & SPA_ASYNC_REBUILD_DONE || + tasks & SPA_ASYNC_DETACH_SPARE) { spa_vdev_resilver_done(spa); } diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 7cf858c05051..241be8fd856c 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -4183,9 +4183,19 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate) if (wasoffline || (oldstate < VDEV_STATE_DEGRADED && - vd->vdev_state >= VDEV_STATE_DEGRADED)) + vd->vdev_state >= VDEV_STATE_DEGRADED)) { spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_ONLINE); + /* + * Asynchronously detach spare vdev if resilver or + * rebuild is not required + */ + if (vd->vdev_unspare && + !dsl_scan_resilvering(spa->spa_dsl_pool) && + !dsl_scan_resilver_scheduled(spa->spa_dsl_pool) && + !vdev_rebuild_active(tvd)) + spa_async_request(spa, SPA_ASYNC_DETACH_SPARE); + } return (spa_vdev_state_exit(spa, vd, 0)); } From d4657835c8a5da816ab098fd2f1d62480865d087 Mon Sep 17 00:00:00 2001 From: Paul Dagnelie Date: Wed, 19 Apr 2023 13:20:02 -0700 Subject: [PATCH 32/59] ZTS: send-c_volume is flaky We use block_device_wait to wait for the zvol block device to actually appear, and we log the result of the dd calls by using an intermediate file. Reviewed-by: George Melikov Reviewed-by: John Wren Kennedy Reviewed-by: Brian Behlendorf Signed-off-by: Paul Dagnelie Closes #14767 --- tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh index 988ed91b9918..1bf234823459 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send-c_volume.ksh @@ -29,6 +29,7 @@ function cleanup { + rm $BACKDIR/copy log_must_busy zfs destroy -r $vol cleanup_pool $POOL2 } @@ -60,7 +61,9 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/full" verify_stream_size $BACKDIR/full $vol verify_stream_size $BACKDIR/full $vol2 -md5=$(dd if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest) +block_device_wait $voldev2 +log_must dd if=$voldev2 of=$BACKDIR/copy bs=1024k count=$megs +md5=$(md5digest $BACKDIR/copy) [[ $md5 = $md5_1 ]] || log_fail "md5 mismatch: $md5 != $md5_1" # Repeat, for an incremental send @@ -72,7 +75,9 @@ log_must eval "zfs recv -d $POOL2 <$BACKDIR/inc" verify_stream_size $BACKDIR/inc $vol 90 $vol@snap verify_stream_size $BACKDIR/inc $vol2 90 $vol2@snap -md5=$(dd skip=$megs if=$voldev2 bs=1024k count=$megs 2>/dev/null | md5digest) +block_device_wait $voldev2 +log_must dd skip=$megs if=$voldev2 of=$BACKDIR/copy bs=1024k count=$megs +md5=$(md5digest $BACKDIR/copy) [[ $md5 = $md5_2 ]] || log_fail "md5 mismatch: $md5 != $md5_2" log_pass "Verify compressed send works with volumes" From 71d191ef25d1c60e6725c07b6b94a0184f7db2eb Mon Sep 17 00:00:00 2001 From: Herb Wartens Date: Wed, 19 Apr 2023 13:22:59 -0700 Subject: [PATCH 33/59] Allow MMP to bypass waiting for other threads At our site we have seen cases when multi-modifier protection is enabled (multihost=on) on our pool and the pool gets suspended due to a single disk that is failing and responding very slowly. Our pools have 90 disks in them and we expect disks to fail. The current version of MMP requires that we wait for other writers before moving on. When a disk is responding very slowly, we observed that waiting here was bad enough to cause the pool to suspend. This change allows the MMP thread to bypass waiting for other threads and reduces the chances the pool gets suspended. Reviewed-by: Brian Behlendorf Signed-off-by: Herb Wartens Closes #14659 --- include/sys/spa.h | 2 ++ module/zfs/mmp.c | 2 +- module/zfs/spa_misc.c | 29 ++++++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/include/sys/spa.h b/include/sys/spa.h index a1c102020223..b96a9ef1d42f 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -977,6 +977,8 @@ extern int spa_import_progress_set_state(uint64_t pool_guid, extern int spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw); extern void spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw); +extern void spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, + krw_t rw); extern void spa_config_exit(spa_t *spa, int locks, const void *tag); extern int spa_config_held(spa_t *spa, int locks, krw_t rw); diff --git a/module/zfs/mmp.c b/module/zfs/mmp.c index ef0e01df390f..25eea0752941 100644 --- a/module/zfs/mmp.c +++ b/module/zfs/mmp.c @@ -445,7 +445,7 @@ mmp_write_uberblock(spa_t *spa) uint64_t offset; hrtime_t lock_acquire_time = gethrtime(); - spa_config_enter(spa, SCL_STATE, mmp_tag, RW_READER); + spa_config_enter_mmp(spa, SCL_STATE, mmp_tag, RW_READER); lock_acquire_time = gethrtime() - lock_acquire_time; if (lock_acquire_time > (MSEC2NSEC(MMP_MIN_INTERVAL) / 10)) zfs_dbgmsg("MMP SCL_STATE acquisition pool '%s' took %llu ns " diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 1475a4a53f4a..54a0eeccf27b 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -493,8 +493,9 @@ spa_config_tryenter(spa_t *spa, int locks, const void *tag, krw_t rw) return (1); } -void -spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) +static void +spa_config_enter_impl(spa_t *spa, int locks, const void *tag, krw_t rw, + int mmp_flag) { (void) tag; int wlocks_held = 0; @@ -509,7 +510,8 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) continue; mutex_enter(&scl->scl_lock); if (rw == RW_READER) { - while (scl->scl_writer || scl->scl_write_wanted) { + while (scl->scl_writer || + (!mmp_flag && scl->scl_write_wanted)) { cv_wait(&scl->scl_cv, &scl->scl_lock); } } else { @@ -527,6 +529,27 @@ spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) ASSERT3U(wlocks_held, <=, locks); } +void +spa_config_enter(spa_t *spa, int locks, const void *tag, krw_t rw) +{ + spa_config_enter_impl(spa, locks, tag, rw, 0); +} + +/* + * The spa_config_enter_mmp() allows the mmp thread to cut in front of + * outstanding write lock requests. This is needed since the mmp updates are + * time sensitive and failure to service them promptly will result in a + * suspended pool. This pool suspension has been seen in practice when there is + * a single disk in a pool that is responding slowly and presumably about to + * fail. + */ + +void +spa_config_enter_mmp(spa_t *spa, int locks, const void *tag, krw_t rw) +{ + spa_config_enter_impl(spa, locks, tag, rw, 1); +} + void spa_config_exit(spa_t *spa, int locks, const void *tag) { From 3e4ed4213d7b4e8892e9def8b06363391d8dbd60 Mon Sep 17 00:00:00 2001 From: rob-wing <98866084+rob-wing@users.noreply.github.com> Date: Thu, 20 Apr 2023 09:07:56 -0800 Subject: [PATCH 34/59] Create zap for root vdev And add it to the AVZ, this is not backwards compatible with older pools due to an assertion in spa_sync() that verifies the number of ZAPs of all vdevs matches the number of ZAPs in the AVZ. Granted, the assertion only applies to #DEBUG builds - still, a feature flag is introduced to avoid the assertion, com.klarasystems:vdev_zaps_v2 Notably, this allows to get/set properties on the root vdev: % zpool set user:prop=value root-0 Before this commit, it was already possible to get/set properties on top-level vdevs with the syntax - (e.g. mirror-0): % zpool set user:prop=value mirror-0 This syntax also applies to the root vdev as it is is of type 'root' with a vdev_id of 0, root-0. The keyword 'root' as an alias for 'root-0'. The following tests have been added: - zpool get all properties from root vdev - zpool set a property on root vdev - verify root vdev ZAP is created Reviewed-by: Brian Behlendorf Signed-off-by: Rob Wing Sponsored-by: Seagate Technology Submitted-by: Klara, Inc. Closes #14405 --- cmd/zdb/zdb.c | 3 + cmd/zpool/zpool_main.c | 67 +++++++++++------ include/sys/fs/zfs.h | 1 + include/sys/vdev_impl.h | 1 + include/zfeature_common.h | 1 + lib/libzfs/libzfs.abi | 11 +-- lib/libzfs/libzfs_pool.c | 1 + lib/libzutil/zutil_import.c | 5 +- man/man7/zpool-features.7 | 16 ++++ module/zcommon/zfeature_common.c | 6 ++ module/zfs/spa.c | 11 +++ module/zfs/vdev.c | 31 +++++++- module/zfs/vdev_label.c | 6 ++ tests/runfiles/common.run | 4 +- tests/zfs-tests/tests/Makefile.am | 3 + .../cli_root/zpool_get/vdev_get.cfg | 73 +++++++++++++++++++ .../cli_root/zpool_get/vdev_get_001_pos.ksh | 62 ++++++++++++++++ .../cli_root/zpool_get/zpool_get.cfg | 1 + .../cli_root/zpool_set/vdev_set_001_pos.ksh | 52 +++++++++++++ .../functional/vdev_zaps/vdev_zaps.kshlib | 13 ++++ .../vdev_zaps/vdev_zaps_001_pos.ksh | 3 +- .../vdev_zaps/vdev_zaps_002_pos.ksh | 1 + .../vdev_zaps/vdev_zaps_003_pos.ksh | 1 + .../vdev_zaps/vdev_zaps_004_pos.ksh | 1 + .../vdev_zaps/vdev_zaps_005_pos.ksh | 1 + .../vdev_zaps/vdev_zaps_006_pos.ksh | 1 + .../vdev_zaps/vdev_zaps_007_pos.ksh | 1 + 27 files changed, 339 insertions(+), 38 deletions(-) create mode 100644 tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg create mode 100755 tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get_001_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_001_pos.ksh diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index e87826f7467b..c93ed4399afd 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -7630,6 +7630,9 @@ mos_leak_vdev(vdev_t *vd) mos_obj_refd(space_map_object(ms->ms_sm)); } + if (vd->vdev_root_zap != 0) + mos_obj_refd(vd->vdev_root_zap); + if (vd->vdev_top_zap != 0) { mos_obj_refd(vd->vdev_top_zap); mos_leak_vdev_top_zap(vd); diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 27e805943443..4965cba52692 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -10001,33 +10001,33 @@ get_callback_vdev(zpool_handle_t *zhp, char *vdevname, void *data) return (0); } -static int -get_callback_vdev_width_cb(void *zhp_data, nvlist_t *nv, void *data) -{ - zpool_handle_t *zhp = zhp_data; - zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data; - char *vdevname = zpool_vdev_name(g_zfs, zhp, nv, - cbp->cb_vdevs.cb_name_flags); - int ret; - - /* Adjust the column widths for the vdev properties */ - ret = vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist); - - return (ret); -} - static int get_callback_vdev_cb(void *zhp_data, nvlist_t *nv, void *data) { zpool_handle_t *zhp = zhp_data; zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data; - char *vdevname = zpool_vdev_name(g_zfs, zhp, nv, - cbp->cb_vdevs.cb_name_flags); + char *vdevname; + const char *type; int ret; - /* Display the properties */ + /* + * zpool_vdev_name() transforms the root vdev name (i.e., root-0) to the + * pool name for display purposes, which is not desired. Fallback to + * zpool_vdev_name() when not dealing with the root vdev. + */ + type = fnvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE); + if (zhp != NULL && strcmp(type, "root") == 0) + vdevname = strdup("root-0"); + else + vdevname = zpool_vdev_name(g_zfs, zhp, nv, + cbp->cb_vdevs.cb_name_flags); + + (void) vdev_expand_proplist(zhp, vdevname, &cbp->cb_proplist); + ret = get_callback_vdev(zhp, vdevname, data); + free(vdevname); + return (ret); } @@ -10042,7 +10042,6 @@ get_callback(zpool_handle_t *zhp, void *data) if (cbp->cb_type == ZFS_TYPE_VDEV) { if (strcmp(cbp->cb_vdevs.cb_names[0], "all-vdevs") == 0) { - for_each_vdev(zhp, get_callback_vdev_width_cb, data); for_each_vdev(zhp, get_callback_vdev_cb, data); } else { /* Adjust column widths for vdev properties */ @@ -10119,6 +10118,7 @@ zpool_do_get(int argc, char **argv) int ret; int c, i; char *propstr = NULL; + char *vdev = NULL; cb.cb_first = B_TRUE; @@ -10216,10 +10216,17 @@ zpool_do_get(int argc, char **argv) } else if (are_all_pools(1, argv)) { /* The first arg is a pool name */ if ((argc == 2 && strcmp(argv[1], "all-vdevs") == 0) || + (argc == 2 && strcmp(argv[1], "root") == 0) || are_vdevs_in_pool(argc - 1, argv + 1, argv[0], &cb.cb_vdevs)) { + + if (strcmp(argv[1], "root") == 0) + vdev = strdup("root-0"); + else + vdev = strdup(argv[1]); + /* ... and the rest are vdev names */ - cb.cb_vdevs.cb_names = argv + 1; + cb.cb_vdevs.cb_names = &vdev; cb.cb_vdevs.cb_names_count = argc - 1; cb.cb_type = ZFS_TYPE_VDEV; argc = 1; /* One pool to process */ @@ -10264,6 +10271,9 @@ zpool_do_get(int argc, char **argv) else zprop_free_list(cb.cb_proplist); + if (vdev != NULL) + free(vdev); + return (ret); } @@ -10365,6 +10375,7 @@ zpool_do_set(int argc, char **argv) { set_cbdata_t cb = { 0 }; int error; + char *vdev = NULL; current_prop_type = ZFS_TYPE_POOL; if (argc > 1 && argv[1][0] == '-') { @@ -10413,13 +10424,20 @@ zpool_do_set(int argc, char **argv) /* argv[1], when supplied, is vdev name */ if (argc == 2) { - if (!are_vdevs_in_pool(1, argv + 1, argv[0], &cb.cb_vdevs)) { + + if (strcmp(argv[1], "root") == 0) + vdev = strdup("root-0"); + else + vdev = strdup(argv[1]); + + if (!are_vdevs_in_pool(1, &vdev, argv[0], &cb.cb_vdevs)) { (void) fprintf(stderr, gettext( "cannot find '%s' in '%s': device not in pool\n"), - argv[1], argv[0]); + vdev, argv[0]); + free(vdev); return (EINVAL); } - cb.cb_vdevs.cb_names = argv + 1; + cb.cb_vdevs.cb_names = &vdev; cb.cb_vdevs.cb_names_count = 1; cb.cb_type = ZFS_TYPE_VDEV; } @@ -10427,6 +10445,9 @@ zpool_do_set(int argc, char **argv) error = for_each_pool(1, argv, B_TRUE, NULL, ZFS_TYPE_POOL, B_FALSE, set_callback, &cb); + if (vdev != NULL) + free(vdev); + return (error); } diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 25babd4ea8cf..0734ff12280e 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -816,6 +816,7 @@ typedef struct zpool_load_policy { #define ZPOOL_CONFIG_FEATURES_FOR_READ "features_for_read" #define ZPOOL_CONFIG_FEATURE_STATS "feature_stats" /* not stored on disk */ #define ZPOOL_CONFIG_ERRATA "errata" /* not stored on disk */ +#define ZPOOL_CONFIG_VDEV_ROOT_ZAP "com.klarasystems:vdev_zap_root" #define ZPOOL_CONFIG_VDEV_TOP_ZAP "com.delphix:vdev_zap_top" #define ZPOOL_CONFIG_VDEV_LEAF_ZAP "com.delphix:vdev_zap_leaf" #define ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS "com.delphix:has_per_vdev_zaps" diff --git a/include/sys/vdev_impl.h b/include/sys/vdev_impl.h index 7cfffe3b4eed..ea3043c82a39 100644 --- a/include/sys/vdev_impl.h +++ b/include/sys/vdev_impl.h @@ -277,6 +277,7 @@ struct vdev { kthread_t *vdev_open_thread; /* thread opening children */ kthread_t *vdev_validate_thread; /* thread validating children */ uint64_t vdev_crtxg; /* txg when top-level was added */ + uint64_t vdev_root_zap; /* * Top-level vdev state. diff --git a/include/zfeature_common.h b/include/zfeature_common.h index ef915a70952e..7066c699e203 100644 --- a/include/zfeature_common.h +++ b/include/zfeature_common.h @@ -79,6 +79,7 @@ typedef enum spa_feature { SPA_FEATURE_HEAD_ERRLOG, SPA_FEATURE_BLAKE3, SPA_FEATURE_BLOCK_CLONING, + SPA_FEATURE_AVZ_V2, SPA_FEATURES } spa_feature_t; diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi index 41e74fd8db19..f9aed4e0d57e 100644 --- a/lib/libzfs/libzfs.abi +++ b/lib/libzfs/libzfs.abi @@ -595,7 +595,7 @@ - + @@ -5808,7 +5808,8 @@ - + + @@ -8694,8 +8695,8 @@ - - + + @@ -8772,7 +8773,7 @@ - + diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index f9b7cc004d6b..ae4c861590fd 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -2859,6 +2859,7 @@ zpool_vdev_is_interior(const char *name) strncmp(name, VDEV_TYPE_SPARE, strlen(VDEV_TYPE_SPARE)) == 0 || strncmp(name, VDEV_TYPE_REPLACING, strlen(VDEV_TYPE_REPLACING)) == 0 || + strncmp(name, VDEV_TYPE_ROOT, strlen(VDEV_TYPE_ROOT)) == 0 || strncmp(name, VDEV_TYPE_MIRROR, strlen(VDEV_TYPE_MIRROR)) == 0) return (B_TRUE); diff --git a/lib/libzutil/zutil_import.c b/lib/libzutil/zutil_import.c index 65f462e42cd0..19d8a4742813 100644 --- a/lib/libzutil/zutil_import.c +++ b/lib/libzutil/zutil_import.c @@ -1927,9 +1927,8 @@ for_each_vdev_cb(void *zhp, nvlist_t *nv, pool_vdev_iter_f func, if (nvlist_lookup_string(nv, ZPOOL_CONFIG_TYPE, &type) != 0) return (ret); - /* Don't run our function on root or indirect vdevs */ - if ((strcmp(type, VDEV_TYPE_ROOT) != 0) && - (strcmp(type, VDEV_TYPE_INDIRECT) != 0)) { + /* Don't run our function on indirect vdevs */ + if (strcmp(type, VDEV_TYPE_INDIRECT) != 0) { ret |= func(zhp, nv, data); } diff --git a/man/man7/zpool-features.7 b/man/man7/zpool-features.7 index 4cd7526858a3..efe9e833996a 100644 --- a/man/man7/zpool-features.7 +++ b/man/man7/zpool-features.7 @@ -858,6 +858,22 @@ by user and group. \*[instant-never] \*[remount-upgrade] . +.feature com.klarasystems vdev_zaps_v2 no +This feature creates a ZAP object for the root vdev. +.Pp +This feature becomes active after the next +.Nm zpool Cm import +or +.Nm zpool reguid . +. +Properties can be retrieved or set on the root vdev using +.Nm zpool Cm get +and +.Nm zpool Cm set +with +.Sy root +as the vdev name which is an alias for +.Sy root-0 . .feature org.openzfs zilsaxattr yes extensible_dataset This feature enables .Sy xattr Ns = Ns Sy sa diff --git a/module/zcommon/zfeature_common.c b/module/zcommon/zfeature_common.c index 6fe1da8ed46f..4c9b7ed72a0f 100644 --- a/module/zcommon/zfeature_common.c +++ b/module/zcommon/zfeature_common.c @@ -731,6 +731,12 @@ zpool_feature_init(void) ZFEATURE_FLAG_READONLY_COMPAT, ZFEATURE_TYPE_BOOLEAN, NULL, sfeatures); + zfeature_register(SPA_FEATURE_AVZ_V2, + "com.klarasystems:vdev_zaps_v2", "vdev_zaps_v2", + "Support for root vdev ZAP.", + ZFEATURE_FLAG_MOS, ZFEATURE_TYPE_BOOLEAN, NULL, + sfeatures); + zfs_mod_list_supported_free(sfeatures); } diff --git a/module/zfs/spa.c b/module/zfs/spa.c index e76700a9caa3..67601211d6c2 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -3044,6 +3044,12 @@ vdev_count_verify_zaps(vdev_t *vd) spa_t *spa = vd->vdev_spa; uint64_t total = 0; + if (spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2) && + vd->vdev_root_zap != 0) { + total++; + ASSERT0(zap_lookup_int(spa->spa_meta_objset, + spa->spa_all_vdev_zaps, vd->vdev_root_zap)); + } if (vd->vdev_top_zap != 0) { total++; ASSERT0(zap_lookup_int(spa->spa_meta_objset, @@ -8626,6 +8632,11 @@ spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) { spa_t *spa = vd->vdev_spa; + if (vd->vdev_root_zap != 0 && + spa_feature_is_active(spa, SPA_FEATURE_AVZ_V2)) { + VERIFY0(zap_add_int(spa->spa_meta_objset, avz, + vd->vdev_root_zap, tx)); + } if (vd->vdev_top_zap != 0) { VERIFY0(zap_add_int(spa->spa_meta_objset, avz, vd->vdev_top_zap, tx)); diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 241be8fd856c..4bfd95861e02 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -397,7 +397,9 @@ vdev_prop_get_int(vdev_t *vd, vdev_prop_t prop, uint64_t *value) uint64_t objid; int err; - if (vd->vdev_top_zap != 0) { + if (vd->vdev_root_zap != 0) { + objid = vd->vdev_root_zap; + } else if (vd->vdev_top_zap != 0) { objid = vd->vdev_top_zap; } else if (vd->vdev_leaf_zap != 0) { objid = vd->vdev_leaf_zap; @@ -898,6 +900,14 @@ vdev_alloc(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, uint_t id, (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_CREATE_TXG, &vd->vdev_crtxg); + if (vd->vdev_ops == &vdev_root_ops && + (alloctype == VDEV_ALLOC_LOAD || + alloctype == VDEV_ALLOC_SPLIT || + alloctype == VDEV_ALLOC_ROOTPOOL)) { + (void) nvlist_lookup_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP, + &vd->vdev_root_zap); + } + /* * If we're a top-level vdev, try to load the allocation parameters. */ @@ -3347,6 +3357,12 @@ vdev_construct_zaps(vdev_t *vd, dmu_tx_t *tx) vdev_zap_allocation_data(vd, tx); } } + if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap == 0 && + spa_feature_is_enabled(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) { + if (!spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) + spa_feature_incr(vd->vdev_spa, SPA_FEATURE_AVZ_V2, tx); + vd->vdev_root_zap = vdev_create_link_zap(vd, tx); + } for (uint64_t i = 0; i < vd->vdev_children; i++) { vdev_construct_zaps(vd->vdev_child[i], tx); @@ -5683,12 +5699,17 @@ vdev_props_set_sync(void *arg, dmu_tx_t *tx) /* * Set vdev property values in the vdev props mos object. */ - if (vd->vdev_top_zap != 0) { + if (vd->vdev_root_zap != 0) { + objid = vd->vdev_root_zap; + } else if (vd->vdev_top_zap != 0) { objid = vd->vdev_top_zap; } else if (vd->vdev_leaf_zap != 0) { objid = vd->vdev_leaf_zap; } else { - panic("vdev not top or leaf"); + /* + * XXX: implement vdev_props_set_check() + */ + panic("vdev not root/top/leaf"); } switch (prop = vdev_name_to_prop(propname)) { @@ -5891,7 +5912,9 @@ vdev_prop_get(vdev_t *vd, nvlist_t *innvl, nvlist_t *outnvl) nvlist_lookup_nvlist(innvl, ZPOOL_VDEV_PROPS_GET_PROPS, &nvprops); - if (vd->vdev_top_zap != 0) { + if (vd->vdev_root_zap != 0) { + objid = vd->vdev_root_zap; + } else if (vd->vdev_top_zap != 0) { objid = vd->vdev_top_zap; } else if (vd->vdev_leaf_zap != 0) { objid = vd->vdev_leaf_zap; diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index f61be65a2c72..85c7134ca4c4 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -573,6 +573,12 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, vd->vdev_top_zap); } + if (vd->vdev_ops == &vdev_root_ops && vd->vdev_root_zap != 0 && + spa_feature_is_active(vd->vdev_spa, SPA_FEATURE_AVZ_V2)) { + fnvlist_add_uint64(nv, ZPOOL_CONFIG_VDEV_ROOT_ZAP, + vd->vdev_root_zap); + } + if (vd->vdev_resilver_deferred) { ASSERT(vd->vdev_ops->vdev_op_leaf); ASSERT(spa->spa_resilver_deferred); diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 4233c0285c4b..cc4ce03677cb 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -394,7 +394,7 @@ tags = ['functional', 'cli_root', 'zpool_export'] [tests/functional/cli_root/zpool_get] tests = ['zpool_get_001_pos', 'zpool_get_002_pos', 'zpool_get_003_pos', - 'zpool_get_004_neg', 'zpool_get_005_pos'] + 'zpool_get_004_neg', 'zpool_get_005_pos', 'vdev_get_001_pos'] tags = ['functional', 'cli_root', 'zpool_get'] [tests/functional/cli_root/zpool_history] @@ -482,7 +482,7 @@ tags = ['functional', 'cli_root', 'zpool_scrub'] [tests/functional/cli_root/zpool_set] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg', - 'zpool_set_ashift', 'zpool_set_features'] + 'zpool_set_ashift', 'zpool_set_features', 'vdev_set_001_pos'] tags = ['functional', 'cli_root', 'zpool_set'] [tests/functional/cli_root/zpool_split] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index a470573616af..e671a3f6b02b 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -178,6 +178,7 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \ functional/cli_root/zpool_expand/zpool_expand.cfg \ functional/cli_root/zpool_export/zpool_export.cfg \ functional/cli_root/zpool_export/zpool_export.kshlib \ + functional/cli_root/zpool_get/vdev_get.cfg \ functional/cli_root/zpool_get/zpool_get.cfg \ functional/cli_root/zpool_get/zpool_get_parsable.cfg \ functional/cli_root/zpool_import/blockfiles/cryptv0.dat.bz2 \ @@ -1032,6 +1033,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool_export/zpool_export_004_pos.ksh \ functional/cli_root/zpool_get/cleanup.ksh \ functional/cli_root/zpool_get/setup.ksh \ + functional/cli_root/zpool_get/vdev_get_001_pos.ksh \ functional/cli_root/zpool_get/zpool_get_001_pos.ksh \ functional/cli_root/zpool_get/zpool_get_002_pos.ksh \ functional/cli_root/zpool_get/zpool_get_003_pos.ksh \ @@ -1146,6 +1148,7 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool_set/cleanup.ksh \ functional/cli_root/zpool_set/setup.ksh \ functional/cli_root/zpool/setup.ksh \ + functional/cli_root/zpool_set/vdev_set_001_pos.ksh \ functional/cli_root/zpool_set/zpool_set_001_pos.ksh \ functional/cli_root/zpool_set/zpool_set_002_neg.ksh \ functional/cli_root/zpool_set/zpool_set_003_neg.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg new file mode 100644 index 000000000000..71a64d4fae7a --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get.cfg @@ -0,0 +1,73 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# + +# Set the expected properties of a vdev +typeset -a properties=( + capacity + state + guid + asize + psize + ashift + size + free + allocated + comment + expandsize + fragmentation + bootsize + parity + path + devid + physpath + encpath + fru + parent + children + numchildren + read_errors + write_errors + checksum_errors + initialize_errors + null_ops + read_ops + write_ops + free_ops + claim_ops + trim_ops + null_bytes + read_bytes + write_bytes + free_bytes + claim_bytes + trim_bytes + removing + allocating + failfast + checksum_n + checksum_t + io_n + io_t +) diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get_001_pos.ksh new file mode 100755 index 000000000000..bca2337861d4 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/vdev_get_001_pos.ksh @@ -0,0 +1,62 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/tests/functional/cli_root/zpool_get/vdev_get.cfg + +# +# DESCRIPTION: +# +# zpool get root works as expected +# +# STRATEGY: +# +# 1. use zpool get to retrieve properties from root vdev +# 2. verify expected properties match detected properties +# + +log_assert "zpool get all on root vdev" + +EXPECT="$(zpool get -H all ${TESTPOOL} root | wc -l)" +if [ $? -ne 0 ]; then + log_fail "cannot retrieve properties from root vdev" +fi + +i=0; +while [ $i -lt "${#properties[@]}" ] +do + log_must zpool get -H "${properties[$i]}" "$TESTPOOL" root + i=$(($i+1)) +done + +EXPECT=$((EXPECT)) +if [ $i -gt $EXPECT ]; then + log_fail "found vdev properties not in vdev_get.cfg: $i/$EXPECT." +elif [ $i -lt $EXPECT ]; then + log_fail "expected properties not found in vdev_get.cfg: $i/$EXPECT." +fi + +log_pass "zpool get all on root vdev" diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg index 097cd52e4777..160a0ca2e6db 100644 --- a/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_get/zpool_get.cfg @@ -104,5 +104,6 @@ if is_linux || is_freebsd; then "feature@head_errlog" "feature@blake3" "feature@block_cloning" + "feature@vdev_zaps_v2" ) fi diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_001_pos.ksh new file mode 100755 index 000000000000..a1f3efb90577 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_set/vdev_set_001_pos.ksh @@ -0,0 +1,52 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2022, Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib + +# +# DESCRIPTION: +# +# zpool set comment property on root vdev +# +# STRATEGY: +# 1. set a property on root vdev +# 2. verify the property is set +# + +log_assert "zpool set comment property on root vdev" + +log_must zpool set comment="openzfs" ${TESTPOOL} root + +COMMENT="$(zpool get -H -o value comment ${TESTPOOL} root)" +if [ $? -ne 0 ]; then + log_fail "cant retrieve comment property from root vdev" +fi + +if [ "$COMMENT" != "openzfs" ]; then + log_fail "unexpected value for comment property: $COMMENT != \"openzfs\"" +fi + +log_pass "zpool set comment property on root vdev" diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps.kshlib b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps.kshlib index ad5bd9e7f81b..c68a5b2c4c83 100644 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps.kshlib +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps.kshlib @@ -34,6 +34,10 @@ function get_top_vd_zap # dsk conf { get_conf_section "$1" "$2" | awk '/com.delphix:vdev_zap_top: [0-9]+/ {print $2}' } +function get_root_vd_zap # conf +{ + awk '/com.klarasystems:vdev_zap_root: [0-9]+/ {print $2}' "$1" +} function assert_has_sentinel # conf { @@ -54,6 +58,15 @@ function assert_zap_common # pool vd lvl zapobj fi } +function assert_root_zap # pool conf +{ + typeset pool=$1 + typeset conf=$2 + + root_zap=$(get_root_vd_zap $conf) + assert_zap_common $pool "root vdev" "root" $root_zap +} + function assert_top_zap # pool vd conf { typeset pool=$1 diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_001_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_001_pos.ksh index b67cc6d973e6..bdc8dcd468ae 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_001_pos.ksh @@ -21,7 +21,7 @@ # # Strategy: # 1. Create a pool with one disk. -# 2. Verify that the disk has a top and leaf ZAP in its config and the MOS. +# 2. Verify that the disk has a root, top and leaf ZAP in its config and the MOS. # . $STF_SUITE/include/libtest.shlib @@ -35,6 +35,7 @@ log_must zpool create -f $TESTPOOL $DISK conf="$TESTDIR/vz001" log_must eval "zdb -PC $TESTPOOL > $conf" +assert_root_zap $TESTPOOL "$conf" assert_top_zap $TESTPOOL $DISK "$conf" assert_leaf_zap $TESTPOOL $DISK "$conf" assert_has_sentinel "$conf" diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_002_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_002_pos.ksh index c571973b080b..35c4f64fa463 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_002_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_002_pos.ksh @@ -36,6 +36,7 @@ conf="$TESTDIR/vz002" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" for DISK in $DISKS; do assert_top_zap $TESTPOOL $DISK "$conf" assert_leaf_zap $TESTPOOL $DISK "$conf" diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_003_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_003_pos.ksh index 015729576a7d..bb6875c339c3 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_003_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_003_pos.ksh @@ -37,6 +37,7 @@ conf="$TESTDIR/vz003" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" assert_top_zap $TESTPOOL "type: 'mirror'" "$conf" for DISK in $DISKS; do assert_leaf_zap $TESTPOOL $DISK "$conf" diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_004_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_004_pos.ksh index 3d0f55d5a9a7..e82e398c6d27 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_004_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_004_pos.ksh @@ -40,6 +40,7 @@ log_must zpool create -f $TESTPOOL $DISK conf="$TESTDIR/vz004" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" orig_top=$(get_top_vd_zap $DISK $conf) orig_leaf=$(get_leaf_vd_zap $DISK $conf) assert_zap_common $TESTPOOL $DISK "top" $orig_top diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_005_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_005_pos.ksh index 1d82218bf283..4b9b45e149d7 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_005_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_005_pos.ksh @@ -37,6 +37,7 @@ log_must zpool create -f $TESTPOOL $DISK conf="$TESTDIR/vz005" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" orig_top=$(get_top_vd_zap $DISK $conf) orig_leaf=$(get_leaf_vd_zap $DISK $conf) assert_zap_common $TESTPOOL $DISK "top" $orig_top diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_006_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_006_pos.ksh index ce94336c7c5d..2ac493b8b0d2 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_006_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_006_pos.ksh @@ -39,6 +39,7 @@ conf="$TESTDIR/vz006" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" orig_top=$(get_top_vd_zap ${DISK_ARR[1]} $conf) assert_zap_common $TESTPOOL ${DISK_ARR[1]} "top" $orig_top assert_leaf_zap $TESTPOOL ${DISK_ARR[1]} "$conf" diff --git a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_007_pos.ksh b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_007_pos.ksh index c7f12c633706..c7a4a62de436 100755 --- a/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_007_pos.ksh +++ b/tests/zfs-tests/tests/functional/vdev_zaps/vdev_zaps_007_pos.ksh @@ -39,6 +39,7 @@ conf="$TESTDIR/vz007" log_must eval "zdb -PC $TESTPOOL > $conf" assert_has_sentinel "$conf" +assert_root_zap $TESTPOOL "$conf" orig_top=$(get_top_vd_zap "type: 'mirror'" $conf) orig_leaf0=$(get_leaf_vd_zap ${DISK_ARR[0]} $conf) orig_leaf1=$(get_leaf_vd_zap ${DISK_ARR[1]} $conf) From a10c64561648f9cb9f90959c57625d3ffdaea156 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Thu, 20 Apr 2023 10:25:16 -0700 Subject: [PATCH 35/59] ZTS: zvol_misc_trim retry busy export Retry the export if the pool is busy due to an open zvol. Observed in the CI on Fedora 37. cannot export 'testpool': pool is busy ERROR: zpool export testpool exited 1 Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #14769 --- .../tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh index 2c4ef28ab826..46cac3ecb6c2 100755 --- a/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh +++ b/tests/zfs-tests/tests/functional/zvol/zvol_misc/zvol_misc_trim.ksh @@ -125,12 +125,12 @@ log_must $trimcmd $zvolpath set_blk_mq 1 -log_must zpool export $TESTPOOL +log_must_busy zpool export $TESTPOOL log_must zpool import $TESTPOOL do_test set_blk_mq 0 -log_must zpool export $TESTPOOL +log_must_busy zpool export $TESTPOOL log_must zpool import $TESTPOOL do_test From ab71b24d20df7ec59c6b3a2b560af263ad262d9b Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Tue, 11 Apr 2023 17:50:43 +0000 Subject: [PATCH 36/59] Linux: zfs_zaccess_trivial() should always call generic_permission() Building with Clang on Linux generates a warning that err could be uninitialized if mnt_ns is a NULL pointer. However, mnt_ns should never be NULL, so there is no need to put this behind an if statement. Taking it outside of the if statement means that the possibility of err being uninitialized goes from being always zero in a way that the compiler could not realize to a way that is always zero in a way that the compiler can realize. Sponsored-By: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Reviewed-by: Youzhong Yang Signed-off-by: Richard Yao Closes #14738 --- module/os/linux/zfs/zfs_acl.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/os/linux/zfs/zfs_acl.c b/module/os/linux/zfs/zfs_acl.c index df4ebc3870be..ff26f47f2e04 100644 --- a/module/os/linux/zfs/zfs_acl.c +++ b/module/os/linux/zfs/zfs_acl.c @@ -2466,8 +2466,7 @@ zfs_zaccess_trivial(znode_t *zp, uint32_t *working_mode, cred_t *cr, #if (defined(HAVE_IOPS_PERMISSION_USERNS) || \ defined(HAVE_IOPS_PERMISSION_IDMAP)) - if (mnt_ns) - err = generic_permission(mnt_ns, ZTOI(zp), mask); + err = generic_permission(mnt_ns, ZTOI(zp), mask); #else err = generic_permission(ZTOI(zp), mask); #endif From 135d9a9048e3716c755373182720d0eba170285f Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Tue, 11 Apr 2023 17:56:16 +0000 Subject: [PATCH 37/59] Linux: Suppress -Wordered-compare-function-pointers in tracepoint code Clang points out that there is a comparison against -1, but we cannot fix it because that is from the kernel headers, which we must support. We can workaround this by using a pragma. Sponsored-By: Wasabi Technology, Inc. Reviewed-by: Brian Behlendorf Reviewed-by: Youzhong Yang Signed-off-by: Richard Yao Closes #14738 --- include/os/linux/zfs/sys/trace_zil.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/os/linux/zfs/sys/trace_zil.h b/include/os/linux/zfs/sys/trace_zil.h index 6dd18c5974b9..fb03d3149f8f 100644 --- a/include/os/linux/zfs/sys/trace_zil.h +++ b/include/os/linux/zfs/sys/trace_zil.h @@ -152,6 +152,9 @@ * zilog_t *, ..., * itx_t *, ...); */ + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wordered-compare-function-pointers" /* BEGIN CSTYLED */ DECLARE_EVENT_CLASS(zfs_zil_process_itx_class, TP_PROTO(zilog_t *zilog, itx_t *itx), @@ -169,6 +172,7 @@ DECLARE_EVENT_CLASS(zfs_zil_process_itx_class, ZILOG_TP_PRINTK_ARGS, ITX_TP_PRINTK_ARGS) ); /* END CSTYLED */ +#pragma clang diagnostic pop #define DEFINE_ZIL_PROCESS_ITX_EVENT(name) \ DEFINE_EVENT(zfs_zil_process_itx_class, name, \ From 8eae2d214cfa53862833eeeda9a5c1e9d5ded47d Mon Sep 17 00:00:00 2001 From: Allan Jude Date: Fri, 21 Apr 2023 13:20:36 -0400 Subject: [PATCH 38/59] Add support for zpool user properties Usage: zpool set org.freebsd:comment="this is my pool" poolname Tests are based on zfs_set's user property tests. Also stop truncating property values at MAXNAMELEN, use ZFS_MAXPROPLEN. Reviewed-by: Brian Behlendorf Signed-off-by: Allan Jude Signed-off-by: Mateusz Piotrowski Sponsored-by: Beckhoff Automation GmbH & Co. KG. Sponsored-by: Klara Inc. Closes #11680 --- cmd/zpool/zpool_main.c | 18 +- include/libzfs.h | 2 + lib/libzfs/libzfs.abi | 131 +++++++------ lib/libzfs/libzfs_pool.c | 101 +++++++++- lib/libzfs/libzfs_util.c | 1 + man/man7/zpoolprops.7 | 55 +++++- module/zfs/spa.c | 134 ++++++++----- tests/runfiles/common.run | 3 +- tests/zfs-tests/tests/Makefile.am | 3 + .../zpool_set/user_property_001_pos.ksh | 89 +++++++++ .../zpool_set/user_property_002_neg.ksh | 88 +++++++++ .../zpool_set/zpool_set_common.kshlib | 178 ++++++++++++++++++ 12 files changed, 691 insertions(+), 112 deletions(-) create mode 100755 tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_001_pos.ksh create mode 100755 tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_002_neg.ksh create mode 100644 tests/zfs-tests/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 4965cba52692..301c5f4bfc6f 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -6071,11 +6071,14 @@ print_pool(zpool_handle_t *zhp, list_cbdata_t *cb) zpool_prop_get_feature(zhp, pl->pl_user_prop, property, sizeof (property)) == 0) { propstr = property; + } else if (zfs_prop_user(pl->pl_user_prop) && + zpool_get_userprop(zhp, pl->pl_user_prop, property, + sizeof (property), NULL) == 0) { + propstr = property; } else { propstr = "-"; } - /* * If this is being called in scripted mode, or if this is the * last column and it is left-justified, don't include a width @@ -10035,7 +10038,7 @@ static int get_callback(zpool_handle_t *zhp, void *data) { zprop_get_cbdata_t *cbp = (zprop_get_cbdata_t *)data; - char value[MAXNAMELEN]; + char value[ZFS_MAXPROPLEN]; zprop_source_t srctype; zprop_list_t *pl; int vid; @@ -10070,6 +10073,17 @@ get_callback(zpool_handle_t *zhp, void *data) continue; if (pl->pl_prop == ZPROP_INVAL && + zfs_prop_user(pl->pl_user_prop)) { + srctype = ZPROP_SRC_LOCAL; + + if (zpool_get_userprop(zhp, pl->pl_user_prop, + value, sizeof (value), &srctype) != 0) + continue; + + zprop_print_one_property(zpool_get_name(zhp), + cbp, pl->pl_user_prop, value, srctype, + NULL, NULL); + } else if (pl->pl_prop == ZPROP_INVAL && (zpool_prop_feature(pl->pl_user_prop) || zpool_prop_unsupported(pl->pl_user_prop))) { srctype = ZPROP_SRC_LOCAL; diff --git a/include/libzfs.h b/include/libzfs.h index 7ec9768d8e93..87d1ed738f2b 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -333,6 +333,8 @@ _LIBZFS_H const char *zpool_get_state_str(zpool_handle_t *); _LIBZFS_H int zpool_set_prop(zpool_handle_t *, const char *, const char *); _LIBZFS_H int zpool_get_prop(zpool_handle_t *, zpool_prop_t, char *, size_t proplen, zprop_source_t *, boolean_t literal); +_LIBZFS_H int zpool_get_userprop(zpool_handle_t *, const char *, char *, + size_t proplen, zprop_source_t *); _LIBZFS_H uint64_t zpool_get_prop_int(zpool_handle_t *, zpool_prop_t, zprop_source_t *); _LIBZFS_H int zpool_props_refresh(zpool_handle_t *); diff --git a/lib/libzfs/libzfs.abi b/lib/libzfs/libzfs.abi index f9aed4e0d57e..732863dcffc7 100644 --- a/lib/libzfs/libzfs.abi +++ b/lib/libzfs/libzfs.abi @@ -259,8 +259,8 @@ - + @@ -492,6 +492,7 @@ + @@ -2267,32 +2268,19 @@ - - - - - - - - - - - - - - - - - + + + + - - - - - + + + + + @@ -3324,17 +3312,11 @@ - - - - - - - - - - + + + + @@ -3907,35 +3889,20 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - + + + + @@ -5131,6 +5098,27 @@ + + + + + + + + + + + + + + + + + + + + + @@ -5163,6 +5151,19 @@ + + + + + + + + + + + + + @@ -5395,9 +5396,6 @@ - - - @@ -6169,6 +6167,14 @@ + + + + + + + + @@ -7852,6 +7858,9 @@ + + + diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index ae4c861590fd..4fb71b4e0dc8 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -426,6 +426,37 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf, return (0); } +/* + * Get a zpool property value for 'propname' and return the value in + * a pre-allocated buffer. + */ +int +zpool_get_userprop(zpool_handle_t *zhp, const char *propname, char *buf, + size_t len, zprop_source_t *srctype) +{ + nvlist_t *nv, *nvl; + uint64_t ival; + const char *value; + zprop_source_t source = ZPROP_SRC_LOCAL; + + nvl = zhp->zpool_props; + if (nvlist_lookup_nvlist(nvl, propname, &nv) == 0) { + if (nvlist_lookup_uint64(nv, ZPROP_SOURCE, &ival) == 0) + source = ival; + verify(nvlist_lookup_string(nv, ZPROP_VALUE, &value) == 0); + } else { + source = ZPROP_SRC_DEFAULT; + value = "-"; + } + + if (srctype) + *srctype = source; + + (void) strlcpy(buf, value, len); + + return (0); +} + /* * Check if the bootfs name has the same pool name as it is set to. * Assuming bootfs is a valid dataset name. @@ -549,6 +580,44 @@ zpool_valid_proplist(libzfs_handle_t *hdl, const char *poolname, (void) no_memory(hdl); goto error; } + continue; + } else if (prop == ZPOOL_PROP_INVAL && + zfs_prop_user(propname)) { + /* + * This is a user property: make sure it's a + * string, and that it's less than ZAP_MAXNAMELEN. + */ + if (nvpair_type(elem) != DATA_TYPE_STRING) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "'%s' must be a string"), propname); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + goto error; + } + + if (strlen(nvpair_name(elem)) >= ZAP_MAXNAMELEN) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "property name '%s' is too long"), + propname); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + goto error; + } + + (void) nvpair_value_string(elem, &strval); + + if (strlen(strval) >= ZFS_MAXPROPLEN) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "property value '%s' is too long"), + strval); + (void) zfs_error(hdl, EZFS_BADPROP, errbuf); + goto error; + } + + if (nvlist_add_string(retprops, propname, + strval) != 0) { + (void) no_memory(hdl); + goto error; + } + continue; } @@ -855,9 +924,30 @@ zpool_expand_proplist(zpool_handle_t *zhp, zprop_list_t **plp, features = zpool_get_features(zhp); if ((*plp)->pl_all && firstexpand) { + /* Handle userprops in the all properties case */ + if (zhp->zpool_props == NULL && zpool_props_refresh(zhp)) + return (-1); + + nvp = NULL; + while ((nvp = nvlist_next_nvpair(zhp->zpool_props, nvp)) != + NULL) { + const char *propname = nvpair_name(nvp); + + if (!zfs_prop_user(propname)) + continue; + + entry = zfs_alloc(hdl, sizeof (zprop_list_t)); + entry->pl_prop = ZPROP_USERPROP; + entry->pl_user_prop = zfs_strdup(hdl, propname); + entry->pl_width = strlen(entry->pl_user_prop); + entry->pl_all = B_TRUE; + + *last = entry; + last = &entry->pl_next; + } + for (i = 0; i < SPA_FEATURES; i++) { - zprop_list_t *entry = zfs_alloc(hdl, - sizeof (zprop_list_t)); + entry = zfs_alloc(hdl, sizeof (zprop_list_t)); entry->pl_prop = ZPROP_USERPROP; entry->pl_user_prop = zfs_asprintf(hdl, "feature@%s", spa_feature_table[i].fi_uname); @@ -874,7 +964,6 @@ zpool_expand_proplist(zpool_handle_t *zhp, zprop_list_t **plp, nvp != NULL; nvp = nvlist_next_nvpair(features, nvp)) { char *propname; boolean_t found; - zprop_list_t *entry; if (zfeature_is_supported(nvpair_name(nvp))) continue; @@ -920,6 +1009,12 @@ zpool_expand_proplist(zpool_handle_t *zhp, zprop_list_t **plp, NULL, literal) == 0) { if (strlen(buf) > entry->pl_width) entry->pl_width = strlen(buf); + } else if (entry->pl_prop == ZPROP_INVAL && + zfs_prop_user(entry->pl_user_prop) && + zpool_get_userprop(zhp, entry->pl_user_prop, buf, + sizeof (buf), NULL) == 0) { + if (strlen(buf) > entry->pl_width) + entry->pl_width = strlen(buf); } } diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index 393971ddf13c..4b8a20160e02 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -1774,6 +1774,7 @@ addlist(libzfs_handle_t *hdl, const char *propname, zprop_list_t **listp, * a user-defined property. */ if (prop == ZPROP_USERPROP && ((type == ZFS_TYPE_POOL && + !zfs_prop_user(propname) && !zpool_prop_feature(propname) && !zpool_prop_unsupported(propname)) || ((type == ZFS_TYPE_DATASET) && !zfs_prop_user(propname) && diff --git a/man/man7/zpoolprops.7 b/man/man7/zpoolprops.7 index 12b9b11903df..7709d85226dc 100644 --- a/man/man7/zpoolprops.7 +++ b/man/man7/zpoolprops.7 @@ -26,8 +26,9 @@ .\" Copyright 2017 Nexenta Systems, Inc. .\" Copyright (c) 2017 Open-E, Inc. All Rights Reserved. .\" Copyright (c) 2021, Colm Buckley +.\" Copyright (c) 2023, Klara Inc. .\" -.Dd May 27, 2021 +.Dd April 18, 2023 .Dt ZPOOLPROPS 7 .Os . @@ -40,6 +41,12 @@ Each pool has several properties associated with it. Some properties are read-only statistics while others are configurable and change the behavior of the pool. .Pp +User properties have no effect on ZFS behavior. +Use them to annotate pools in a way that is meaningful in your environment. +For more information about user properties, see the +.Sx User Properties +section. +.Pp The following are read-only properties: .Bl -tag -width "unsupported@guid" .It Sy allocated @@ -431,3 +438,49 @@ backwards compatibility. Once feature flags are enabled on a pool this property will no longer have a value. .El +. +.Ss User Properties +In addition to the standard native properties, ZFS supports arbitrary user +properties. +User properties have no effect on ZFS behavior, but applications or +administrators can use them to annotate pools. +.Pp +User property names must contain a colon +.Pq Qq Sy \&: +character to distinguish them from native properties. +They may contain lowercase letters, numbers, and the following punctuation +characters: colon +.Pq Qq Sy \&: , +dash +.Pq Qq Sy - , +period +.Pq Qq Sy \&. , +and underscore +.Pq Qq Sy _ . +The expected convention is that the property name is divided into two portions +such as +.Ar module : Ns Ar property , +but this namespace is not enforced by ZFS. +User property names can be at most 256 characters, and cannot begin with a dash +.Pq Qq Sy - . +.Pp +When making programmatic use of user properties, it is strongly suggested to use +a reversed DNS domain name for the +.Ar module +component of property names to reduce the chance that two +independently-developed packages use the same property name for different +purposes. +.Pp +The values of user properties are arbitrary strings and +are never validated. +All of the commands that operate on properties +.Po Nm zpool Cm list , +.Nm zpool Cm get , +.Nm zpool Cm set , +and so forth +.Pc +can be used to manipulate both native properties and user properties. +Use +.Nm zpool Cm set Ar name Ns = +to clear a user property. +Property values are limited to 8192 bytes. diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 67601211d6c2..dd4a442d97a1 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -296,6 +296,22 @@ spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, const char *strval, nvlist_free(propval); } +/* + * Add a user property (source=src, propname=propval) to an nvlist. + */ +static void +spa_prop_add_user(nvlist_t *nvl, const char *propname, char *strval, + zprop_source_t src) +{ + nvlist_t *propval; + + VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); + VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); + VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); + nvlist_free(propval); +} + /* * Get property values from the spa configuration. */ @@ -471,7 +487,8 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) zprop_source_t src = ZPROP_SRC_DEFAULT; zpool_prop_t prop; - if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) + if ((prop = zpool_name_to_prop(za.za_name)) == + ZPOOL_PROP_INVAL && !zfs_prop_user(za.za_name)) continue; switch (za.za_integer_length) { @@ -514,7 +531,13 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) kmem_free(strval, za.za_num_integers); break; } - spa_prop_add_list(*nvp, prop, strval, 0, src); + if (prop != ZPOOL_PROP_INVAL) { + spa_prop_add_list(*nvp, prop, strval, 0, src); + } else { + src = ZPROP_SRC_LOCAL; + spa_prop_add_user(*nvp, za.za_name, strval, + src); + } kmem_free(strval, za.za_num_integers); break; @@ -556,36 +579,47 @@ spa_prop_validate(spa_t *spa, nvlist_t *props) switch (prop) { case ZPOOL_PROP_INVAL: - if (!zpool_prop_feature(propname)) { - error = SET_ERROR(EINVAL); - break; - } - /* * Sanitize the input. */ - if (nvpair_type(elem) != DATA_TYPE_UINT64) { + if (zfs_prop_user(propname)) { + if (strlen(propname) >= ZAP_MAXNAMELEN) { + error = SET_ERROR(ENAMETOOLONG); + break; + } + + if (strlen(fnvpair_value_string(elem)) >= + ZAP_MAXVALUELEN) { + error = SET_ERROR(E2BIG); + break; + } + } else if (zpool_prop_feature(propname)) { + if (nvpair_type(elem) != DATA_TYPE_UINT64) { + error = SET_ERROR(EINVAL); + break; + } + + if (nvpair_value_uint64(elem, &intval) != 0) { + error = SET_ERROR(EINVAL); + break; + } + + if (intval != 0) { + error = SET_ERROR(EINVAL); + break; + } + + fname = strchr(propname, '@') + 1; + if (zfeature_lookup_name(fname, NULL) != 0) { + error = SET_ERROR(EINVAL); + break; + } + + has_feature = B_TRUE; + } else { error = SET_ERROR(EINVAL); break; } - - if (nvpair_value_uint64(elem, &intval) != 0) { - error = SET_ERROR(EINVAL); - break; - } - - if (intval != 0) { - error = SET_ERROR(EINVAL); - break; - } - - fname = strchr(propname, '@') + 1; - if (zfeature_lookup_name(fname, NULL) != 0) { - error = SET_ERROR(EINVAL); - break; - } - - has_feature = B_TRUE; break; case ZPOOL_PROP_VERSION: @@ -792,6 +826,12 @@ spa_prop_set(spa_t *spa, nvlist_t *nvp) prop == ZPOOL_PROP_READONLY) continue; + if (prop == ZPOOL_PROP_INVAL && + zfs_prop_user(nvpair_name(elem))) { + need_sync = B_TRUE; + break; + } + if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { uint64_t ver = 0; @@ -8800,24 +8840,11 @@ spa_sync_props(void *arg, dmu_tx_t *tx) const char *strval, *fname; zpool_prop_t prop; const char *propname; + const char *elemname = nvpair_name(elem); zprop_type_t proptype; spa_feature_t fid; - switch (prop = zpool_name_to_prop(nvpair_name(elem))) { - case ZPOOL_PROP_INVAL: - /* - * We checked this earlier in spa_prop_validate(). - */ - ASSERT(zpool_prop_feature(nvpair_name(elem))); - - fname = strchr(nvpair_name(elem), '@') + 1; - VERIFY0(zfeature_lookup_name(fname, &fid)); - - spa_feature_enable(spa, fid, tx); - spa_history_log_internal(spa, "set", tx, - "%s=enabled", nvpair_name(elem)); - break; - + switch (prop = zpool_name_to_prop(elemname)) { case ZPOOL_PROP_VERSION: intval = fnvpair_value_uint64(elem); /* @@ -8860,7 +8887,7 @@ spa_sync_props(void *arg, dmu_tx_t *tx) spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); } spa_history_log_internal(spa, "set", tx, - "%s=%s", nvpair_name(elem), strval); + "%s=%s", elemname, strval); break; case ZPOOL_PROP_COMPATIBILITY: strval = fnvpair_value_string(elem); @@ -8879,6 +8906,20 @@ spa_sync_props(void *arg, dmu_tx_t *tx) "%s=%s", nvpair_name(elem), strval); break; + case ZPOOL_PROP_INVAL: + if (zpool_prop_feature(elemname)) { + fname = strchr(elemname, '@') + 1; + VERIFY0(zfeature_lookup_name(fname, &fid)); + + spa_feature_enable(spa, fid, tx); + spa_history_log_internal(spa, "set", tx, + "%s=enabled", elemname); + break; + } else if (!zfs_prop_user(elemname)) { + ASSERT(zpool_prop_feature(elemname)); + break; + } + zfs_fallthrough; default: /* * Set pool property values in the poolprops mos object. @@ -8893,6 +8934,11 @@ spa_sync_props(void *arg, dmu_tx_t *tx) /* normalize the property name */ propname = zpool_prop_to_name(prop); proptype = zpool_prop_get_type(prop); + if (prop == ZPOOL_PROP_INVAL && + zfs_prop_user(elemname)) { + propname = elemname; + proptype = PROP_TYPE_STRING; + } if (nvpair_type(elem) == DATA_TYPE_STRING) { ASSERT(proptype == PROP_TYPE_STRING); @@ -8901,7 +8947,7 @@ spa_sync_props(void *arg, dmu_tx_t *tx) spa->spa_pool_props_object, propname, 1, strlen(strval) + 1, strval, tx)); spa_history_log_internal(spa, "set", tx, - "%s=%s", nvpair_name(elem), strval); + "%s=%s", elemname, strval); } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { intval = fnvpair_value_uint64(elem); @@ -8914,7 +8960,7 @@ spa_sync_props(void *arg, dmu_tx_t *tx) spa->spa_pool_props_object, propname, 8, 1, &intval, tx)); spa_history_log_internal(spa, "set", tx, - "%s=%lld", nvpair_name(elem), + "%s=%lld", elemname, (longlong_t)intval); switch (prop) { diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index cc4ce03677cb..55991cfeaf78 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -482,7 +482,8 @@ tags = ['functional', 'cli_root', 'zpool_scrub'] [tests/functional/cli_root/zpool_set] tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg', - 'zpool_set_ashift', 'zpool_set_features', 'vdev_set_001_pos'] + 'zpool_set_ashift', 'zpool_set_features', 'vdev_set_001_pos', + 'user_property_001_pos', 'user_property_002_neg'] tags = ['functional', 'cli_root', 'zpool_set'] [tests/functional/cli_root/zpool_split] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index e671a3f6b02b..74295b86ddc2 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -1149,10 +1149,13 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/cli_root/zpool_set/setup.ksh \ functional/cli_root/zpool/setup.ksh \ functional/cli_root/zpool_set/vdev_set_001_pos.ksh \ + functional/cli_root/zpool_set/zpool_set_common.kshlib \ functional/cli_root/zpool_set/zpool_set_001_pos.ksh \ functional/cli_root/zpool_set/zpool_set_002_neg.ksh \ functional/cli_root/zpool_set/zpool_set_003_neg.ksh \ functional/cli_root/zpool_set/zpool_set_ashift.ksh \ + functional/cli_root/zpool_set/user_property_001_pos.ksh \ + functional/cli_root/zpool_set/user_property_002_neg.ksh \ functional/cli_root/zpool_set/zpool_set_features.ksh \ functional/cli_root/zpool_split/cleanup.ksh \ functional/cli_root/zpool_split/setup.ksh \ diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_001_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_001_pos.ksh new file mode 100755 index 000000000000..4b9097933f37 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_001_pos.ksh @@ -0,0 +1,89 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2016 by Delphix. All rights reserved. +# Copyright (c) 2023 by Klara Inc. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib + +# +# DESCRIPTION: +# ZFS can set any valid user-defined pool property. +# +# STRATEGY: +# 1. Combine all kind of valid characters into a valid user-defined +# property name. +# 2. Random get a string as the value. +# 3. Verify all the valid user-defined pool properties can be set to a +# pool. +# + +verify_runnable "both" + +log_assert "ZFS can set any valid user-defined pool property." +log_onexit cleanup_user_prop $TESTPOOL + +typeset -a names=() +typeset -a values=() + +# Longest property name (255 bytes, which is the 256-byte limit minus 1 byte +# for the null byte) +names+=("$(awk 'BEGIN { printf "x:"; while (c++ < (256 - 2 - 1)) printf "a" }')") +values+=("long-property-name") +# Longest property value (the limits are 1024 on FreeBSD and 4096 on Linux, so +# pick the right one; the longest value can use limit minus 1 bytes for the +# null byte) +if is_linux; then + typeset ZFS_MAXPROPLEN=4096 +else + typeset ZFS_MAXPROPLEN=1024 +fi +names+=("long:property:value") +values+=("$(awk -v max="$ZFS_MAXPROPLEN" 'BEGIN { while (c++ < (max - 1)) printf "A" }')") +# Valid property names +for i in {1..10}; do + typeset -i len + ((len = RANDOM % 32)) + names+=("$(valid_user_property $len)") + ((len = RANDOM % 512)) + values+=("$(user_property_value $len)") +done + +typeset -i i=0 +while ((i < ${#names[@]})); do + typeset name="${names[$i]}" + typeset value="${values[$i]}" + + log_must eval "zpool set $name='$value' $TESTPOOL" + log_must eval "check_user_prop $TESTPOOL $name '$value'" + + ((i += 1)) +done + +log_pass "ZFS can set any valid user-defined pool property passed." diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_002_neg.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_002_neg.ksh new file mode 100755 index 000000000000..7c8fcba6e471 --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_set/user_property_002_neg.ksh @@ -0,0 +1,88 @@ +#!/bin/ksh -p +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2007 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2016 by Delphix. All rights reserved. +# Copyright (c) 2023 by Klara Inc. +# + +. $STF_SUITE/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib + +# +# DESCRIPTION: +# ZFS can handle any invalid user-defined pool property. +# +# STRATEGY: +# 1. Combine all kind of invalid user pool property names. +# 2. Random get a string as the value. +# 3. Verify all the invalid user-defined pool properties can not be set +# to the pool. +# + +verify_runnable "both" + +log_assert "ZFS can handle any invalid user pool property." +log_onexit cleanup_user_prop $TESTPOOL + +typeset -a names=() +typeset -a values=() + +# Too long property name (256 bytes, which is the 256-byte limit minus 1 byte +# for the null byte plus 1 byte to reach back over the limit) +names+=("$(awk 'BEGIN { printf "x:"; while (c++ < (256 - 2 - 1 + 1)) printf "a" }')") +values+=("too-long-property-name") +# Too long property value (the limits are 1024 on FreeBSD and 4096 on Linux, so +# pick the right one; the too long value is, e.g., the limit minus 1 bytes for the +# null byte plus 1 byte to reach back over the limit) +if is_linux; then + typeset ZFS_MAXPROPLEN=4096 +else + typeset ZFS_MAXPROPLEN=1024 +fi +names+=("too:long:property:value") +values+=("$(awk -v max="$ZFS_MAXPROPLEN" 'BEGIN { while (c++ < (max - 1 + 1)) printf "A" }')") +# Invalid property names +for i in {1..10}; do + typeset -i len + ((len = RANDOM % 32)) + names+=("$(invalid_user_property $len)") + ((len = RANDOM % 512)) + values+=("$(user_property_value $len)") +done + +typeset -i i=0 +while ((i < ${#names[@]})); do + typeset name="${names[$i]}" + typeset value="${values[$i]}" + + log_mustnot zpool set $name=$value $TESTPOOL + log_mustnot check_user_prop $TESTPOOL \"$name\" \"$value\" + + ((i += 1)) +done + +log_pass "ZFS can handle invalid user pool property passed." diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib b/tests/zfs-tests/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib new file mode 100644 index 000000000000..346e4a16b2ad --- /dev/null +++ b/tests/zfs-tests/tests/functional/cli_root/zpool_set/zpool_set_common.kshlib @@ -0,0 +1,178 @@ +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. +# Use is subject to license terms. +# + +# +# Copyright (c) 2014, 2016 by Delphix. All rights reserved. +# Copyright (c) 2023 by Klara Inc. +# + +. $STF_SUITE/include/libtest.shlib + +set -A VALID_NAME_CHAR a b c d e f g h i j k l m n o p q r s t u v w x y z \ + 0 1 2 3 4 5 6 7 8 9 ':' '-' '.' '_' +set -A INVALID_NAME_CHAR A B C D E F G H I J K L M N O P Q R S T U V W X Y Z \ + '`' '~' '!' '@' '#' '$' '%' '^' '&' '(' ')' '+' '=' '|' "\\" '{' '[' ']' \ + '}' ';' '"' '<' ',' '>' '?' '/' ' ' +set -A ALL_CHAR ${VALID_NAME_CHAR[*]} ${INVALID_NAME_CHAR[*]} + +# +# Cleanup all the user properties of the pool. +# +# $1 pool name +# +function cleanup_user_prop +{ + typeset pool=$1 + + typeset user_prop + user_prop=$(zpool get -H -o property all $pool | grep ":") + + typeset prop + for prop in $user_prop; do + zpool set $prop="" $pool || + log_must zpool set $prop="" $pool + done +} + +# +# Random select character from the specified character set and combine into a +# random string +# +# $1 character set name +# $2 String length +# +function random_string +{ + typeset char_set=${1:-VALID_NAME_CHAR} + typeset -i len=${2:-5} + + eval typeset -i count=\${#$char_set[@]} + + # No consumers want an empty string. + ((len == 0)) && len=3 + + typeset str + typeset -i i=0 + while ((i < len)); do + typeset -i ind + ((ind = RANDOM % count)) + eval str=\${str}\${$char_set[\$ind]} + + ((i += 1)) + done + + echo "$str" +} + +# +# Get valid user-defined property name +# +# $1 user-defined property name length +# +function valid_user_property +{ + typeset -i sumlen=${1:-10} + ((sumlen < 2 )) && sumlen=2 + typeset -i len + ((len = RANDOM % sumlen)) + typeset part1 part2 + + while true; do + part1="$(random_string VALID_NAME_CHAR $len)" + if [[ "$part1" == "-"* ]]; then + continue + fi + break + done + ((len = sumlen - (len + 1))) + + while true; do + part2="$(random_string VALID_NAME_CHAR $len)" + if [[ -z $part1 && -z $part2 ]]; then + continue + fi + break + done + + echo "${part1}:${part2}" +} + +# +# Get invalid user-defined property name +# +# $1 user-defined property name length +# +function invalid_user_property +{ + typeset -i sumlen=${1:-10} + ((sumlen == 0)) && sumlen=1 + typeset -i len + ((len = RANDOM % sumlen)) + + typeset part1 part2 + while true; do + part1="$(random_string VALID_NAME_CHAR $len)" + ((len = sumlen - len)) + part2="$(random_string INVALID_NAME_CHAR $len)" + + # Avoid $part1 is *:* and $part2 is "=*" + if [[ "$part1" == *":"* && "$part2" == "="* ]]; then + continue + fi + break + done + + echo "${part1}${part2}" +} + +# +# Get user-defined property value +# +# $1 user-defined property name length +# +function user_property_value +{ + typeset -i len=${1:-100} + + random_string ALL_CHAR $len +} + +# +# Check if the user-defined property is identical to the expected value. +# +# $1 pool +# $2 user property +# $3 expected value +# +function check_user_prop +{ + typeset pool=$1 + typeset user_prop="$2" + typeset expect_value="$3" + typeset value=$(zpool get -p -H -o value "$user_prop" $pool 2>&1) + + [ "$expect_value" = "$value" ] +} From 62cc9d4f6b5ad3a16b97f6389828b20331863f40 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Fri, 21 Apr 2023 19:22:52 +0200 Subject: [PATCH 39/59] FreeBSD: make zfs_vfs_held() definition consistent with declaration Noticed while attempting to change FreeBSD's boolean_t into an actual bool: in include/sys/zfs_ioctl_impl.h, zfs_vfs_held() is declared to return a boolean_t, but in module/os/freebsd/zfs/zfs_ioctl_os.c it is defined to return an int. Make the definition match the declaration. Reviewed-by: Alexander Motin Reviewed-by: Brian Atkinson Signed-off-by: Dimitry Andric Closes #14776 --- module/os/freebsd/zfs/zfs_ioctl_os.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/os/freebsd/zfs/zfs_ioctl_os.c b/module/os/freebsd/zfs/zfs_ioctl_os.c index 8f44cced5d95..a835e013d630 100644 --- a/module/os/freebsd/zfs/zfs_ioctl_os.c +++ b/module/os/freebsd/zfs/zfs_ioctl_os.c @@ -59,7 +59,7 @@ zfs_vfs_ref(zfsvfs_t **zfvp) return (error); } -int +boolean_t zfs_vfs_held(zfsvfs_t *zfsvfs) { return (zfsvfs->z_vfs != NULL); From a7982d5d30dabd2e206011f54226b25d6c70c4d6 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Wed, 5 Apr 2023 21:12:17 +0000 Subject: [PATCH 40/59] FreeBSD: fix up EXDEV handling for clone_range API contract requires VOPs to handle EXDEV internally, worst case by falling back to the generic copy routine. This broke with the recent changes. While here whack custom loop to lock 2 vnodes with vn_lock_pair, which provides the same functionality internally. write start/finish around it plays no role so got eliminated. One difference is that vn_lock_pair always takes an exclusive lock on both vnodes. I did not patch around it because current code takes an exclusive lock on the target vnode. zfs supports shared-locking for writes, so this serializes different calls to the routine as is, despite range locking inside. At the same time you may notice the source vnode can get some traffic if only shared-locked, thus once more this goes the safer route of exclusive-locking. Note this should be patched to use shared-locking for both once the feature is considered stable. Technically the switch to vn_lock_pair should be a separate change, but it would only introduce churn immediately whacked by the rest of the patch. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Mateusz Guzik Sponsored by: Rubicon Communications, LLC ("Netgate") Closes #14723 --- module/os/freebsd/zfs/zfs_vnops_os.c | 63 +++++++++++++++------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 8abd7239ad2e..0ec4d40ce790 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -6249,56 +6249,59 @@ zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap) * need something else than vn_generic_copy_file_range(). */ - /* Lock both vnodes, avoiding risk of deadlock. */ - do { - mp = NULL; - error = vn_start_write(outvp, &mp, V_WAIT); - if (error == 0) { - error = vn_lock(outvp, LK_EXCLUSIVE); - if (error == 0) { - if (invp == outvp) - break; - error = vn_lock(invp, LK_SHARED | LK_NOWAIT); - if (error == 0) - break; - VOP_UNLOCK(outvp); - if (mp != NULL) - vn_finished_write(mp); - mp = NULL; - error = vn_lock(invp, LK_SHARED); - if (error == 0) - VOP_UNLOCK(invp); - } + vn_start_write(outvp, &mp, V_WAIT); + if (invp == outvp) { + if (vn_lock(outvp, LK_EXCLUSIVE) != 0) { + goto bad_write_fallback; } - if (mp != NULL) - vn_finished_write(mp); - } while (error == 0); - if (error != 0) - return (error); + } else { +#if __FreeBSD_version >= 1400086 + vn_lock_pair(invp, false, LK_EXCLUSIVE, outvp, false, + LK_EXCLUSIVE); +#else + vn_lock_pair(invp, false, outvp, false); +#endif + if (VN_IS_DOOMED(invp) || VN_IS_DOOMED(outvp)) { + goto bad_locked_fallback; + } + } + #ifdef MAC error = mac_vnode_check_write(curthread->td_ucred, ap->a_outcred, outvp); if (error != 0) - goto unlock; + goto out_locked; #endif io.uio_offset = *ap->a_outoffp; io.uio_resid = *ap->a_lenp; error = vn_rlimit_fsize(outvp, &io, ap->a_fsizetd); if (error != 0) - goto unlock; + goto out_locked; error = zfs_clone_range(VTOZ(invp), ap->a_inoffp, VTOZ(outvp), - ap->a_outoffp, &len, ap->a_fsizetd->td_ucred); + ap->a_outoffp, &len, ap->a_outcred); + if (error == EXDEV) + goto bad_locked_fallback; *ap->a_lenp = (size_t)len; - -unlock: +out_locked: if (invp != outvp) VOP_UNLOCK(invp); VOP_UNLOCK(outvp); if (mp != NULL) vn_finished_write(mp); + return (error); +bad_locked_fallback: + if (invp != outvp) + VOP_UNLOCK(invp); + VOP_UNLOCK(outvp); +bad_write_fallback: + if (mp != NULL) + vn_finished_write(mp); + error = vn_generic_copy_file_range(ap->a_invp, ap->a_inoffp, + ap->a_outvp, ap->a_outoffp, ap->a_lenp, ap->a_flags, + ap->a_incred, ap->a_outcred, ap->a_fsizetd); return (error); } From ff0e135e25fa6b523f93d8022304b1be60a67be7 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Wed, 5 Apr 2023 21:28:52 +0000 Subject: [PATCH 41/59] FreeBSD: try to fallback early if can't do optimized copy Not complete, but already shaves on some locking. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Mateusz Guzik Sponsored by: Rubicon Communications, LLC ("Netgate") Closes #14723 --- module/os/freebsd/zfs/zfs_vnops_os.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_vnops_os.c b/module/os/freebsd/zfs/zfs_vnops_os.c index 0ec4d40ce790..d29f00a0cbe4 100644 --- a/module/os/freebsd/zfs/zfs_vnops_os.c +++ b/module/os/freebsd/zfs/zfs_vnops_os.c @@ -6235,6 +6235,7 @@ struct vop_copy_file_range_args { static int zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap) { + zfsvfs_t *outzfsvfs; struct vnode *invp = ap->a_invp; struct vnode *outvp = ap->a_outvp; struct mount *mp; @@ -6250,6 +6251,13 @@ zfs_freebsd_copy_file_range(struct vop_copy_file_range_args *ap) */ vn_start_write(outvp, &mp, V_WAIT); + if (__predict_true(mp == outvp->v_mount)) { + outzfsvfs = (zfsvfs_t *)mp->mnt_data; + if (!spa_feature_is_enabled(dmu_objset_spa(outzfsvfs->z_os), + SPA_FEATURE_BLOCK_CLONING)) { + goto bad_write_fallback; + } + } if (invp == outvp) { if (vn_lock(outvp, LK_EXCLUSIVE) != 0) { goto bad_write_fallback; From 81a2b2e6a6d9c71d314588207de9775a5ea6f72f Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Tue, 25 Apr 2023 01:15:42 +0200 Subject: [PATCH 42/59] FreeBSD: add missing vop_fplookup assignments It became illegal to not have them as of 5f6df177758b9dff88e4b6069aeb2359e8b0c493 ("vfs: validate that vop vectors provide all or none fplookup vops") upstream. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Mateusz Guzik Closes #14788 --- module/os/freebsd/zfs/zfs_ctldir.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c index ca2f4419d1c4..d00efa44f2bc 100644 --- a/module/os/freebsd/zfs/zfs_ctldir.c +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -801,6 +801,9 @@ static struct vop_vector zfsctl_ops_root = { .vop_default = &default_vnodeops, #if __FreeBSD_version >= 1300121 .vop_fplookup_vexec = VOP_EAGAIN, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = VOP_EAGAIN, #endif .vop_open = zfsctl_common_open, .vop_close = zfsctl_common_close, @@ -1129,6 +1132,9 @@ static struct vop_vector zfsctl_ops_snapdir = { .vop_default = &default_vnodeops, #if __FreeBSD_version >= 1300121 .vop_fplookup_vexec = VOP_EAGAIN, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = VOP_EAGAIN, #endif .vop_open = zfsctl_common_open, .vop_close = zfsctl_common_close, @@ -1236,6 +1242,9 @@ static struct vop_vector zfsctl_ops_snapshot = { .vop_default = NULL, /* ensure very restricted access */ #if __FreeBSD_version >= 1300121 .vop_fplookup_vexec = VOP_EAGAIN, +#endif +#if __FreeBSD_version >= 1300139 + .vop_fplookup_symlink = VOP_EAGAIN, #endif .vop_open = zfsctl_common_open, .vop_close = zfsctl_common_close, From 6b6aaf6dc2e65c63c74fbd7840c14627e9a91ce2 Mon Sep 17 00:00:00 2001 From: Rich Ercolani <214141+rincebrain@users.noreply.github.com> Date: Mon, 24 Apr 2023 19:55:07 -0400 Subject: [PATCH 43/59] Taught zdb -bb to print metadata totals People often want estimates of how much of their pool is occupied by metadata, but they end up using lots of text processing on zdb's output to get it. So let's just...provide it for them. Now, zdb -bbbs will output something like: Blocks LSIZE PSIZE ASIZE avg comp %Total Type [...] 68 1.06M 272K 544K 8K 4.00 0.00 L6 Total 1.71K 212M 6.85M 13.7M 8K 30.91 0.00 L5 Total 1.71K 212M 6.85M 13.7M 8K 30.91 0.00 L4 Total 1.73K 214M 6.92M 13.8M 8K 30.89 0.00 L3 Total 18.7K 2.29G 111M 221M 11.8K 21.19 0.00 L2 Total 3.56M 454G 28.4G 56.9G 16.0K 15.97 0.19 L1 Total 308M 36.8T 28.2T 28.6T 95.1K 1.30 99.80 L0 Total 311M 37.3T 28.3T 28.6T 94.2K 1.32 100.00 Total 50.4M 774G 113G 291G 5.77K 6.85 0.99 Metadata Total Reviewed-by: Tino Reichardt Reviewed-by: Brian Behlendorf Signed-off-by: Rich Ercolani Closes #14746 --- cmd/zdb/zdb.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index c93ed4399afd..64ec3eb0028c 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -6812,12 +6812,15 @@ dump_block_stats(spa_t *spa) if (dump_opt['b'] >= 2) { int l, t, level; + char csize[32], lsize[32], psize[32], asize[32]; + char avg[32], gang[32]; (void) printf("\nBlocks\tLSIZE\tPSIZE\tASIZE" "\t avg\t comp\t%%Total\tType\n"); + zfs_blkstat_t *mdstats = umem_zalloc(sizeof (zfs_blkstat_t), + UMEM_NOFAIL); + for (t = 0; t <= ZDB_OT_TOTAL; t++) { - char csize[32], lsize[32], psize[32], asize[32]; - char avg[32], gang[32]; const char *typename; /* make sure nicenum has enough space */ @@ -6860,6 +6863,15 @@ dump_block_stats(spa_t *spa) if (zb->zb_asize == 0) continue; + if (level != ZB_TOTAL && t < DMU_OT_NUMTYPES && + (level > 0 || DMU_OT_IS_METADATA(t))) { + mdstats->zb_count += zb->zb_count; + mdstats->zb_lsize += zb->zb_lsize; + mdstats->zb_psize += zb->zb_psize; + mdstats->zb_asize += zb->zb_asize; + mdstats->zb_gangs += zb->zb_gangs; + } + if (dump_opt['b'] < 3 && level != ZB_TOTAL) continue; @@ -6905,6 +6917,24 @@ dump_block_stats(spa_t *spa) } } } + zdb_nicenum(mdstats->zb_count, csize, + sizeof (csize)); + zdb_nicenum(mdstats->zb_lsize, lsize, + sizeof (lsize)); + zdb_nicenum(mdstats->zb_psize, psize, + sizeof (psize)); + zdb_nicenum(mdstats->zb_asize, asize, + sizeof (asize)); + zdb_nicenum(mdstats->zb_asize / mdstats->zb_count, avg, + sizeof (avg)); + zdb_nicenum(mdstats->zb_gangs, gang, sizeof (gang)); + + (void) printf("%6s\t%5s\t%5s\t%5s\t%5s" + "\t%5.2f\t%6.2f\t", + csize, lsize, psize, asize, avg, + (double)mdstats->zb_lsize / mdstats->zb_psize, + 100.0 * mdstats->zb_asize / tzb->zb_asize); + (void) printf("%s\n", "Metadata Total"); /* Output a table summarizing block sizes in the pool */ if (dump_opt['b'] >= 2) { From 6d59d5df9808902a3cb6064605c753ec2ab8d2d7 Mon Sep 17 00:00:00 2001 From: Han Gao Date: Wed, 26 Apr 2023 07:05:45 +0800 Subject: [PATCH 44/59] Add loongarch64 support Add loongarch64 definitions & lua module setjmp asm LoongArch is a new RISC ISA, which is a bit like MIPS or RISC-V. Reviewed-by: Richard Yao Reviewed-by: Brian Behlendorf Signed-off-by: Han Gao Signed-off-by: WANG Xuerui Closes #13422 --- include/os/linux/spl/sys/isa_defs.h | 18 +++++- lib/libspl/include/sys/isa_defs.h | 18 +++++- module/lua/ldo.c | 2 + module/lua/setjmp/setjmp.S | 2 + module/lua/setjmp/setjmp_loongarch64.S | 82 ++++++++++++++++++++++++++ 5 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 module/lua/setjmp/setjmp_loongarch64.S diff --git a/include/os/linux/spl/sys/isa_defs.h b/include/os/linux/spl/sys/isa_defs.h index 5801ec92bc2b..7c95c94c1cf1 100644 --- a/include/os/linux/spl/sys/isa_defs.h +++ b/include/os/linux/spl/sys/isa_defs.h @@ -195,10 +195,26 @@ #define _SUNOS_VTOC_16 +/* + * LoongArch arch specific defines + * only LoongArch64 is supported yet + */ +#elif defined(__loongarch__) && defined(__loongarch_lp64) + +#if !defined(_LP64) +#define _LP64 +#endif + +#define _ZFS_LITTLE_ENDIAN +#define _SUNOS_VTOC_16 + +/* not all LoongArch cores support unaligned accesses in hardware */ +#define _ALIGNMENT_REQUIRED 1 + #else /* * Currently supported: - * x86_64, x32, i386, arm, powerpc, s390, sparc, mips, and RV64G + * x86_64, x32, i386, arm, powerpc, s390, sparc, mips, RV64G, and LoongArch64 */ #error "Unsupported ISA type" #endif diff --git a/lib/libspl/include/sys/isa_defs.h b/lib/libspl/include/sys/isa_defs.h index 114cca4f1545..302f31e989cb 100644 --- a/lib/libspl/include/sys/isa_defs.h +++ b/lib/libspl/include/sys/isa_defs.h @@ -246,10 +246,26 @@ extern "C" { #define _SUNOS_VTOC_16 +/* + * LoongArch arch specific defines + * only LoongArch64 is supported yet + */ +#elif defined(__loongarch__) && defined(__loongarch_lp64) + +#if !defined(_LP64) +#define _LP64 +#endif + +#define _ZFS_LITTLE_ENDIAN +#define _SUNOS_VTOC_16 + +/* not all LoongArch cores support unaligned accesses in hardware */ +#define _ALIGNMENT_REQUIRED 1 + #else /* * Currently supported: - * x86_64, x32, i386, arm, powerpc, s390, sparc, mips, and RV64G + * x86_64, x32, i386, arm, powerpc, s390, sparc, mips, RV64G, and LoongArch64 */ #error "Unsupported ISA type" #endif diff --git a/module/lua/ldo.c b/module/lua/ldo.c index bf525588e260..38bd4e08a73d 100644 --- a/module/lua/ldo.c +++ b/module/lua/ldo.c @@ -84,6 +84,8 @@ static intptr_t stack_remaining(void) { #define JMP_BUF_CNT 18 #elif defined(__riscv) #define JMP_BUF_CNT 64 +#elif defined(__loongarch_lp64) +#define JMP_BUF_CNT 64 #else #define JMP_BUF_CNT 1 #endif diff --git a/module/lua/setjmp/setjmp.S b/module/lua/setjmp/setjmp.S index 1f461a0a4ef3..6f03eea92711 100644 --- a/module/lua/setjmp/setjmp.S +++ b/module/lua/setjmp/setjmp.S @@ -16,4 +16,6 @@ #include "setjmp_s390x.S" #elif defined(__riscv) #include "setjmp_rv64g.S" +#elif defined(__loongarch_lp64) +#include "setjmp_loongarch64.S" #endif diff --git a/module/lua/setjmp/setjmp_loongarch64.S b/module/lua/setjmp/setjmp_loongarch64.S new file mode 100644 index 000000000000..216b829ff236 --- /dev/null +++ b/module/lua/setjmp/setjmp_loongarch64.S @@ -0,0 +1,82 @@ +/*- + * Copyright 2022 Han Gao + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if __loongarch_lp64 + +#define ENTRY(symbol) \ + .text; \ + .globl symbol; \ + .align 3; \ + .type symbol, @function; \ + symbol: + +#define END(function) \ + .size function, .- function; + +ENTRY(setjmp) + st.d $ra, $a0, 0*8 + st.d $sp, $a0, 1*8 + st.d $r21, $a0, 2*8 + st.d $fp, $a0, 3*8 + st.d $s0, $a0, 4*8 + st.d $s1, $a0, 5*8 + st.d $s2, $a0, 6*8 + st.d $s3, $a0, 7*8 + st.d $s4, $a0, 8*8 + st.d $s5, $a0, 9*8 + st.d $s6, $a0, 10*8 + st.d $s7, $a0, 11*8 + st.d $s8, $a0, 12*8 + + li.w $a0, 0 + jr $ra +END(setjmp) + +ENTRY(longjmp) + ld.d $ra, $a0, 0*8 + ld.d $sp, $a0, 1*8 + ld.d $r21, $a0, 2*8 + ld.d $fp, $a0, 3*8 + ld.d $s0, $a0, 4*8 + ld.d $s1, $a0, 5*8 + ld.d $s2, $a0, 6*8 + ld.d $s3, $a0, 7*8 + ld.d $s4, $a0, 8*8 + ld.d $s5, $a0, 9*8 + ld.d $s6, $a0, 10*8 + ld.d $s7, $a0, 11*8 + ld.d $s8, $a0, 12*8 + + sltui $a0, $a1, 1 + add.d $a0, $a0, $a1 // a0 = (a1 == 0) ? 1 : a1 + jr $ra +END(longjmp) + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif + +#endif From 0e8a42bbee2c29f1a2c49fdbda403e839af4b38d Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 25 Apr 2023 16:40:55 -0700 Subject: [PATCH 45/59] Revert "Fix data race between zil_commit() and zil_suspend()" This reverts commit 4c856fb333ac57d9b4a6ddd44407fd022a702f00 to resolve a newly introduced deadlock which in practice in more disruptive that the issue this commit intended to address. Reviewed-by: Richard Yao Reviewed-by: Mark Maybee Signed-off-by: Brian Behlendorf Issue #14775 Closes #14790 --- include/sys/zil_impl.h | 1 - module/zfs/zil.c | 27 --------------------------- 2 files changed, 28 deletions(-) diff --git a/include/sys/zil_impl.h b/include/sys/zil_impl.h index f44a01afee5c..bb85bf6d1eb1 100644 --- a/include/sys/zil_impl.h +++ b/include/sys/zil_impl.h @@ -183,7 +183,6 @@ struct zilog { uint64_t zl_destroy_txg; /* txg of last zil_destroy() */ uint64_t zl_replayed_seq[TXG_SIZE]; /* last replayed rec seq */ uint64_t zl_replaying_seq; /* current replay seq number */ - krwlock_t zl_suspend_lock; /* protects suspend count */ uint32_t zl_suspend; /* log suspend count */ kcondvar_t zl_cv_suspend; /* log suspend completion */ uint8_t zl_suspending; /* log is currently suspending */ diff --git a/module/zfs/zil.c b/module/zfs/zil.c index eb26e4b32998..d1631c2ac9db 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -3317,21 +3317,6 @@ zil_commit(zilog_t *zilog, uint64_t foid) return; } - /* - * The ->zl_suspend_lock rwlock ensures that all in-flight - * zil_commit() operations finish before suspension begins and that - * no more begin. Without it, it is possible for the scheduler to - * preempt us right after the zilog->zl_suspend suspend check, run - * another thread that runs zil_suspend() and after the other thread - * has finished its call to zil_commit_impl(), resume this thread while - * zil is suspended. This can trigger an assertion failure in - * VERIFY(list_is_empty(&lwb->lwb_itxs)). If it is held, it means that - * `zil_suspend()` is executing in another thread, so we go to - * txg_wait_synced(). - */ - if (!rw_tryenter(&zilog->zl_suspend_lock, RW_READER)) - goto wait; - /* * If the ZIL is suspended, we don't want to dirty it by calling * zil_commit_itx_assign() below, nor can we write out @@ -3340,14 +3325,11 @@ zil_commit(zilog_t *zilog, uint64_t foid) * semantics, and avoid calling those functions altogether. */ if (zilog->zl_suspend > 0) { - rw_exit(&zilog->zl_suspend_lock); -wait: txg_wait_synced(zilog->zl_dmu_pool, 0); return; } zil_commit_impl(zilog, foid); - rw_exit(&zilog->zl_suspend_lock); } void @@ -3612,8 +3594,6 @@ zil_alloc(objset_t *os, zil_header_t *zh_phys) cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL); cv_init(&zilog->zl_lwb_io_cv, NULL, CV_DEFAULT, NULL); - rw_init(&zilog->zl_suspend_lock, NULL, RW_DEFAULT, NULL); - return (zilog); } @@ -3653,8 +3633,6 @@ zil_free(zilog_t *zilog) cv_destroy(&zilog->zl_cv_suspend); cv_destroy(&zilog->zl_lwb_io_cv); - rw_destroy(&zilog->zl_suspend_lock); - kmem_free(zilog, sizeof (zilog_t)); } @@ -3782,14 +3760,11 @@ zil_suspend(const char *osname, void **cookiep) return (error); zilog = dmu_objset_zil(os); - rw_enter(&zilog->zl_suspend_lock, RW_WRITER); - mutex_enter(&zilog->zl_lock); zh = zilog->zl_header; if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */ mutex_exit(&zilog->zl_lock); - rw_exit(&zilog->zl_suspend_lock); dmu_objset_rele(os, suspend_tag); return (SET_ERROR(EBUSY)); } @@ -3803,7 +3778,6 @@ zil_suspend(const char *osname, void **cookiep) if (cookiep == NULL && !zilog->zl_suspending && (zilog->zl_suspend > 0 || BP_IS_HOLE(&zh->zh_log))) { mutex_exit(&zilog->zl_lock); - rw_exit(&zilog->zl_suspend_lock); dmu_objset_rele(os, suspend_tag); return (0); } @@ -3812,7 +3786,6 @@ zil_suspend(const char *osname, void **cookiep) dsl_pool_rele(dmu_objset_pool(os), suspend_tag); zilog->zl_suspend++; - rw_exit(&zilog->zl_suspend_lock); if (zilog->zl_suspend > 1) { /* From d960beca61b3c5ede8eb595ceb091666e0d4a17c Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 26 Apr 2023 08:43:39 -0700 Subject: [PATCH 46/59] zdb: Fix minor memory leak Commit 6b6aaf6dc2e65c63c74fbd7840c14627e9a91ce2 introduced a small memory leak in zdb. This was detected by the LeakSanitizer and was causing all ztest runs to fail. Reviewed-by: Igor Kozhukhov Reviewed-by: Rich Ercolani Signed-off-by: Brian Behlendorf Closes #14796 --- cmd/zdb/zdb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index 64ec3eb0028c..ec5d1acacf85 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -6940,6 +6940,8 @@ dump_block_stats(spa_t *spa) if (dump_opt['b'] >= 2) { dump_size_histograms(zcb); } + + umem_free(mdstats, sizeof (zfs_blkstat_t)); } (void) printf("\n"); From b69cb06664ad74f5907f3a6675657b7d4b3daa44 Mon Sep 17 00:00:00 2001 From: Rob N Date: Thu, 27 Apr 2023 01:50:44 +1000 Subject: [PATCH 47/59] tests/zdb_encrypted: parse numbers a little more robustly On FreeBSD, `wc` prints some leading spaces, while on Linux it does not. So we tell ksh to expect an integer, and it does the rest. Reviewed-by: Brian Behlendorf Signed-off-by: Rob Norris Closes #14791 Closes #14797 --- .../zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted.ksh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted.ksh b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted.ksh index 4572f64947a1..0218c2ea1033 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zdb/zdb_encrypted.ksh @@ -55,8 +55,8 @@ log_must eval "echo $PASSPHRASE | zfs create -o mountpoint=$TESTDIR2" \ echo 'my great encrypted text' > $file -obj="$(ls -i $file | cut -d' ' -f1)" -size="$(wc -c < $file)" +typeset -i obj=$(ls -i $file | cut -d' ' -f1) +typeset -i size=$(wc -c < $file) log_note "test file $file is objid $obj, size $size" From 88b8777159f69237ca59b31e8c246c567e836c68 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Thu, 20 Apr 2023 08:59:38 +0000 Subject: [PATCH 48/59] FreeBSD: add missing vn state transition for .zfs Signed-off-by: Mateusz Guzik Closes #14774 --- module/os/freebsd/zfs/zfs_ctldir.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c index d00efa44f2bc..8ba5e0b242a2 100644 --- a/module/os/freebsd/zfs/zfs_ctldir.c +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -204,6 +204,10 @@ sfs_vgetx(struct mount *mp, int flags, uint64_t parent_id, uint64_t id, return (error); } +#if __FreeBSD_version >= 1400077 + vn_set_state(vp, VSTATE_CONSTRUCTED); +#endif + *vpp = vp; return (0); } From e37a89d5d0ba07da09998de6701a6c0ec43903b7 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Thu, 20 Apr 2023 09:00:03 +0000 Subject: [PATCH 49/59] FreeBSD: fix up EINVAL from getdirentries on .zfs Without the change: /.zfs /.zfs/snapshot find: /.zfs: Invalid argument Signed-off-by: Mateusz Guzik Closes #14774 --- module/os/freebsd/zfs/zfs_ctldir.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c index 8ba5e0b242a2..420d887b661e 100644 --- a/module/os/freebsd/zfs/zfs_ctldir.c +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -679,6 +679,17 @@ zfsctl_root_readdir(struct vop_readdir_args *ap) ASSERT3S(vp->v_type, ==, VDIR); + /* + * FIXME: this routine only ever emits 3 entries and does not tolerate + * being called with a buffer too small to handle all of them. + * + * The check below facilitates the idiom of repeating calls until the + * count to return is 0. + */ + if (zfs_uio_offset(&uio) == 3 * sizeof(entry)) { + return (0); + } + error = sfs_readdir_common(zfsvfs->z_root, ZFSCTL_INO_ROOT, ap, &uio, &dots_offset); if (error != 0) { From bba7cbf0a481ab16f9a9a4874b7dbd5682e4d3a4 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Wed, 26 Apr 2023 12:20:43 -0400 Subject: [PATCH 50/59] Fix positive ABD size assertion in abd_verify(). Gang ABDs without childred are legal, and they do have zero size. For other ABD types zero size doesn't have much sense and likely not working correctly now. Reviewed-by: Igor Kozhukhov Reviewed-by: Brian Behlendorf Signed-off-by: Alexander Motin Sponsored by: iXsystems, Inc. Closes #14795 --- module/zfs/abd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/zfs/abd.c b/module/zfs/abd.c index d4921d0ba7db..26222d2efe3f 100644 --- a/module/zfs/abd.c +++ b/module/zfs/abd.c @@ -109,7 +109,6 @@ void abd_verify(abd_t *abd) { #ifdef ZFS_DEBUG - ASSERT3U(abd->abd_size, >, 0); ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE); ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR | ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE | @@ -118,6 +117,7 @@ abd_verify(abd_t *abd) IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER)); IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER); if (abd_is_linear(abd)) { + ASSERT3U(abd->abd_size, >, 0); ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL); } else if (abd_is_gang(abd)) { uint_t child_sizes = 0; @@ -130,6 +130,7 @@ abd_verify(abd_t *abd) } ASSERT3U(abd->abd_size, ==, child_sizes); } else { + ASSERT3U(abd->abd_size, >, 0); abd_verify_scatter(abd); } #endif From b5411618f727c4ce5f787bb97d1c87f20c66027a Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 26 Apr 2023 11:49:16 -0700 Subject: [PATCH 51/59] Fix checkstyle warning Resolve a missed checkstyle warning. Reviewed-by: Alexander Motin Reviewed-by: Mateusz Guzik Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #14799 --- module/os/freebsd/zfs/zfs_ctldir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/os/freebsd/zfs/zfs_ctldir.c b/module/os/freebsd/zfs/zfs_ctldir.c index 420d887b661e..28445a18b809 100644 --- a/module/os/freebsd/zfs/zfs_ctldir.c +++ b/module/os/freebsd/zfs/zfs_ctldir.c @@ -686,7 +686,7 @@ zfsctl_root_readdir(struct vop_readdir_args *ap) * The check below facilitates the idiom of repeating calls until the * count to return is 0. */ - if (zfs_uio_offset(&uio) == 3 * sizeof(entry)) { + if (zfs_uio_offset(&uio) == 3 * sizeof (entry)) { return (0); } From ee728008a4279dbbbe5332f8b9a886f3b8d91e00 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Wed, 26 Apr 2023 21:40:26 +0200 Subject: [PATCH 52/59] Fix BLAKE3 aarch64 assembly for FreeBSD and macOS The x18 register isn't useable within FreeBSD kernel space, so we have to fix the BLAKE3 aarch64 assembly for not using it. The source files are here: https://github.com/mcmilk/BLAKE3-tests Reviewed-by: Kyle Evans Signed-off-by: Tino Reichardt Closes #14728 --- .../icp/asm-aarch64/blake3/b3_aarch64_sse2.S | 4193 +++++++-------- .../icp/asm-aarch64/blake3/b3_aarch64_sse41.S | 4523 ++++++++--------- 2 files changed, 4131 insertions(+), 4585 deletions(-) diff --git a/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S b/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S index 8237f0eb5a4e..dc2719d142db 100644 --- a/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S +++ b/module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S @@ -22,480 +22,61 @@ /* * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3 * Copyright (c) 2019-2022 Samuel Neves and Matthew Krupcale - * Copyright (c) 2022 Tino Reichardt + * Copyright (c) 2022-2023 Tino Reichardt * * This is converted assembly: SSE2 -> ARMv8-A * Used tools: SIMDe https://github.com/simd-everywhere/simde + * + * Should work on FreeBSD, Linux and macOS + * see: https://github.com/mcmilk/BLAKE3-tests/blob/master/contrib/simde.sh */ #if defined(__aarch64__) .text - .section .rodata.cst16,"aM",@progbits,16 - .p2align 4 -.LCPI0_0: - .word 1779033703 - .word 3144134277 - .word 1013904242 - .word 2773480762 -.LCPI0_1: - .xword 0 - .xword -4294967296 -.LCPI0_2: - .xword -1 - .xword 4294967295 + .section .note.gnu.property,"a",@note + .p2align 3 + .word 4 + .word 16 + .word 5 + .asciz "GNU" + .word 3221225472 + .word 4 + .word 3 + .word 0 +.Lsec_end0: .text .globl zfs_blake3_compress_in_place_sse2 .p2align 2 .type zfs_blake3_compress_in_place_sse2,@function zfs_blake3_compress_in_place_sse2: .cfi_startproc - ldp q3, q2, [x0] - ldp q5, q6, [x1] - add x10, x1, #32 - lsr x11, x3, #32 - fmov s4, w3 - ld2 { v17.4s, v18.4s }, [x10] - adrp x10, .LCPI0_2 - and w8, w2, #0xff - mov v4.s[1], w11 - ldr q1, [x10, :lo12:.LCPI0_2] - and w9, w4, #0xff - adrp x12, .LCPI0_0 - mov v4.s[2], w8 - uzp1 v19.4s, v5.4s, v6.4s - add v3.4s, v2.4s, v3.4s - ldr q7, [x12, :lo12:.LCPI0_0] - mov v4.s[3], w9 - add v3.4s, v3.4s, v19.4s - uzp2 v5.4s, v5.4s, v6.4s - ext v21.16b, v18.16b, v18.16b, #12 - uzp1 v6.4s, v19.4s, v19.4s - ext v22.16b, v19.16b, v19.16b, #12 - eor v4.16b, v3.16b, v4.16b - ext v20.16b, v17.16b, v17.16b, #12 - ext v6.16b, v6.16b, v19.16b, #8 - ext v19.16b, v19.16b, v22.16b, #12 - zip1 v22.2d, v21.2d, v5.2d - rev32 v24.8h, v4.8h - mov v4.16b, v1.16b - zip2 v23.4s, v5.4s, v21.4s - uzp2 v6.4s, v6.4s, v5.4s - bsl v4.16b, v22.16b, v20.16b - add v3.4s, v3.4s, v5.4s - zip1 v5.4s, v23.4s, v20.4s - zip1 v22.4s, v20.4s, v23.4s - add v23.4s, v24.4s, v7.4s - ext v7.16b, v6.16b, v6.16b, #4 - ext v25.16b, v4.16b, v4.16b, #12 - ext v5.16b, v22.16b, v5.16b, #8 - eor v2.16b, v23.16b, v2.16b - uzp1 v4.4s, v4.4s, v25.4s - uzp1 v22.4s, v7.4s, v7.4s - ext v25.16b, v7.16b, v7.16b, #12 - ext v22.16b, v22.16b, v7.16b, #8 - ext v7.16b, v7.16b, v25.16b, #12 - ushr v25.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - orr v2.16b, v2.16b, v25.16b - add v3.4s, v3.4s, v2.4s - eor v24.16b, v3.16b, v24.16b - add v3.4s, v3.4s, v17.4s - ushr v17.4s, v24.4s, #8 - shl v18.4s, v24.4s, #24 - orr v17.16b, v18.16b, v17.16b - add v18.4s, v17.4s, v23.4s - eor v2.16b, v18.16b, v2.16b - ushr v23.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - ext v3.16b, v3.16b, v3.16b, #12 - orr v2.16b, v2.16b, v23.16b - ext v17.16b, v17.16b, v17.16b, #8 - add v3.4s, v2.4s, v3.4s - adrp x11, .LCPI0_1 - eor v17.16b, v3.16b, v17.16b - ldr q16, [x11, :lo12:.LCPI0_1] - ext v18.16b, v18.16b, v18.16b, #4 - rev32 v24.8h, v17.8h - movi v0.2d, #0xffffffff00000000 - add v23.4s, v3.4s, v21.4s - mov v21.s[1], v20.s[2] - add v20.4s, v18.4s, v24.4s - bit v19.16b, v21.16b, v0.16b - eor v3.16b, v20.16b, v2.16b - uzp2 v2.4s, v22.4s, v19.4s - zip1 v17.2d, v5.2d, v19.2d - zip2 v18.4s, v19.4s, v5.4s - ushr v21.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - ext v22.16b, v2.16b, v2.16b, #4 - bsl v16.16b, v4.16b, v17.16b - zip1 v17.4s, v18.4s, v4.4s - zip1 v18.4s, v4.4s, v18.4s - orr v21.16b, v3.16b, v21.16b - ext v25.16b, v16.16b, v16.16b, #12 - ext v3.16b, v18.16b, v17.16b, #8 - uzp1 v18.4s, v22.4s, v22.4s - ext v26.16b, v22.16b, v22.16b, #12 - add v23.4s, v23.4s, v21.4s - uzp1 v17.4s, v16.4s, v25.4s - ext v16.16b, v18.16b, v22.16b, #8 - ext v18.16b, v22.16b, v26.16b, #12 - eor v22.16b, v23.16b, v24.16b - add v6.4s, v23.4s, v6.4s - ushr v23.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - orr v22.16b, v22.16b, v23.16b - add v20.4s, v22.4s, v20.4s - eor v21.16b, v20.16b, v21.16b - ushr v23.4s, v21.4s, #7 - shl v21.4s, v21.4s, #25 - ext v6.16b, v6.16b, v6.16b, #4 - orr v21.16b, v21.16b, v23.16b - ext v22.16b, v22.16b, v22.16b, #8 - add v6.4s, v21.4s, v6.4s - eor v22.16b, v6.16b, v22.16b - ext v20.16b, v20.16b, v20.16b, #12 - add v6.4s, v6.4s, v19.4s - rev32 v19.8h, v22.8h - add v20.4s, v20.4s, v19.4s - eor v21.16b, v20.16b, v21.16b - ushr v22.4s, v21.4s, #12 - shl v21.4s, v21.4s, #20 - orr v21.16b, v21.16b, v22.16b - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - ushr v22.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v19.16b, v19.16b, v22.16b - add v20.4s, v19.4s, v20.4s - eor v21.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #12 - ushr v22.4s, v21.4s, #7 - shl v21.4s, v21.4s, #25 - add v6.4s, v6.4s, v4.4s - orr v21.16b, v21.16b, v22.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - ext v20.16b, v20.16b, v20.16b, #4 - rev32 v19.8h, v19.8h - add v20.4s, v20.4s, v19.4s - add v6.4s, v6.4s, v5.4s - mov v5.s[1], v4.s[2] - eor v4.16b, v20.16b, v21.16b - ushr v21.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - orr v21.16b, v4.16b, v21.16b - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - add v2.4s, v6.4s, v2.4s - ushr v6.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v6.16b, v19.16b, v6.16b - add v19.4s, v6.4s, v20.4s - eor v20.16b, v19.16b, v21.16b - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - orr v20.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v2.4s, v20.4s, v2.4s - eor v6.16b, v2.16b, v6.16b - ext v19.16b, v19.16b, v19.16b, #12 - rev32 v6.8h, v6.8h - add v19.4s, v19.4s, v6.4s - mov v22.16b, v0.16b - eor v20.16b, v19.16b, v20.16b - bsl v22.16b, v5.16b, v7.16b - ushr v21.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - add v2.4s, v2.4s, v22.4s - orr v20.16b, v20.16b, v21.16b - add v2.4s, v2.4s, v20.4s - eor v6.16b, v2.16b, v6.16b - ushr v21.4s, v6.4s, #8 - shl v6.4s, v6.4s, #24 - orr v6.16b, v6.16b, v21.16b - add v19.4s, v6.4s, v19.4s - eor v20.16b, v19.16b, v20.16b - ext v2.16b, v2.16b, v2.16b, #12 - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - add v2.4s, v2.4s, v17.4s - orr v20.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v2.4s, v2.4s, v20.4s - eor v6.16b, v2.16b, v6.16b - uzp2 v5.4s, v16.4s, v22.4s - zip1 v7.2d, v3.2d, v22.2d - zip2 v16.4s, v22.4s, v3.4s - ext v19.16b, v19.16b, v19.16b, #4 - rev32 v22.8h, v6.8h - ext v23.16b, v5.16b, v5.16b, #4 - bif v7.16b, v17.16b, v1.16b - zip1 v24.4s, v16.4s, v17.4s - zip1 v16.4s, v17.4s, v16.4s - add v21.4s, v2.4s, v3.4s - mov v3.s[1], v17.s[2] - add v17.4s, v19.4s, v22.4s - mov v19.16b, v0.16b - ext v25.16b, v7.16b, v7.16b, #12 - ext v4.16b, v16.16b, v24.16b, #8 - uzp1 v16.4s, v23.4s, v23.4s - bsl v19.16b, v3.16b, v18.16b - eor v2.16b, v17.16b, v20.16b - uzp1 v7.4s, v7.4s, v25.4s - ext v25.16b, v16.16b, v23.16b, #8 - zip1 v3.2d, v4.2d, v19.2d - ushr v20.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - ext v24.16b, v23.16b, v23.16b, #12 - uzp2 v6.4s, v25.4s, v19.4s - zip2 v18.4s, v19.4s, v4.4s - bif v3.16b, v7.16b, v1.16b - orr v20.16b, v2.16b, v20.16b - ext v16.16b, v23.16b, v24.16b, #12 - ext v23.16b, v6.16b, v6.16b, #4 - zip1 v24.4s, v18.4s, v7.4s - zip1 v18.4s, v7.4s, v18.4s - ext v25.16b, v3.16b, v3.16b, #12 - add v21.4s, v21.4s, v20.4s - ext v2.16b, v18.16b, v24.16b, #8 - uzp1 v18.4s, v23.4s, v23.4s - ext v24.16b, v23.16b, v23.16b, #12 - uzp1 v3.4s, v3.4s, v25.4s - eor v22.16b, v21.16b, v22.16b - ext v25.16b, v18.16b, v23.16b, #8 - dup v18.4s, v2.s[3] - ext v23.16b, v23.16b, v24.16b, #12 - add v5.4s, v21.4s, v5.4s - trn1 v21.4s, v3.4s, v3.4s - ushr v24.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - ext v18.16b, v21.16b, v18.16b, #8 - orr v21.16b, v22.16b, v24.16b - add v17.4s, v21.4s, v17.4s - eor v20.16b, v17.16b, v20.16b - ushr v22.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - ext v5.16b, v5.16b, v5.16b, #4 - orr v20.16b, v20.16b, v22.16b - ext v21.16b, v21.16b, v21.16b, #8 - add v5.4s, v20.4s, v5.4s - eor v21.16b, v5.16b, v21.16b - ext v17.16b, v17.16b, v17.16b, #12 - add v5.4s, v5.4s, v19.4s - rev32 v19.8h, v21.8h - add v17.4s, v17.4s, v19.4s - eor v20.16b, v17.16b, v20.16b - ushr v21.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - orr v20.16b, v20.16b, v21.16b - add v5.4s, v5.4s, v20.4s - eor v19.16b, v5.16b, v19.16b - ushr v21.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v19.16b, v19.16b, v21.16b - add v17.4s, v19.4s, v17.4s - eor v20.16b, v17.16b, v20.16b - ext v5.16b, v5.16b, v5.16b, #12 - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - add v5.4s, v5.4s, v7.4s - orr v20.16b, v20.16b, v21.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v5.4s, v5.4s, v20.4s - eor v19.16b, v5.16b, v19.16b - ext v17.16b, v17.16b, v17.16b, #4 - rev32 v22.8h, v19.8h - add v21.4s, v5.4s, v4.4s - mov v4.s[1], v7.s[2] - add v19.4s, v17.4s, v22.4s - bit v16.16b, v4.16b, v0.16b - eor v5.16b, v19.16b, v20.16b - uzp2 v4.4s, v25.4s, v16.4s - zip1 v7.2d, v2.2d, v16.2d - zip2 v17.4s, v16.4s, v2.4s - ushr v20.4s, v5.4s, #12 - shl v5.4s, v5.4s, #20 - ext v24.16b, v4.16b, v4.16b, #4 - bif v7.16b, v3.16b, v1.16b - zip1 v25.4s, v17.4s, v3.4s - zip1 v17.4s, v3.4s, v17.4s - orr v20.16b, v5.16b, v20.16b - ext v26.16b, v7.16b, v7.16b, #12 - ext v5.16b, v17.16b, v25.16b, #8 - uzp1 v17.4s, v24.4s, v24.4s - ext v25.16b, v24.16b, v24.16b, #12 - bit v23.16b, v18.16b, v0.16b - add v21.4s, v21.4s, v20.4s - uzp1 v7.4s, v7.4s, v26.4s - ext v26.16b, v17.16b, v24.16b, #8 - ext v17.16b, v24.16b, v25.16b, #12 - eor v22.16b, v21.16b, v22.16b - add v6.4s, v21.4s, v6.4s - zip1 v21.2d, v5.2d, v23.2d - zip2 v24.4s, v23.4s, v5.4s - bif v21.16b, v7.16b, v1.16b - zip1 v1.4s, v24.4s, v7.4s - zip1 v24.4s, v7.4s, v24.4s - ext v1.16b, v24.16b, v1.16b, #8 - ushr v24.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - orr v22.16b, v22.16b, v24.16b - add v19.4s, v22.4s, v19.4s - ext v24.16b, v21.16b, v21.16b, #12 - eor v20.16b, v19.16b, v20.16b - uzp1 v21.4s, v21.4s, v24.4s - ushr v24.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - orr v20.16b, v20.16b, v24.16b - ext v6.16b, v6.16b, v6.16b, #4 - ext v22.16b, v22.16b, v22.16b, #8 - add v6.4s, v20.4s, v6.4s - eor v22.16b, v6.16b, v22.16b - ext v19.16b, v19.16b, v19.16b, #12 - add v6.4s, v6.4s, v16.4s - rev32 v16.8h, v22.8h - add v19.4s, v19.4s, v16.4s - eor v20.16b, v19.16b, v20.16b - ushr v22.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - orr v20.16b, v20.16b, v22.16b - add v6.4s, v6.4s, v20.4s - eor v16.16b, v6.16b, v16.16b - ext v6.16b, v6.16b, v6.16b, #12 - add v3.4s, v6.4s, v3.4s - ushr v6.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - orr v6.16b, v16.16b, v6.16b - add v16.4s, v6.4s, v19.4s - eor v19.16b, v16.16b, v20.16b - ushr v20.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v19.16b, v19.16b, v20.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v3.4s, v3.4s, v19.4s - eor v6.16b, v3.16b, v6.16b - ext v16.16b, v16.16b, v16.16b, #4 - add v2.4s, v3.4s, v2.4s - rev32 v3.8h, v6.8h - add v6.4s, v16.4s, v3.4s - eor v16.16b, v6.16b, v19.16b - ushr v19.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - orr v16.16b, v16.16b, v19.16b - add v2.4s, v2.4s, v16.4s - eor v3.16b, v2.16b, v3.16b - add v2.4s, v2.4s, v4.4s - ushr v4.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v3.16b, v3.16b, v4.16b - add v4.4s, v3.4s, v6.4s - eor v6.16b, v4.16b, v16.16b - ushr v16.4s, v6.4s, #7 - shl v6.4s, v6.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - orr v6.16b, v6.16b, v16.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v6.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #12 - rev32 v3.8h, v3.8h - add v4.4s, v4.4s, v3.4s - eor v6.16b, v4.16b, v6.16b - ushr v16.4s, v6.4s, #12 - shl v6.4s, v6.4s, #20 - add v2.4s, v2.4s, v23.4s - orr v6.16b, v6.16b, v16.16b - add v2.4s, v2.4s, v6.4s - eor v3.16b, v2.16b, v3.16b - ushr v16.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v3.16b, v3.16b, v16.16b - add v4.4s, v3.4s, v4.4s - eor v6.16b, v4.16b, v6.16b - ext v2.16b, v2.16b, v2.16b, #12 - ushr v16.4s, v6.4s, #7 - shl v6.4s, v6.4s, #25 - add v2.4s, v2.4s, v7.4s - orr v6.16b, v6.16b, v16.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v2.4s, v6.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #4 - rev32 v3.8h, v3.8h - add v2.4s, v2.4s, v5.4s - mov v5.s[1], v7.s[2] - add v4.4s, v4.4s, v3.4s - bsl v0.16b, v5.16b, v17.16b - eor v5.16b, v4.16b, v6.16b - ushr v6.4s, v5.4s, #12 - shl v5.4s, v5.4s, #20 - orr v5.16b, v5.16b, v6.16b - add v2.4s, v2.4s, v5.4s - eor v3.16b, v2.16b, v3.16b - ushr v6.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v3.16b, v3.16b, v6.16b - add v4.4s, v3.4s, v4.4s - uzp2 v18.4s, v26.4s, v18.4s - eor v5.16b, v4.16b, v5.16b - add v2.4s, v2.4s, v18.4s - ushr v6.4s, v5.4s, #7 - shl v5.4s, v5.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - orr v5.16b, v5.16b, v6.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v5.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #12 - add v0.4s, v2.4s, v0.4s - rev32 v2.8h, v3.8h - add v3.4s, v4.4s, v2.4s - eor v4.16b, v3.16b, v5.16b - ushr v5.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - orr v4.16b, v4.16b, v5.16b - add v0.4s, v0.4s, v4.4s - eor v2.16b, v0.16b, v2.16b - ushr v5.4s, v2.4s, #8 - shl v2.4s, v2.4s, #24 - orr v2.16b, v2.16b, v5.16b - add v3.4s, v2.4s, v3.4s - eor v4.16b, v3.16b, v4.16b - ext v0.16b, v0.16b, v0.16b, #12 - ushr v5.4s, v4.4s, #7 - shl v4.4s, v4.4s, #25 - add v0.4s, v0.4s, v21.4s - orr v4.16b, v4.16b, v5.16b - ext v2.16b, v2.16b, v2.16b, #8 - add v0.4s, v0.4s, v4.4s - eor v2.16b, v0.16b, v2.16b - ext v3.16b, v3.16b, v3.16b, #4 - add v0.4s, v0.4s, v1.4s - rev32 v1.8h, v2.8h - add v2.4s, v3.4s, v1.4s - eor v3.16b, v2.16b, v4.16b - ushr v4.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - orr v3.16b, v3.16b, v4.16b - add v0.4s, v0.4s, v3.4s - eor v1.16b, v0.16b, v1.16b - ushr v4.4s, v1.4s, #8 - shl v1.4s, v1.4s, #24 - orr v1.16b, v1.16b, v4.16b - add v2.4s, v1.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ext v0.16b, v0.16b, v0.16b, #4 - ext v2.16b, v2.16b, v2.16b, #12 - ushr v4.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - ext v1.16b, v1.16b, v1.16b, #8 + hint #25 + .cfi_negate_ra_state + sub sp, sp, #96 + stp x29, x30, [sp, #64] + add x29, sp, #64 + str x19, [sp, #80] + .cfi_def_cfa w29, 32 + .cfi_offset w19, -16 + .cfi_offset w30, -24 + .cfi_offset w29, -32 + mov x19, x0 + mov w5, w4 + mov x4, x3 + mov w3, w2 + mov x2, x1 + mov x0, sp + mov x1, x19 + bl compress_pre + ldp q0, q1, [sp] + ldp q2, q3, [sp, #32] eor v0.16b, v2.16b, v0.16b - orr v2.16b, v3.16b, v4.16b - eor v1.16b, v2.16b, v1.16b - stp q0, q1, [x0] + eor v1.16b, v3.16b, v1.16b + ldp x29, x30, [sp, #64] + stp q0, q1, [x19] + ldr x19, [sp, #80] + add sp, sp, #96 + hint #29 ret .Lfunc_end0: .size zfs_blake3_compress_in_place_sse2, .Lfunc_end0-zfs_blake3_compress_in_place_sse2 @@ -504,403 +85,39 @@ zfs_blake3_compress_in_place_sse2: .section .rodata.cst16,"aM",@progbits,16 .p2align 4 .LCPI1_0: - .word 1779033703 - .word 3144134277 - .word 1013904242 - .word 2773480762 -.LCPI1_1: - .xword 0 - .xword -4294967296 -.LCPI1_2: - .xword -1 - .xword 4294967295 + .xword -4942790177982912921 + .xword -6534734903820487822 .text - .globl zfs_blake3_compress_xof_sse2 .p2align 2 - .type zfs_blake3_compress_xof_sse2,@function -zfs_blake3_compress_xof_sse2: + .type compress_pre,@function +compress_pre: .cfi_startproc - ldp q3, q2, [x0] - ldp q5, q6, [x1] - add x10, x1, #32 - lsr x11, x3, #32 - fmov s4, w3 - ld2 { v17.4s, v18.4s }, [x10] - adrp x10, .LCPI1_2 - and w8, w2, #0xff - mov v4.s[1], w11 - ldr q1, [x10, :lo12:.LCPI1_2] - and w9, w4, #0xff - adrp x12, .LCPI1_0 - mov v4.s[2], w8 - uzp1 v19.4s, v5.4s, v6.4s - add v3.4s, v2.4s, v3.4s - ldr q7, [x12, :lo12:.LCPI1_0] - mov v4.s[3], w9 - add v3.4s, v3.4s, v19.4s - uzp2 v5.4s, v5.4s, v6.4s - ext v21.16b, v18.16b, v18.16b, #12 - uzp1 v6.4s, v19.4s, v19.4s - ext v22.16b, v19.16b, v19.16b, #12 - eor v4.16b, v3.16b, v4.16b - ext v20.16b, v17.16b, v17.16b, #12 - ext v6.16b, v6.16b, v19.16b, #8 - ext v19.16b, v19.16b, v22.16b, #12 - zip1 v22.2d, v21.2d, v5.2d - rev32 v24.8h, v4.8h - mov v4.16b, v1.16b - zip2 v23.4s, v5.4s, v21.4s - uzp2 v6.4s, v6.4s, v5.4s - bsl v4.16b, v22.16b, v20.16b - add v3.4s, v3.4s, v5.4s - zip1 v5.4s, v23.4s, v20.4s - zip1 v22.4s, v20.4s, v23.4s - add v23.4s, v24.4s, v7.4s - ext v7.16b, v6.16b, v6.16b, #4 - ext v25.16b, v4.16b, v4.16b, #12 - ext v5.16b, v22.16b, v5.16b, #8 - eor v2.16b, v23.16b, v2.16b - uzp1 v4.4s, v4.4s, v25.4s - uzp1 v22.4s, v7.4s, v7.4s - ext v25.16b, v7.16b, v7.16b, #12 - ext v22.16b, v22.16b, v7.16b, #8 - ext v7.16b, v7.16b, v25.16b, #12 - ushr v25.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - orr v2.16b, v2.16b, v25.16b - add v3.4s, v3.4s, v2.4s - eor v24.16b, v3.16b, v24.16b - add v3.4s, v3.4s, v17.4s - ushr v17.4s, v24.4s, #8 - shl v18.4s, v24.4s, #24 - orr v17.16b, v18.16b, v17.16b - add v18.4s, v17.4s, v23.4s - eor v2.16b, v18.16b, v2.16b - ushr v23.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - ext v3.16b, v3.16b, v3.16b, #12 - orr v2.16b, v2.16b, v23.16b - ext v17.16b, v17.16b, v17.16b, #8 - add v3.4s, v2.4s, v3.4s - adrp x11, .LCPI1_1 - eor v17.16b, v3.16b, v17.16b - ldr q16, [x11, :lo12:.LCPI1_1] - ext v18.16b, v18.16b, v18.16b, #4 - rev32 v24.8h, v17.8h - movi v0.2d, #0xffffffff00000000 - add v23.4s, v3.4s, v21.4s - mov v21.s[1], v20.s[2] - add v20.4s, v18.4s, v24.4s - bit v19.16b, v21.16b, v0.16b - eor v3.16b, v20.16b, v2.16b - uzp2 v2.4s, v22.4s, v19.4s - zip1 v17.2d, v5.2d, v19.2d - zip2 v18.4s, v19.4s, v5.4s - ushr v21.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - ext v22.16b, v2.16b, v2.16b, #4 - bsl v16.16b, v4.16b, v17.16b - zip1 v17.4s, v18.4s, v4.4s - zip1 v18.4s, v4.4s, v18.4s - orr v21.16b, v3.16b, v21.16b - ext v25.16b, v16.16b, v16.16b, #12 - ext v3.16b, v18.16b, v17.16b, #8 - uzp1 v18.4s, v22.4s, v22.4s - ext v26.16b, v22.16b, v22.16b, #12 - add v23.4s, v23.4s, v21.4s - uzp1 v17.4s, v16.4s, v25.4s - ext v16.16b, v18.16b, v22.16b, #8 - ext v18.16b, v22.16b, v26.16b, #12 - eor v22.16b, v23.16b, v24.16b - add v6.4s, v23.4s, v6.4s - ushr v23.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - orr v22.16b, v22.16b, v23.16b - add v20.4s, v22.4s, v20.4s - eor v21.16b, v20.16b, v21.16b - ushr v23.4s, v21.4s, #7 - shl v21.4s, v21.4s, #25 - ext v6.16b, v6.16b, v6.16b, #4 - orr v21.16b, v21.16b, v23.16b - ext v22.16b, v22.16b, v22.16b, #8 - add v6.4s, v21.4s, v6.4s - eor v22.16b, v6.16b, v22.16b - ext v20.16b, v20.16b, v20.16b, #12 - add v6.4s, v6.4s, v19.4s - rev32 v19.8h, v22.8h - add v20.4s, v20.4s, v19.4s - eor v21.16b, v20.16b, v21.16b - ushr v22.4s, v21.4s, #12 - shl v21.4s, v21.4s, #20 - orr v21.16b, v21.16b, v22.16b - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - ushr v22.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v19.16b, v19.16b, v22.16b - add v20.4s, v19.4s, v20.4s - eor v21.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #12 - ushr v22.4s, v21.4s, #7 - shl v21.4s, v21.4s, #25 - add v6.4s, v6.4s, v4.4s - orr v21.16b, v21.16b, v22.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - ext v20.16b, v20.16b, v20.16b, #4 - rev32 v19.8h, v19.8h - add v20.4s, v20.4s, v19.4s - add v6.4s, v6.4s, v5.4s - mov v5.s[1], v4.s[2] - eor v4.16b, v20.16b, v21.16b - ushr v21.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - orr v21.16b, v4.16b, v21.16b - add v6.4s, v6.4s, v21.4s - eor v19.16b, v6.16b, v19.16b - add v2.4s, v6.4s, v2.4s - ushr v6.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v6.16b, v19.16b, v6.16b - add v19.4s, v6.4s, v20.4s - eor v20.16b, v19.16b, v21.16b - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - orr v20.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v2.4s, v20.4s, v2.4s - eor v6.16b, v2.16b, v6.16b - ext v19.16b, v19.16b, v19.16b, #12 - rev32 v6.8h, v6.8h - add v19.4s, v19.4s, v6.4s - mov v22.16b, v0.16b - eor v20.16b, v19.16b, v20.16b - bsl v22.16b, v5.16b, v7.16b - ushr v21.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - add v2.4s, v2.4s, v22.4s - orr v20.16b, v20.16b, v21.16b - add v2.4s, v2.4s, v20.4s - eor v6.16b, v2.16b, v6.16b - ushr v21.4s, v6.4s, #8 - shl v6.4s, v6.4s, #24 - orr v6.16b, v6.16b, v21.16b - add v19.4s, v6.4s, v19.4s - eor v20.16b, v19.16b, v20.16b - ext v2.16b, v2.16b, v2.16b, #12 - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - add v2.4s, v2.4s, v17.4s - orr v20.16b, v20.16b, v21.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v2.4s, v2.4s, v20.4s - eor v6.16b, v2.16b, v6.16b - uzp2 v5.4s, v16.4s, v22.4s - zip1 v7.2d, v3.2d, v22.2d - zip2 v16.4s, v22.4s, v3.4s - ext v19.16b, v19.16b, v19.16b, #4 - rev32 v22.8h, v6.8h - ext v23.16b, v5.16b, v5.16b, #4 - bif v7.16b, v17.16b, v1.16b - zip1 v24.4s, v16.4s, v17.4s - zip1 v16.4s, v17.4s, v16.4s - add v21.4s, v2.4s, v3.4s - mov v3.s[1], v17.s[2] - add v17.4s, v19.4s, v22.4s - mov v19.16b, v0.16b - ext v25.16b, v7.16b, v7.16b, #12 - ext v4.16b, v16.16b, v24.16b, #8 - uzp1 v16.4s, v23.4s, v23.4s - bsl v19.16b, v3.16b, v18.16b - eor v2.16b, v17.16b, v20.16b - uzp1 v7.4s, v7.4s, v25.4s - ext v25.16b, v16.16b, v23.16b, #8 - zip1 v3.2d, v4.2d, v19.2d - ushr v20.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - ext v24.16b, v23.16b, v23.16b, #12 - uzp2 v6.4s, v25.4s, v19.4s - zip2 v18.4s, v19.4s, v4.4s - bif v3.16b, v7.16b, v1.16b - orr v20.16b, v2.16b, v20.16b - ext v16.16b, v23.16b, v24.16b, #12 - ext v23.16b, v6.16b, v6.16b, #4 - zip1 v24.4s, v18.4s, v7.4s - zip1 v18.4s, v7.4s, v18.4s - ext v25.16b, v3.16b, v3.16b, #12 - add v21.4s, v21.4s, v20.4s - ext v2.16b, v18.16b, v24.16b, #8 - uzp1 v18.4s, v23.4s, v23.4s - ext v24.16b, v23.16b, v23.16b, #12 - uzp1 v3.4s, v3.4s, v25.4s - eor v22.16b, v21.16b, v22.16b - ext v25.16b, v18.16b, v23.16b, #8 - dup v18.4s, v2.s[3] - ext v23.16b, v23.16b, v24.16b, #12 - add v5.4s, v21.4s, v5.4s - trn1 v21.4s, v3.4s, v3.4s - ushr v24.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - ext v18.16b, v21.16b, v18.16b, #8 - orr v21.16b, v22.16b, v24.16b - add v17.4s, v21.4s, v17.4s - eor v20.16b, v17.16b, v20.16b - ushr v22.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - ext v5.16b, v5.16b, v5.16b, #4 - orr v20.16b, v20.16b, v22.16b - ext v21.16b, v21.16b, v21.16b, #8 - add v5.4s, v20.4s, v5.4s - eor v21.16b, v5.16b, v21.16b - ext v17.16b, v17.16b, v17.16b, #12 - add v5.4s, v5.4s, v19.4s - rev32 v19.8h, v21.8h - add v17.4s, v17.4s, v19.4s - eor v20.16b, v17.16b, v20.16b - ushr v21.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - orr v20.16b, v20.16b, v21.16b - add v5.4s, v5.4s, v20.4s - eor v19.16b, v5.16b, v19.16b - ushr v21.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v19.16b, v19.16b, v21.16b - add v17.4s, v19.4s, v17.4s - eor v20.16b, v17.16b, v20.16b - ext v5.16b, v5.16b, v5.16b, #12 - ushr v21.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - add v5.4s, v5.4s, v7.4s - orr v20.16b, v20.16b, v21.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v5.4s, v5.4s, v20.4s - eor v19.16b, v5.16b, v19.16b - ext v17.16b, v17.16b, v17.16b, #4 - rev32 v22.8h, v19.8h - add v21.4s, v5.4s, v4.4s - mov v4.s[1], v7.s[2] - add v19.4s, v17.4s, v22.4s - bit v16.16b, v4.16b, v0.16b - eor v5.16b, v19.16b, v20.16b - uzp2 v4.4s, v25.4s, v16.4s - zip1 v7.2d, v2.2d, v16.2d - zip2 v17.4s, v16.4s, v2.4s - ushr v20.4s, v5.4s, #12 - shl v5.4s, v5.4s, #20 - ext v24.16b, v4.16b, v4.16b, #4 - bif v7.16b, v3.16b, v1.16b - zip1 v25.4s, v17.4s, v3.4s - zip1 v17.4s, v3.4s, v17.4s - orr v20.16b, v5.16b, v20.16b - ext v26.16b, v7.16b, v7.16b, #12 - ext v5.16b, v17.16b, v25.16b, #8 - uzp1 v17.4s, v24.4s, v24.4s - ext v25.16b, v24.16b, v24.16b, #12 - bit v23.16b, v18.16b, v0.16b - add v21.4s, v21.4s, v20.4s - uzp1 v7.4s, v7.4s, v26.4s - ext v26.16b, v17.16b, v24.16b, #8 - ext v17.16b, v24.16b, v25.16b, #12 - eor v22.16b, v21.16b, v22.16b - add v6.4s, v21.4s, v6.4s - zip1 v21.2d, v5.2d, v23.2d - zip2 v24.4s, v23.4s, v5.4s - bif v21.16b, v7.16b, v1.16b - zip1 v1.4s, v24.4s, v7.4s - zip1 v24.4s, v7.4s, v24.4s - ext v1.16b, v24.16b, v1.16b, #8 - ushr v24.4s, v22.4s, #8 - shl v22.4s, v22.4s, #24 - orr v22.16b, v22.16b, v24.16b - add v19.4s, v22.4s, v19.4s - ext v24.16b, v21.16b, v21.16b, #12 - eor v20.16b, v19.16b, v20.16b - uzp1 v21.4s, v21.4s, v24.4s - ushr v24.4s, v20.4s, #7 - shl v20.4s, v20.4s, #25 - orr v20.16b, v20.16b, v24.16b - ext v6.16b, v6.16b, v6.16b, #4 - ext v22.16b, v22.16b, v22.16b, #8 - add v6.4s, v20.4s, v6.4s - eor v22.16b, v6.16b, v22.16b - ext v19.16b, v19.16b, v19.16b, #12 - add v6.4s, v6.4s, v16.4s - rev32 v16.8h, v22.8h - add v19.4s, v19.4s, v16.4s - eor v20.16b, v19.16b, v20.16b - ushr v22.4s, v20.4s, #12 - shl v20.4s, v20.4s, #20 - orr v20.16b, v20.16b, v22.16b - add v6.4s, v6.4s, v20.4s - eor v16.16b, v6.16b, v16.16b - ext v6.16b, v6.16b, v6.16b, #12 - add v3.4s, v6.4s, v3.4s - ushr v6.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - orr v6.16b, v16.16b, v6.16b - add v16.4s, v6.4s, v19.4s - eor v19.16b, v16.16b, v20.16b - ushr v20.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v19.16b, v19.16b, v20.16b - ext v6.16b, v6.16b, v6.16b, #8 - add v3.4s, v3.4s, v19.4s - eor v6.16b, v3.16b, v6.16b - ext v16.16b, v16.16b, v16.16b, #4 - add v2.4s, v3.4s, v2.4s - rev32 v3.8h, v6.8h - add v6.4s, v16.4s, v3.4s - eor v16.16b, v6.16b, v19.16b - ushr v19.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - orr v16.16b, v16.16b, v19.16b - add v2.4s, v2.4s, v16.4s - eor v3.16b, v2.16b, v3.16b - add v2.4s, v2.4s, v4.4s - ushr v4.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v3.16b, v3.16b, v4.16b - add v4.4s, v3.4s, v6.4s - eor v6.16b, v4.16b, v16.16b - ushr v16.4s, v6.4s, #7 - shl v6.4s, v6.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - orr v6.16b, v6.16b, v16.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v6.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #12 - rev32 v3.8h, v3.8h - add v4.4s, v4.4s, v3.4s - eor v6.16b, v4.16b, v6.16b - ushr v16.4s, v6.4s, #12 - shl v6.4s, v6.4s, #20 - add v2.4s, v2.4s, v23.4s - orr v6.16b, v6.16b, v16.16b - add v2.4s, v2.4s, v6.4s - eor v3.16b, v2.16b, v3.16b - ushr v16.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v3.16b, v3.16b, v16.16b - add v4.4s, v3.4s, v4.4s - eor v6.16b, v4.16b, v6.16b - ext v2.16b, v2.16b, v2.16b, #12 - ushr v16.4s, v6.4s, #7 - shl v6.4s, v6.4s, #25 - add v2.4s, v2.4s, v7.4s - orr v6.16b, v6.16b, v16.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v2.4s, v6.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #4 - rev32 v3.8h, v3.8h + hint #34 + fmov s1, w3 + movi d0, #0x0000ff000000ff + ldr q2, [x1] + fmov d3, x4 + adrp x8, .LCPI1_0 + mov v1.s[1], w5 + str q2, [x0] + ldr q4, [x8, :lo12:.LCPI1_0] + add x8, x2, #32 + ldr q5, [x1, #16] + and v0.8b, v1.8b, v0.8b + stp q5, q4, [x0, #16] + mov v3.d[1], v0.d[0] + str q3, [x0, #48] + ldp q0, q6, [x2] + uzp1 v1.4s, v0.4s, v6.4s + uzp2 v0.4s, v0.4s, v6.4s + add v2.4s, v2.4s, v1.4s + uzp1 v18.4s, v1.4s, v1.4s add v2.4s, v2.4s, v5.4s - mov v5.s[1], v7.s[2] - add v4.4s, v4.4s, v3.4s - bsl v0.16b, v5.16b, v17.16b - eor v5.16b, v4.16b, v6.16b + eor v3.16b, v2.16b, v3.16b + add v2.4s, v2.4s, v0.4s + rev32 v3.8h, v3.8h + add v4.4s, v3.4s, v4.4s + eor v5.16b, v4.16b, v5.16b ushr v6.4s, v5.4s, #12 shl v5.4s, v5.4s, #20 orr v5.16b, v5.16b, v6.16b @@ -909,78 +126,477 @@ zfs_blake3_compress_xof_sse2: ushr v6.4s, v3.4s, #8 shl v3.4s, v3.4s, #24 orr v3.16b, v3.16b, v6.16b + ld2 { v6.4s, v7.4s }, [x8] add v4.4s, v3.4s, v4.4s - uzp2 v18.4s, v26.4s, v18.4s + ext v3.16b, v3.16b, v3.16b, #8 + add v2.4s, v2.4s, v6.4s eor v5.16b, v4.16b, v5.16b - add v2.4s, v2.4s, v18.4s - ushr v6.4s, v5.4s, #7 + ext v4.16b, v4.16b, v4.16b, #4 + ext v6.16b, v6.16b, v6.16b, #12 + ext v2.16b, v2.16b, v2.16b, #12 + ushr v16.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v16.16b + ext v16.16b, v7.16b, v7.16b, #12 + add v2.4s, v2.4s, v5.4s + mov v7.16b, v16.16b + eor v3.16b, v3.16b, v2.16b + add v2.4s, v2.4s, v16.4s + mov v7.s[1], v6.s[2] + rev32 v3.8h, v3.8h + add v4.4s, v4.4s, v3.4s + eor v5.16b, v4.16b, v5.16b + ushr v17.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v17.16b + add v2.4s, v2.4s, v5.4s + eor v3.16b, v2.16b, v3.16b + ushr v17.4s, v3.4s, #8 + shl v3.4s, v3.4s, #24 + orr v3.16b, v3.16b, v17.16b + ext v17.16b, v18.16b, v1.16b, #8 + add v4.4s, v3.4s, v4.4s + uzp2 v17.4s, v17.4s, v0.4s + ext v3.16b, v3.16b, v3.16b, #8 + eor v5.16b, v4.16b, v5.16b + add v2.4s, v2.4s, v17.4s + ext v4.16b, v4.16b, v4.16b, #12 + ushr v18.4s, v5.4s, #7 shl v5.4s, v5.4s, #25 ext v2.16b, v2.16b, v2.16b, #4 - orr v5.16b, v5.16b, v6.16b - ext v3.16b, v3.16b, v3.16b, #8 - add v2.4s, v5.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ext v4.16b, v4.16b, v4.16b, #12 - add v0.4s, v2.4s, v0.4s - rev32 v2.8h, v3.8h - add v3.4s, v4.4s, v2.4s - eor v4.16b, v3.16b, v5.16b - ushr v5.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - orr v4.16b, v4.16b, v5.16b - add v0.4s, v0.4s, v4.4s - eor v2.16b, v0.16b, v2.16b - ushr v5.4s, v2.4s, #8 + orr v5.16b, v5.16b, v18.16b + ext v18.16b, v1.16b, v1.16b, #12 + add v2.4s, v2.4s, v5.4s + ext v1.16b, v1.16b, v18.16b, #12 + zip1 v18.2d, v16.2d, v0.2d + zip2 v0.4s, v0.4s, v16.4s + eor v3.16b, v3.16b, v2.16b + rev64 v1.4s, v1.4s + mov v18.s[3], v6.s[3] + zip1 v16.4s, v0.4s, v6.4s + rev32 v3.8h, v3.8h + trn2 v1.4s, v1.4s, v7.4s + zip1 v0.4s, v6.4s, v0.4s + add v4.4s, v4.4s, v3.4s + add v2.4s, v2.4s, v1.4s + ext v6.16b, v0.16b, v16.16b, #8 + eor v5.16b, v4.16b, v5.16b + ushr v7.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v7.16b + add v7.4s, v2.4s, v5.4s + eor v2.16b, v7.16b, v3.16b + ext v7.16b, v7.16b, v7.16b, #12 + ushr v3.4s, v2.4s, #8 shl v2.4s, v2.4s, #24 - orr v2.16b, v2.16b, v5.16b - add v3.4s, v2.4s, v3.4s - eor v4.16b, v3.16b, v4.16b - ext v0.16b, v0.16b, v0.16b, #12 - ushr v5.4s, v4.4s, #7 - shl v4.4s, v4.4s, #25 - add v0.4s, v0.4s, v21.4s - orr v4.16b, v4.16b, v5.16b - ext v2.16b, v2.16b, v2.16b, #8 - add v0.4s, v0.4s, v4.4s - eor v2.16b, v0.16b, v2.16b - ext v3.16b, v3.16b, v3.16b, #4 - add v0.4s, v0.4s, v1.4s - rev32 v1.8h, v2.8h - add v2.4s, v3.4s, v1.4s - eor v3.16b, v2.16b, v4.16b + orr v3.16b, v2.16b, v3.16b + ext v2.16b, v18.16b, v18.16b, #12 + add v4.4s, v3.4s, v4.4s + uzp1 v2.4s, v18.4s, v2.4s + ext v3.16b, v3.16b, v3.16b, #8 + eor v5.16b, v4.16b, v5.16b + add v7.4s, v7.4s, v2.4s + ext v4.16b, v4.16b, v4.16b, #4 + ushr v18.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v18.16b + add v7.4s, v7.4s, v5.4s + eor v3.16b, v3.16b, v7.16b + add v7.4s, v7.4s, v6.4s + rev32 v3.8h, v3.8h + add v4.4s, v4.4s, v3.4s + eor v5.16b, v4.16b, v5.16b + ushr v0.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v0.16b, v5.16b, v0.16b + add v5.4s, v7.4s, v0.4s + ext v7.16b, v17.16b, v17.16b, #4 + eor v3.16b, v5.16b, v3.16b + uzp1 v17.4s, v7.4s, v7.4s + ushr v16.4s, v3.4s, #8 + shl v3.4s, v3.4s, #24 + orr v3.16b, v3.16b, v16.16b + ext v16.16b, v17.16b, v7.16b, #8 + add v4.4s, v3.4s, v4.4s + uzp2 v16.4s, v16.4s, v1.4s + ext v3.16b, v3.16b, v3.16b, #8 + eor v0.16b, v4.16b, v0.16b + add v5.4s, v5.4s, v16.4s + ext v4.16b, v4.16b, v4.16b, #12 + ushr v17.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ext v5.16b, v5.16b, v5.16b, #4 + orr v0.16b, v0.16b, v17.16b + ext v17.16b, v7.16b, v7.16b, #12 + add v5.4s, v5.4s, v0.4s + ext v7.16b, v7.16b, v17.16b, #12 + mov v17.16b, v6.16b + eor v3.16b, v3.16b, v5.16b + rev64 v7.4s, v7.4s + mov v17.s[1], v2.s[2] + rev32 v3.8h, v3.8h + add v4.4s, v4.4s, v3.4s + eor v18.16b, v4.16b, v0.16b + trn2 v0.4s, v7.4s, v17.4s + ushr v7.4s, v18.4s, #12 + shl v17.4s, v18.4s, #20 + add v5.4s, v5.4s, v0.4s + zip1 v18.2d, v6.2d, v1.2d + zip2 v1.4s, v1.4s, v6.4s + orr v7.16b, v17.16b, v7.16b + mov v18.s[3], v2.s[3] + zip1 v6.4s, v1.4s, v2.4s + add v5.4s, v5.4s, v7.4s + zip1 v1.4s, v2.4s, v1.4s + eor v3.16b, v5.16b, v3.16b + ext v5.16b, v5.16b, v5.16b, #12 + ext v6.16b, v1.16b, v6.16b, #8 + ushr v17.4s, v3.4s, #8 + shl v3.4s, v3.4s, #24 + orr v17.16b, v3.16b, v17.16b + ext v3.16b, v18.16b, v18.16b, #12 + add v4.4s, v17.4s, v4.4s + uzp1 v3.4s, v18.4s, v3.4s + ext v17.16b, v17.16b, v17.16b, #8 + eor v7.16b, v4.16b, v7.16b + add v5.4s, v5.4s, v3.4s + ext v4.16b, v4.16b, v4.16b, #4 + ushr v18.4s, v7.4s, #7 + shl v7.4s, v7.4s, #25 + orr v7.16b, v7.16b, v18.16b + add v5.4s, v5.4s, v7.4s + eor v17.16b, v17.16b, v5.16b + add v5.4s, v5.4s, v6.4s + rev32 v17.8h, v17.8h + add v4.4s, v4.4s, v17.4s + eor v2.16b, v4.16b, v7.16b + ext v7.16b, v16.16b, v16.16b, #4 + ushr v1.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v2.16b, v1.16b + add v2.4s, v5.4s, v1.4s + eor v5.16b, v2.16b, v17.16b + uzp1 v17.4s, v7.4s, v7.4s + ushr v16.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + orr v5.16b, v5.16b, v16.16b + ext v16.16b, v17.16b, v7.16b, #8 + add v4.4s, v5.4s, v4.4s + uzp2 v16.4s, v16.4s, v0.4s + ext v5.16b, v5.16b, v5.16b, #8 + eor v1.16b, v4.16b, v1.16b + add v2.4s, v2.4s, v16.4s + ext v4.16b, v4.16b, v4.16b, #12 + ushr v17.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ext v2.16b, v2.16b, v2.16b, #4 + orr v1.16b, v1.16b, v17.16b + ext v17.16b, v7.16b, v7.16b, #12 + add v2.4s, v2.4s, v1.4s + ext v7.16b, v7.16b, v17.16b, #12 + mov v17.16b, v6.16b + eor v5.16b, v5.16b, v2.16b + rev64 v7.4s, v7.4s + mov v17.s[1], v3.s[2] + rev32 v5.8h, v5.8h + add v4.4s, v4.4s, v5.4s + eor v18.16b, v4.16b, v1.16b + trn2 v1.4s, v7.4s, v17.4s + ushr v7.4s, v18.4s, #12 + shl v17.4s, v18.4s, #20 + add v2.4s, v2.4s, v1.4s + zip1 v18.2d, v6.2d, v0.2d + zip2 v0.4s, v0.4s, v6.4s + orr v7.16b, v17.16b, v7.16b + mov v18.s[3], v3.s[3] + add v2.4s, v2.4s, v7.4s + eor v5.16b, v2.16b, v5.16b + ext v2.16b, v2.16b, v2.16b, #12 + ushr v17.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + orr v5.16b, v5.16b, v17.16b + add v17.4s, v5.4s, v4.4s + ext v4.16b, v18.16b, v18.16b, #12 + ext v5.16b, v5.16b, v5.16b, #8 + eor v7.16b, v17.16b, v7.16b + uzp1 v4.4s, v18.4s, v4.4s + ext v17.16b, v17.16b, v17.16b, #4 + ushr v18.4s, v7.4s, #7 + shl v7.4s, v7.4s, #25 + add v2.4s, v2.4s, v4.4s + orr v7.16b, v7.16b, v18.16b + add v2.4s, v2.4s, v7.4s + eor v5.16b, v5.16b, v2.16b + rev32 v5.8h, v5.8h + add v6.4s, v17.4s, v5.4s + zip1 v17.4s, v0.4s, v3.4s + zip1 v0.4s, v3.4s, v0.4s + eor v3.16b, v6.16b, v7.16b + ext v0.16b, v0.16b, v17.16b, #8 + ushr v7.4s, v3.4s, #12 + shl v3.4s, v3.4s, #20 + add v2.4s, v2.4s, v0.4s + orr v3.16b, v3.16b, v7.16b + ext v7.16b, v16.16b, v16.16b, #4 + add v2.4s, v2.4s, v3.4s + uzp1 v17.4s, v7.4s, v7.4s + eor v5.16b, v2.16b, v5.16b + ushr v16.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + orr v5.16b, v5.16b, v16.16b + ext v16.16b, v17.16b, v7.16b, #8 + add v6.4s, v5.4s, v6.4s + uzp2 v16.4s, v16.4s, v1.4s + ext v5.16b, v5.16b, v5.16b, #8 + eor v3.16b, v6.16b, v3.16b + add v2.4s, v2.4s, v16.4s + ext v6.16b, v6.16b, v6.16b, #12 + ushr v17.4s, v3.4s, #7 + shl v3.4s, v3.4s, #25 + ext v2.16b, v2.16b, v2.16b, #4 + orr v3.16b, v3.16b, v17.16b + add v17.4s, v2.4s, v3.4s + eor v2.16b, v5.16b, v17.16b + ext v5.16b, v7.16b, v7.16b, #12 + rev32 v18.8h, v2.8h + ext v2.16b, v7.16b, v5.16b, #12 + mov v5.16b, v0.16b + add v6.4s, v6.4s, v18.4s + rev64 v2.4s, v2.4s + mov v5.s[1], v4.s[2] + eor v3.16b, v6.16b, v3.16b + trn2 v2.4s, v2.4s, v5.4s + ushr v5.4s, v3.4s, #12 + shl v3.4s, v3.4s, #20 + add v7.4s, v17.4s, v2.4s + orr v3.16b, v3.16b, v5.16b + add v5.4s, v7.4s, v3.4s + eor v7.16b, v5.16b, v18.16b + zip1 v18.2d, v0.2d, v1.2d + ext v5.16b, v5.16b, v5.16b, #12 + zip2 v0.4s, v1.4s, v0.4s + ushr v17.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + mov v18.s[3], v4.s[3] + orr v7.16b, v7.16b, v17.16b + ext v17.16b, v18.16b, v18.16b, #12 + add v6.4s, v7.4s, v6.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v19.16b, v6.16b, v3.16b + uzp1 v3.4s, v18.4s, v17.4s + ext v6.16b, v6.16b, v6.16b, #4 + ushr v17.4s, v19.4s, #7 + shl v18.4s, v19.4s, #25 + add v5.4s, v5.4s, v3.4s + orr v17.16b, v18.16b, v17.16b + add v5.4s, v5.4s, v17.4s + eor v7.16b, v7.16b, v5.16b + rev32 v7.8h, v7.8h + add v1.4s, v6.4s, v7.4s + zip1 v6.4s, v0.4s, v4.4s + zip1 v0.4s, v4.4s, v0.4s + eor v4.16b, v1.16b, v17.16b + ext v6.16b, v0.16b, v6.16b, #8 + ushr v0.4s, v4.4s, #12 + shl v4.4s, v4.4s, #20 + add v5.4s, v5.4s, v6.4s + zip1 v20.2d, v6.2d, v2.2d + orr v0.16b, v4.16b, v0.16b + mov v20.s[3], v3.s[3] + add v4.4s, v5.4s, v0.4s + eor v5.16b, v4.16b, v7.16b + ext v7.16b, v16.16b, v16.16b, #4 + ushr v16.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + uzp1 v17.4s, v7.4s, v7.4s + orr v5.16b, v5.16b, v16.16b + ext v16.16b, v17.16b, v7.16b, #8 + add v1.4s, v5.4s, v1.4s + uzp2 v16.4s, v16.4s, v2.4s + zip2 v2.4s, v2.4s, v6.4s + eor v0.16b, v1.16b, v0.16b + add v4.4s, v4.4s, v16.4s + ext v1.16b, v1.16b, v1.16b, #12 + ext v16.16b, v16.16b, v16.16b, #4 + ushr v17.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ext v4.16b, v4.16b, v4.16b, #4 + orr v17.16b, v0.16b, v17.16b + ext v0.16b, v5.16b, v5.16b, #8 + ext v5.16b, v7.16b, v7.16b, #12 + add v4.4s, v4.4s, v17.4s + eor v0.16b, v0.16b, v4.16b + rev32 v18.8h, v0.8h + ext v0.16b, v7.16b, v5.16b, #12 + mov v5.16b, v6.16b + add v7.4s, v1.4s, v18.4s + rev64 v1.4s, v0.4s + mov v5.s[1], v3.s[2] + eor v17.16b, v7.16b, v17.16b + trn2 v1.4s, v1.4s, v5.4s + ushr v19.4s, v17.4s, #12 + shl v17.4s, v17.4s, #20 + add v4.4s, v4.4s, v1.4s + orr v17.16b, v17.16b, v19.16b + add v19.4s, v4.4s, v17.4s + eor v4.16b, v19.16b, v18.16b + ext v19.16b, v19.16b, v19.16b, #12 + ushr v18.4s, v4.4s, #8 + shl v4.4s, v4.4s, #24 + orr v18.16b, v4.16b, v18.16b + ext v4.16b, v20.16b, v20.16b, #12 + add v7.4s, v18.4s, v7.4s + uzp1 v4.4s, v20.4s, v4.4s + ext v18.16b, v18.16b, v18.16b, #8 + eor v17.16b, v7.16b, v17.16b + add v19.4s, v19.4s, v4.4s + ext v7.16b, v7.16b, v7.16b, #4 + ushr v20.4s, v17.4s, #7 + shl v17.4s, v17.4s, #25 + orr v17.16b, v17.16b, v20.16b + add v19.4s, v19.4s, v17.4s + eor v18.16b, v18.16b, v19.16b + rev32 v18.8h, v18.8h + add v6.4s, v7.4s, v18.4s + zip1 v7.4s, v2.4s, v3.4s + zip1 v2.4s, v3.4s, v2.4s + eor v3.16b, v6.16b, v17.16b + ext v2.16b, v2.16b, v7.16b, #8 + ushr v7.4s, v3.4s, #12 + shl v3.4s, v3.4s, #20 + add v17.4s, v19.4s, v2.4s + zip1 v1.2d, v2.2d, v1.2d + zip2 v0.4s, v0.4s, v2.4s + orr v3.16b, v3.16b, v7.16b + mov v1.s[3], v4.s[3] + add v7.4s, v17.4s, v3.4s + eor v17.16b, v7.16b, v18.16b + ext v7.16b, v7.16b, v7.16b, #4 + ushr v18.4s, v17.4s, #8 + shl v17.4s, v17.4s, #24 + orr v17.16b, v17.16b, v18.16b + ext v18.16b, v16.16b, v16.16b, #8 + add v6.4s, v17.4s, v6.4s + uzp2 v5.4s, v18.4s, v5.4s + eor v3.16b, v6.16b, v3.16b + ext v5.16b, v5.16b, v18.16b, #4 + ext v6.16b, v6.16b, v6.16b, #12 + ushr v18.4s, v3.4s, #7 + shl v3.4s, v3.4s, #25 + add v5.4s, v7.4s, v5.4s + ext v7.16b, v17.16b, v17.16b, #8 + ext v17.16b, v16.16b, v16.16b, #12 + orr v3.16b, v3.16b, v18.16b + ext v16.16b, v16.16b, v17.16b, #12 + add v5.4s, v3.4s, v5.4s + mov v17.16b, v2.16b + rev64 v16.4s, v16.4s + eor v7.16b, v7.16b, v5.16b + mov v17.s[1], v4.s[2] + rev32 v7.8h, v7.8h + trn2 v16.4s, v16.4s, v17.4s + add v6.4s, v6.4s, v7.4s + add v5.4s, v5.4s, v16.4s + eor v3.16b, v6.16b, v3.16b + ushr v17.4s, v3.4s, #12 + shl v3.4s, v3.4s, #20 + orr v3.16b, v3.16b, v17.16b + add v5.4s, v5.4s, v3.4s + eor v7.16b, v5.16b, v7.16b + ext v5.16b, v5.16b, v5.16b, #12 + ushr v16.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + orr v7.16b, v7.16b, v16.16b + ext v16.16b, v1.16b, v1.16b, #12 + add v6.4s, v7.4s, v6.4s + uzp1 v1.4s, v1.4s, v16.4s + eor v3.16b, v6.16b, v3.16b + add v1.4s, v5.4s, v1.4s + ext v5.16b, v7.16b, v7.16b, #8 + ext v6.16b, v6.16b, v6.16b, #4 + ushr v16.4s, v3.4s, #7 + shl v3.4s, v3.4s, #25 + orr v3.16b, v3.16b, v16.16b + add v1.4s, v1.4s, v3.4s + eor v5.16b, v5.16b, v1.16b + rev32 v5.8h, v5.8h + add v2.4s, v6.4s, v5.4s + zip1 v6.4s, v0.4s, v4.4s + zip1 v0.4s, v4.4s, v0.4s + eor v3.16b, v2.16b, v3.16b + ext v0.16b, v0.16b, v6.16b, #8 ushr v4.4s, v3.4s, #12 shl v3.4s, v3.4s, #20 - orr v3.16b, v3.16b, v4.16b - add v0.4s, v0.4s, v3.4s - eor v1.16b, v0.16b, v1.16b - ushr v4.4s, v1.4s, #8 - shl v1.4s, v1.4s, #24 - orr v1.16b, v1.16b, v4.16b - add v2.4s, v1.4s, v2.4s - eor v3.16b, v2.16b, v3.16b - ushr v4.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 + add v0.4s, v1.4s, v0.4s + orr v1.16b, v3.16b, v4.16b + add v0.4s, v0.4s, v1.4s + eor v3.16b, v0.16b, v5.16b ext v0.16b, v0.16b, v0.16b, #4 - ext v1.16b, v1.16b, v1.16b, #8 - ext v2.16b, v2.16b, v2.16b, #12 + ushr v4.4s, v3.4s, #8 + shl v3.4s, v3.4s, #24 orr v3.16b, v3.16b, v4.16b - eor v0.16b, v2.16b, v0.16b - eor v3.16b, v3.16b, v1.16b - stp q0, q3, [x5] - ldr q0, [x0] - eor v0.16b, v0.16b, v2.16b - str q0, [x5, #32] - ldr q0, [x0, #16] - eor v0.16b, v0.16b, v1.16b - str q0, [x5, #48] + add v2.4s, v3.4s, v2.4s + ext v3.16b, v3.16b, v3.16b, #8 + eor v1.16b, v2.16b, v1.16b + ext v2.16b, v2.16b, v2.16b, #12 + ushr v4.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + stp q2, q3, [x0, #32] + orr v1.16b, v1.16b, v4.16b + stp q0, q1, [x0] ret .Lfunc_end1: - .size zfs_blake3_compress_xof_sse2, .Lfunc_end1-zfs_blake3_compress_xof_sse2 + .size compress_pre, .Lfunc_end1-compress_pre + .cfi_endproc + + .globl zfs_blake3_compress_xof_sse2 + .p2align 2 + .type zfs_blake3_compress_xof_sse2,@function +zfs_blake3_compress_xof_sse2: + .cfi_startproc + hint #25 + .cfi_negate_ra_state + sub sp, sp, #96 + stp x29, x30, [sp, #64] + add x29, sp, #64 + stp x20, x19, [sp, #80] + .cfi_def_cfa w29, 32 + .cfi_offset w19, -8 + .cfi_offset w20, -16 + .cfi_offset w30, -24 + .cfi_offset w29, -32 + mov x20, x0 + mov x19, x5 + mov w5, w4 + mov x4, x3 + mov w3, w2 + mov x2, x1 + mov x0, sp + mov x1, x20 + bl compress_pre + ldp q0, q1, [sp] + ldp q2, q3, [sp, #32] + eor v0.16b, v2.16b, v0.16b + eor v1.16b, v3.16b, v1.16b + ldp x29, x30, [sp, #64] + stp q0, q1, [x19] + ldr q0, [x20] + eor v0.16b, v0.16b, v2.16b + str q0, [x19, #32] + ldr q0, [x20, #16] + eor v0.16b, v0.16b, v3.16b + str q0, [x19, #48] + ldp x20, x19, [sp, #80] + add sp, sp, #96 + hint #29 + ret +.Lfunc_end2: + .size zfs_blake3_compress_xof_sse2, .Lfunc_end2-zfs_blake3_compress_xof_sse2 .cfi_endproc .section .rodata.cst16,"aM",@progbits,16 .p2align 4 -.LCPI2_0: +.LCPI3_0: .word 0 .word 1 .word 2 @@ -991,19 +607,21 @@ zfs_blake3_compress_xof_sse2: .type zfs_blake3_hash_many_sse2,@function zfs_blake3_hash_many_sse2: .cfi_startproc + hint #25 + .cfi_negate_ra_state stp d15, d14, [sp, #-160]! stp d13, d12, [sp, #16] stp d11, d10, [sp, #32] stp d9, d8, [sp, #48] stp x29, x30, [sp, #64] + add x29, sp, #64 stp x28, x27, [sp, #80] stp x26, x25, [sp, #96] stp x24, x23, [sp, #112] stp x22, x21, [sp, #128] stp x20, x19, [sp, #144] - mov x29, sp - sub sp, sp, #384 - .cfi_def_cfa w29, 160 + sub sp, sp, #464 + .cfi_def_cfa w29, 96 .cfi_offset w19, -8 .cfi_offset w20, -16 .cfi_offset w21, -24 @@ -1024,1414 +642,1406 @@ zfs_blake3_hash_many_sse2: .cfi_offset b13, -144 .cfi_offset b14, -152 .cfi_offset b15, -160 - ldr x26, [x29, #168] - ldrb w27, [x29, #160] mov w19, w6 mov x20, x4 - mov x22, x2 - mov x28, x1 + mov x24, x1 + ldr x26, [x29, #104] + ldrb w27, [x29, #96] cmp x1, #4 - mov x24, x0 str x3, [sp, #40] - b.lo .LBB2_8 - adrp x9, .LCPI2_0 - ldr q0, [x9, :lo12:.LCPI2_0] - sbfx w11, w5, #0, #1 - dup v1.4s, w11 - mov w9, #58983 + b.lo .LBB3_6 + adrp x8, .LCPI3_0 + sbfx w9, w5, #0, #1 mov w10, #44677 - and v0.16b, v1.16b, v0.16b mov w11, #62322 - mov w12, #62778 - orr w8, w7, w19 - movk w9, #27145, lsl #16 movk w10, #47975, lsl #16 movk w11, #15470, lsl #16 + ldr q0, [x8, :lo12:.LCPI3_0] + dup v1.4s, w9 + mov w9, #58983 + orr w8, w7, w19 + movk w9, #27145, lsl #16 + and v0.16b, v1.16b, v0.16b + dup v1.4s, w11 + movi v24.4s, #64 + dup v2.4s, w9 + mov w9, #62778 + movk w9, #42319, lsl #16 str q0, [sp, #16] orr v0.4s, #128, lsl #24 - movk w12, #42319, lsl #16 + stp q2, q1, [sp, #48] str q0, [sp] -.LBB2_2: - ldr x0, [sp, #40] - mov x13, x0 - ld1r { v20.4s }, [x13], #4 - add x14, x0, #8 - add x15, x0, #12 - add x16, x0, #16 - add x17, x0, #20 - add x18, x0, #24 - add x0, x0, #28 - ld1r { v17.4s }, [x14] - ld1r { v6.4s }, [x15] - ld1r { v8.4s }, [x16] - ld1r { v9.4s }, [x17] - ld1r { v31.4s }, [x18] - ld1r { v26.4s }, [x13] - ld1r { v15.4s }, [x0] - cbz x22, .LBB2_7 + dup v0.4s, w10 + str q0, [sp, #80] + b .LBB3_3 +.LBB3_2: + zip1 v0.4s, v12.4s, v31.4s + add x10, x20, #4 + zip1 v1.4s, v29.4s, v30.4s + tst w5, #0x1 + zip1 v2.4s, v28.4s, v23.4s + csel x20, x10, x20, ne + zip1 v3.4s, v13.4s, v25.4s + add x0, x0, #32 + zip2 v6.4s, v12.4s, v31.4s + sub x24, x24, #4 + zip1 v4.2d, v0.2d, v1.2d + cmp x24, #3 + zip2 v7.4s, v29.4s, v30.4s + zip1 v5.2d, v2.2d, v3.2d + zip2 v0.2d, v0.2d, v1.2d + zip2 v1.2d, v2.2d, v3.2d + zip2 v2.4s, v28.4s, v23.4s + zip2 v3.4s, v13.4s, v25.4s + stp q4, q5, [x26] + zip2 v4.2d, v6.2d, v7.2d + stp q0, q1, [x26, #32] + zip1 v0.2d, v6.2d, v7.2d + zip1 v1.2d, v2.2d, v3.2d + zip2 v2.2d, v2.2d, v3.2d + stp q0, q1, [x26, #64] + stp q4, q2, [x26, #96] + add x26, x26, #128 + b.ls .LBB3_6 +.LBB3_3: + ldr x14, [sp, #40] + mov x10, x14 + add x11, x14, #8 + add x12, x14, #12 + add x13, x14, #16 + ld1r { v12.4s }, [x10], #4 + ld1r { v29.4s }, [x11] + add x11, x14, #20 + ld1r { v30.4s }, [x12] + add x12, x14, #24 + ld1r { v28.4s }, [x13] + ld1r { v23.4s }, [x11] + add x11, x14, #28 + ld1r { v13.4s }, [x12] + ld1r { v31.4s }, [x10] + ld1r { v25.4s }, [x11] + cbz x2, .LBB3_2 ldr q1, [sp, #16] dup v0.4s, w20 - ldp x13, x14, [x24] - ldp x15, x16, [x24, #16] + lsr x12, x20, #32 + mov x10, xzr + ldp x13, x14, [x0, #16] add v1.4s, v0.4s, v1.4s + mov x15, x2 movi v0.4s, #128, lsl #24 - str q1, [sp, #64] + mov w4, w8 + str q1, [sp, #112] eor v0.16b, v1.16b, v0.16b ldr q1, [sp] - lsr x18, x20, #32 - mov x17, xzr cmgt v0.4s, v1.4s, v0.4s - dup v1.4s, w18 + dup v1.4s, w12 + ldp x11, x12, [x0] sub v0.4s, v1.4s, v0.4s - mov w18, w8 - str q0, [sp, #48] -.LBB2_4: - mov w2, #16 - bfi x2, x17, #6, #58 - ldr q1, [x13, x2] - ldr q3, [x14, x2] - ldr q2, [x15, x2] - ldr q4, [x16, x2] - mov w2, #32 - bfi x2, x17, #6, #58 - ldr q5, [x13, x2] - ldr q18, [x14, x2] - ldr q19, [x15, x2] - ldr q23, [x16, x2] - mov w2, #48 - lsl x3, x17, #6 - bfi x2, x17, #6, #58 - add x17, x17, #1 - ldr q0, [x13, x3] - ldr q21, [x14, x3] - ldr q7, [x15, x3] - ldr q16, [x16, x3] - cmp x17, x22 - ldr q13, [x13, x2] - ldr q14, [x14, x2] - ldr q29, [x15, x2] - ldr q10, [x16, x2] - csel w2, w27, wzr, eq - orr w18, w2, w18 - mov x0, xzr - and w18, w18, #0xff - add x3, x3, #256 -.LBB2_5: - ldr x2, [x24, x0] - add x0, x0, #8 - cmp x0, #32 - add x2, x2, x3 - prfm pldl1keep, [x2] - b.ne .LBB2_5 - dup v22.4s, w18 - str q22, [sp, #192] - zip1 v27.4s, v0.4s, v21.4s - zip2 v21.4s, v0.4s, v21.4s - zip1 v0.4s, v7.4s, v16.4s - zip2 v22.4s, v7.4s, v16.4s - zip1 v7.4s, v1.4s, v3.4s - zip1 v25.4s, v2.4s, v4.4s - zip2 v16.4s, v2.4s, v4.4s - zip1 v11.4s, v19.4s, v23.4s - zip2 v12.4s, v19.4s, v23.4s - zip1 v19.4s, v13.4s, v14.4s - zip2 v23.4s, v13.4s, v14.4s - zip1 v13.4s, v29.4s, v10.4s - zip2 v14.4s, v29.4s, v10.4s - add v10.4s, v20.4s, v8.4s - add v2.4s, v26.4s, v9.4s - ext v20.16b, v22.16b, v21.16b, #8 - ext v26.16b, v25.16b, v7.16b, #8 - zip2 v24.4s, v1.4s, v3.4s - add v1.4s, v6.4s, v15.4s - ext v6.16b, v0.16b, v27.16b, #8 - ext v20.16b, v21.16b, v20.16b, #8 - mov v21.d[1], v22.d[0] - ext v22.16b, v7.16b, v26.16b, #8 - mov v7.d[1], v25.d[0] - add v3.4s, v17.4s, v31.4s - str q1, [sp, #144] - ext v1.16b, v27.16b, v6.16b, #8 - mov v6.16b, v7.16b - zip1 v28.4s, v5.4s, v18.4s - stur q1, [x29, #-80] - mov v1.16b, v27.16b - mov v27.16b, v24.16b - add v3.4s, v3.4s, v6.4s - ldr q6, [sp, #64] - ext v29.16b, v16.16b, v24.16b, #8 - mov v1.d[1], v0.d[0] - ext v0.16b, v11.16b, v28.16b, #8 - mov v27.d[1], v16.d[0] - ext v16.16b, v14.16b, v23.16b, #8 - stur q7, [x29, #-144] - ext v7.16b, v24.16b, v29.16b, #8 - ext v29.16b, v28.16b, v0.16b, #8 - ext v0.16b, v23.16b, v16.16b, #8 - mov v23.d[1], v14.d[0] - stp q0, q23, [sp, #80] - add v0.4s, v10.4s, v1.4s - eor v16.16b, v0.16b, v6.16b - ldr q6, [sp, #48] - add v2.4s, v2.4s, v21.4s - mov v28.d[1], v11.d[0] - zip2 v18.4s, v5.4s, v18.4s - eor v10.16b, v2.16b, v6.16b - movi v6.4s, #64 - eor v11.16b, v3.16b, v6.16b - ldr q6, [sp, #144] - dup v17.4s, w9 - ext v30.16b, v12.16b, v18.16b, #8 - rev32 v16.8h, v16.8h - dup v5.4s, w10 - ext v25.16b, v18.16b, v30.16b, #8 - mov v30.16b, v23.16b - mov v23.16b, v1.16b - str q1, [sp, #160] - rev32 v10.8h, v10.8h - add v1.4s, v16.4s, v17.4s - add v17.4s, v6.4s, v27.4s - ldr q6, [sp, #192] - dup v4.4s, w11 - rev32 v11.8h, v11.8h - add v5.4s, v10.4s, v5.4s - eor v8.16b, v1.16b, v8.16b - stur q21, [x29, #-128] - mov v18.d[1], v12.d[0] - add v4.4s, v11.4s, v4.4s - eor v9.16b, v5.16b, v9.16b - ushr v12.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - ldur q21, [x29, #-80] - ext v26.16b, v13.16b, v19.16b, #8 - eor v31.16b, v4.16b, v31.16b - orr v8.16b, v8.16b, v12.16b - ushr v12.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - ext v26.16b, v19.16b, v26.16b, #8 - mov v19.d[1], v13.d[0] - orr v9.16b, v9.16b, v12.16b - ushr v12.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v17.16b, v6.16b - orr v31.16b, v31.16b, v12.16b - dup v12.4s, w12 - rev32 v13.8h, v13.8h - add v12.4s, v13.4s, v12.4s - add v0.4s, v0.4s, v21.4s - eor v14.16b, v12.16b, v15.16b - add v0.4s, v0.4s, v8.4s - add v2.4s, v2.4s, v20.4s - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v16.16b, v0.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v22.4s - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v7.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v14.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v13.16b, v17.16b, v13.16b - add v1.4s, v16.4s, v1.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v13.4s, #8 - shl v13.4s, v13.4s, #24 - eor v8.16b, v1.16b, v8.16b - add v5.4s, v10.4s, v5.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v13.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v14.16b, v12.16b, v14.16b - add v0.4s, v0.4s, v28.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #7 - shl v14.4s, v14.4s, #25 - add v0.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v18.4s - orr v14.16b, v14.16b, v15.16b - eor v13.16b, v0.16b, v13.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v19.4s - rev32 v13.8h, v13.8h - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v30.4s - add v4.4s, v4.4s, v13.4s - rev32 v16.8h, v16.8h - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - eor v9.16b, v4.16b, v9.16b - add v12.4s, v12.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v17.16b, v11.16b - mov v24.16b, v7.16b - stur q7, [x29, #-112] - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v1.4s, v10.4s - rev32 v11.8h, v11.8h - mov v7.16b, v26.16b - add v3.4s, v3.4s, v26.4s - ldr q26, [sp, #80] - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v14.16b, v1.16b, v14.16b - add v5.4s, v5.4s, v11.4s - add v0.4s, v0.4s, v29.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v8.16b, v5.16b, v8.16b - add v0.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v25.4s - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v13.16b, v0.16b, v13.16b - add v2.4s, v2.4s, v31.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v13.4s, #8 - shl v13.4s, v13.4s, #24 - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v26.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v17.16b, v11.16b - add v4.4s, v13.4s, v4.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v9.16b, v4.16b, v9.16b - add v12.4s, v16.4s, v12.4s - str q22, [sp, #128] - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v10.4s, v1.4s - ldur q22, [x29, #-128] - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v14.16b, v1.16b, v14.16b - add v5.4s, v11.4s, v5.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #7 - shl v14.4s, v14.4s, #25 - eor v8.16b, v5.16b, v8.16b - mov v6.16b, v18.16b - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - ldur q18, [x29, #-144] - orr v8.16b, v8.16b, v15.16b - add v0.4s, v0.4s, v22.4s - add v0.4s, v0.4s, v8.4s - add v2.4s, v2.4s, v20.4s - eor v16.16b, v0.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v24.4s - rev32 v16.8h, v16.8h - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v18.4s - add v1.4s, v1.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v14.4s - eor v8.16b, v1.16b, v8.16b - add v5.4s, v5.4s, v10.4s - rev32 v11.8h, v11.8h - eor v13.16b, v17.16b, v13.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v4.4s, v11.4s - rev32 v13.8h, v13.8h - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v12.4s, v13.4s + str q0, [sp, #96] +.LBB3_5: + add x17, x11, x10 + add x21, x12, x10 + add x16, x13, x10 + add x6, x14, x10 + subs x15, x15, #1 + add x10, x10, #64 + ldp q0, q1, [x17] + csel w3, w27, wzr, eq + orr w3, w3, w4 + mov w4, w19 + and w3, w3, #0xff + ldp q3, q6, [x21] + dup v2.4s, w3 + zip1 v21.4s, v0.4s, v3.4s + zip2 v19.4s, v0.4s, v3.4s + ldp q5, q7, [x16] + zip1 v17.4s, v1.4s, v6.4s + zip2 v22.4s, v1.4s, v6.4s + ldp q16, q18, [x6] + zip1 v4.4s, v5.4s, v16.4s + zip2 v0.4s, v5.4s, v16.4s + ldp q26, q27, [x17, #32] + zip1 v1.4s, v7.4s, v18.4s + zip2 v3.4s, v7.4s, v18.4s + zip2 v20.2d, v19.2d, v0.2d + mov v19.d[1], v0.d[0] + dup v18.4s, w9 + ldp q8, q9, [x21, #32] + stur q19, [x29, #-208] + zip2 v7.4s, v26.4s, v8.4s + zip1 v10.4s, v26.4s, v8.4s + ldp q11, q5, [x16, #32] + zip2 v26.2d, v17.2d, v1.2d + stp q7, q26, [sp, #192] + mov v17.d[1], v1.d[0] + add v1.4s, v23.4s, v31.4s + ldp q16, q6, [x6, #32] + stur q17, [x29, #-256] + add v1.4s, v1.4s, v19.4s + zip1 v8.4s, v11.4s, v16.4s + zip2 v7.4s, v11.4s, v16.4s + zip1 v11.4s, v27.4s, v9.4s + zip2 v9.4s, v27.4s, v9.4s + zip2 v27.2d, v21.2d, v4.2d + mov v21.d[1], v4.d[0] + str q7, [sp, #224] + add v4.4s, v28.4s, v12.4s + zip1 v15.4s, v5.4s, v6.4s + zip2 v14.4s, v5.4s, v6.4s + stur q27, [x29, #-192] + zip2 v16.2d, v22.2d, v3.2d + stp q20, q21, [x29, #-240] + add v0.4s, v4.4s, v21.4s + ldp q6, q4, [sp, #96] + mov v22.d[1], v3.d[0] + add v5.4s, v25.4s, v30.4s + add v3.4s, v13.4s, v29.4s + eor v6.16b, v1.16b, v6.16b + add v1.4s, v1.4s, v20.4s + str q22, [sp, #256] + eor v4.16b, v0.16b, v4.16b + add v5.4s, v5.4s, v22.4s + add v3.4s, v3.4s, v17.4s + ldr q17, [sp, #48] + rev32 v6.8h, v6.8h + rev32 v4.8h, v4.8h + eor v2.16b, v5.16b, v2.16b + eor v7.16b, v3.16b, v24.16b add v0.4s, v0.4s, v27.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v14.16b, v12.16b, v14.16b - add v0.4s, v0.4s, v8.4s - add v2.4s, v2.4s, v6.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v16.16b, v0.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v23.4s - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v7.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v14.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v13.16b, v17.16b, v13.16b - add v1.4s, v16.4s, v1.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v13.4s, #8 - shl v13.4s, v13.4s, #24 - eor v8.16b, v1.16b, v8.16b - add v5.4s, v10.4s, v5.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v13.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v14.16b, v12.16b, v14.16b - add v0.4s, v0.4s, v21.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #7 - shl v14.4s, v14.4s, #25 - add v0.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v19.4s - orr v14.16b, v14.16b, v15.16b - eor v13.16b, v0.16b, v13.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v29.4s - str q28, [sp, #112] - rev32 v13.8h, v13.8h - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v26.4s - add v4.4s, v4.4s, v13.4s - rev32 v16.8h, v16.8h - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - ldp q28, q23, [sp, #112] - eor v9.16b, v4.16b, v9.16b - add v12.4s, v12.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v17.16b, v11.16b - ldr q21, [sp, #96] - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v1.4s, v10.4s - rev32 v11.8h, v11.8h - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v14.16b, v1.16b, v14.16b - add v5.4s, v5.4s, v11.4s - add v0.4s, v0.4s, v25.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v8.16b, v5.16b, v8.16b - add v0.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v23.4s - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v13.16b, v0.16b, v13.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v21.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v13.4s, #8 - shl v13.4s, v13.4s, #24 - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v28.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v17.16b, v11.16b - add v4.4s, v13.4s, v4.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v9.16b, v4.16b, v9.16b - add v12.4s, v16.4s, v12.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v10.4s, v1.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v14.16b, v1.16b, v14.16b - add v5.4s, v11.4s, v5.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #7 - shl v14.4s, v14.4s, #25 - eor v8.16b, v5.16b, v8.16b - mov v30.16b, v29.16b - mov v29.16b, v25.16b - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - ldur q25, [x29, #-112] - orr v8.16b, v8.16b, v15.16b - add v0.4s, v0.4s, v20.4s - add v0.4s, v0.4s, v8.4s - add v2.4s, v2.4s, v6.4s - eor v16.16b, v0.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v7.4s - rev32 v16.8h, v16.8h - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v25.4s - add v1.4s, v1.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v14.4s - eor v8.16b, v1.16b, v8.16b - add v5.4s, v5.4s, v10.4s - rev32 v11.8h, v11.8h - eor v13.16b, v17.16b, v13.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v4.4s, v11.4s - rev32 v13.8h, v13.8h - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v12.4s, v13.4s - add v0.4s, v0.4s, v18.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v14.16b, v12.16b, v14.16b - add v0.4s, v0.4s, v8.4s - add v2.4s, v2.4s, v19.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v16.16b, v0.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v22.4s - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v21.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v14.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v13.16b, v17.16b, v13.16b - add v1.4s, v16.4s, v1.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v13.4s, #8 - shl v13.4s, v13.4s, #24 - eor v8.16b, v1.16b, v8.16b - add v5.4s, v10.4s, v5.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v13.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v14.16b, v12.16b, v14.16b - add v0.4s, v0.4s, v27.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #7 - shl v14.4s, v14.4s, #25 - add v0.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v30.4s - orr v14.16b, v14.16b, v15.16b - eor v13.16b, v0.16b, v13.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v29.4s - rev32 v13.8h, v13.8h - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v28.4s - add v4.4s, v4.4s, v13.4s - rev32 v16.8h, v16.8h - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - eor v9.16b, v4.16b, v9.16b - add v12.4s, v12.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v17.16b, v11.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v1.4s, v10.4s - rev32 v11.8h, v11.8h - ldr q24, [sp, #160] - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v14.16b, v1.16b, v14.16b - add v5.4s, v5.4s, v11.4s - stur q7, [x29, #-64] - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v14.4s, #12 - shl v14.4s, v14.4s, #20 - eor v8.16b, v5.16b, v8.16b - mov v7.16b, v26.16b - add v3.4s, v3.4s, v26.4s - ldur q26, [x29, #-80] - orr v14.16b, v14.16b, v15.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - add v0.4s, v0.4s, v23.4s - orr v8.16b, v8.16b, v15.16b - add v15.4s, v0.4s, v9.4s - add v2.4s, v2.4s, v24.4s - eor v0.16b, v15.16b, v13.16b - add v2.4s, v2.4s, v31.4s - ushr v13.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v14.4s - add v17.4s, v17.4s, v26.4s - orr v0.16b, v0.16b, v13.16b - ushr v13.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - orr v16.16b, v16.16b, v13.16b - ushr v13.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v17.16b, v11.16b - add v4.4s, v0.4s, v4.4s - orr v10.16b, v10.16b, v13.16b - ushr v13.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v9.16b, v4.16b, v9.16b - add v12.4s, v16.4s, v12.4s - orr v11.16b, v11.16b, v13.16b - ushr v13.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v12.16b, v31.16b - orr v9.16b, v9.16b, v13.16b - ushr v13.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - add v1.4s, v10.4s, v1.4s - orr v31.16b, v31.16b, v13.16b - eor v13.16b, v1.16b, v14.16b - add v5.4s, v11.4s, v5.4s - ushr v14.4s, v13.4s, #7 - shl v13.4s, v13.4s, #25 - eor v8.16b, v5.16b, v8.16b - orr v13.16b, v13.16b, v14.16b - ushr v14.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - stur q6, [x29, #-96] - orr v8.16b, v8.16b, v14.16b - add v14.4s, v15.4s, v6.4s - ldur q6, [x29, #-64] - mov v18.16b, v19.16b - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v18.4s - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v21.4s - rev32 v16.8h, v16.8h - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v6.4s - add v1.4s, v1.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v13.4s - eor v8.16b, v1.16b, v8.16b - add v5.4s, v5.4s, v10.4s - rev32 v11.8h, v11.8h - eor v0.16b, v17.16b, v0.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v4.4s, v11.4s - rev32 v0.8h, v0.8h + add v21.4s, v4.4s, v17.4s + rev32 v31.8h, v2.8h + ldr q2, [sp, #80] + rev32 v7.8h, v7.8h + mov v27.16b, v16.16b + eor v17.16b, v21.16b, v28.16b + add v29.4s, v6.4s, v2.4s + ldr q2, [sp, #64] + add v24.4s, v31.4s, v18.4s str q27, [sp, #176] - mov v27.16b, v30.16b - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v12.4s, v0.4s - add v14.4s, v14.4s, v25.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v12.16b, v13.16b - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v27.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #12 - shl v13.4s, v13.4s, #20 - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v20.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v7.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v13.4s - mov v30.16b, v23.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v0.16b, v17.16b, v0.16b - add v1.4s, v16.4s, v1.4s - ldur q23, [x29, #-144] - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - eor v8.16b, v1.16b, v8.16b - add v5.4s, v10.4s, v5.4s - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v0.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v13.16b, v12.16b, v13.16b - add v14.4s, v14.4s, v23.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #7 - shl v13.4s, v13.4s, #25 - add v14.4s, v14.4s, v9.4s - add v2.4s, v2.4s, v29.4s - orr v13.16b, v13.16b, v15.16b - eor v0.16b, v14.16b, v0.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v30.4s - rev32 v0.8h, v0.8h - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v13.4s + ushr v19.4s, v17.4s, #12 + shl v17.4s, v17.4s, #20 + add v30.4s, v7.4s, v2.4s + eor v18.16b, v29.16b, v23.16b + orr v12.16b, v17.16b, v19.16b + eor v17.16b, v30.16b, v13.16b + eor v19.16b, v24.16b, v25.16b + ushr v23.4s, v18.4s, #12 + shl v18.4s, v18.4s, #20 + ushr v25.4s, v17.4s, #12 + shl v17.4s, v17.4s, #20 + ushr v28.4s, v19.4s, #12 + shl v19.4s, v19.4s, #20 + orr v13.16b, v18.16b, v23.16b + orr v25.16b, v17.16b, v25.16b + orr v2.16b, v19.16b, v28.16b + add v28.4s, v0.4s, v12.4s + add v0.4s, v3.4s, v26.4s + add v18.4s, v1.4s, v13.4s + add v3.4s, v5.4s, v16.4s + eor v1.16b, v28.16b, v4.16b + add v17.4s, v0.4s, v25.4s + eor v0.16b, v18.16b, v6.16b + add v19.4s, v3.4s, v2.4s + ushr v16.4s, v1.4s, #8 + shl v3.4s, v1.4s, #24 + eor v4.16b, v17.16b, v7.16b + ushr v6.4s, v0.4s, #8 + shl v1.4s, v0.4s, #24 + eor v5.16b, v19.16b, v31.16b + ushr v23.4s, v4.4s, #8 + shl v4.4s, v4.4s, #24 + orr v7.16b, v3.16b, v16.16b + orr v6.16b, v1.16b, v6.16b + ushr v31.4s, v5.4s, #8 + shl v0.4s, v5.4s, #24 + orr v5.16b, v4.16b, v23.16b + add v4.4s, v7.4s, v21.4s + ldr q21, [sp, #192] + add v3.4s, v6.4s, v29.4s + orr v31.16b, v0.16b, v31.16b + add v23.4s, v5.4s, v30.4s + eor v0.16b, v4.16b, v12.16b + eor v1.16b, v3.16b, v13.16b + add v16.4s, v31.4s, v24.4s + eor v20.16b, v23.16b, v25.16b + ushr v24.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v29.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ushr v30.4s, v20.4s, #7 + shl v20.4s, v20.4s, #25 + orr v25.16b, v0.16b, v24.16b + orr v0.16b, v1.16b, v29.16b + mov v29.16b, v10.16b + orr v1.16b, v20.16b, v30.16b + mov v20.16b, v10.16b + mov v24.16b, v21.16b + ldr q20, [sp, #224] + mov v29.d[1], v8.d[0] + mov v13.16b, v9.16b + zip2 v30.2d, v10.2d, v8.2d + zip2 v8.2d, v21.2d, v20.2d + mov v26.16b, v11.16b + mov v24.d[1], v20.d[0] + add v20.4s, v28.4s, v29.4s + mov v13.d[1], v14.d[0] + str q8, [sp, #128] + eor v2.16b, v16.16b, v2.16b + mov v26.d[1], v15.d[0] + str q24, [sp, #192] + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v13.4s + ushr v12.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + zip2 v10.2d, v9.2d, v14.2d + add v18.4s, v18.4s, v24.4s add v17.4s, v17.4s, v26.4s - add v4.4s, v4.4s, v0.4s - rev32 v16.8h, v16.8h - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - ldur q22, [x29, #-128] - eor v9.16b, v4.16b, v9.16b - add v12.4s, v12.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v17.16b, v11.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v1.4s, v10.4s - rev32 v11.8h, v11.8h - ldr q26, [sp, #176] - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v1.16b, v13.16b - add v5.4s, v5.4s, v11.4s - add v14.4s, v14.4s, v24.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #12 - shl v13.4s, v13.4s, #20 - eor v8.16b, v5.16b, v8.16b - add v14.4s, v14.4s, v9.4s - add v2.4s, v2.4s, v22.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v0.16b, v14.16b, v0.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v28.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v13.4s - add v17.4s, v17.4s, v26.4s - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v17.16b, v11.16b - add v4.4s, v0.4s, v4.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v9.16b, v4.16b, v9.16b - add v12.4s, v16.4s, v12.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v10.4s, v1.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v13.16b, v1.16b, v13.16b - add v5.4s, v11.4s, v5.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #7 - shl v13.4s, v13.4s, #25 - eor v8.16b, v5.16b, v8.16b - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - orr v8.16b, v8.16b, v15.16b - add v14.4s, v14.4s, v18.4s - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v27.4s - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v7.4s - rev32 v16.8h, v16.8h - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s + mov v14.16b, v26.16b + eor v26.16b, v20.16b, v31.16b + stp q10, q30, [sp, #224] + add v19.4s, v19.4s, v25.4s + orr v2.16b, v2.16b, v12.16b + add v18.4s, v18.4s, v1.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + eor v7.16b, v18.16b, v7.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + eor v6.16b, v17.16b, v6.16b + rev32 v7.8h, v7.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + rev32 v6.8h, v6.8h + add v16.4s, v16.4s, v7.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + add v4.4s, v4.4s, v6.4s + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + add v20.4s, v20.4s, v30.4s + zip2 v21.2d, v11.2d, v15.2d + ushr v11.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v10.4s + add v20.4s, v20.4s, v0.4s + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v18.4s, v18.4s, v8.4s + add v19.4s, v19.4s, v25.4s + eor v26.16b, v20.16b, v26.16b + orr v2.16b, v2.16b, v11.16b add v17.4s, v17.4s, v21.4s - add v1.4s, v1.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v13.4s - eor v8.16b, v1.16b, v8.16b - add v5.4s, v5.4s, v10.4s - rev32 v11.8h, v11.8h - eor v0.16b, v17.16b, v0.16b - add v14.4s, v14.4s, v6.4s - ldur q6, [x29, #-96] - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v4.4s, v11.4s - rev32 v0.8h, v0.8h - stur q20, [x29, #-160] - mov v20.16b, v29.16b - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v12.4s, v0.4s - mov v19.16b, v29.16b - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v12.16b, v13.16b - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v20.4s - mov v19.16b, v28.16b - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #12 - shl v13.4s, v13.4s, #20 - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v6.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v19.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v13.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v0.16b, v17.16b, v0.16b - add v1.4s, v16.4s, v1.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - eor v8.16b, v1.16b, v8.16b - add v5.4s, v10.4s, v5.4s - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v0.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v13.16b, v12.16b, v13.16b - add v14.4s, v14.4s, v25.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #7 - shl v13.4s, v13.4s, #25 - add v14.4s, v14.4s, v9.4s - add v2.4s, v2.4s, v30.4s - orr v13.16b, v13.16b, v15.16b - eor v0.16b, v14.16b, v0.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v24.4s - rev32 v0.8h, v0.8h - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v13.4s - add v17.4s, v17.4s, v26.4s - mov v29.16b, v27.16b - add v4.4s, v4.4s, v0.4s - rev32 v16.8h, v16.8h - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - ldur q27, [x29, #-160] - eor v9.16b, v4.16b, v9.16b - add v12.4s, v12.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v17.16b, v11.16b - ldur q6, [x29, #-80] - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v1.4s, v10.4s - rev32 v11.8h, v11.8h - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v1.16b, v13.16b - add v5.4s, v5.4s, v11.4s - add v14.4s, v14.4s, v22.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #12 - shl v13.4s, v13.4s, #20 - eor v8.16b, v5.16b, v8.16b - add v14.4s, v14.4s, v9.4s - add v2.4s, v2.4s, v27.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v0.16b, v14.16b, v0.16b - add v2.4s, v2.4s, v31.4s - add v3.4s, v3.4s, v6.4s - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - eor v16.16b, v2.16b, v16.16b - add v3.4s, v3.4s, v13.4s - add v17.4s, v17.4s, v23.4s - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v3.16b, v10.16b - add v17.4s, v17.4s, v8.4s - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - eor v11.16b, v17.16b, v11.16b - add v4.4s, v0.4s, v4.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v11.4s, #8 - shl v11.4s, v11.4s, #24 - eor v9.16b, v4.16b, v9.16b - add v12.4s, v16.4s, v12.4s - orr v11.16b, v11.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v31.16b, v12.16b, v31.16b - add v1.4s, v10.4s, v1.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - eor v13.16b, v1.16b, v13.16b - add v5.4s, v11.4s, v5.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #7 - shl v13.4s, v13.4s, #25 - eor v8.16b, v5.16b, v8.16b - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - orr v8.16b, v8.16b, v15.16b - add v14.4s, v14.4s, v29.4s - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v20.4s - mov v28.16b, v7.16b - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - add v3.4s, v3.4s, v19.4s - rev32 v16.8h, v16.8h - eor v10.16b, v2.16b, v10.16b - add v3.4s, v3.4s, v31.4s - add v17.4s, v17.4s, v28.4s - add v1.4s, v1.4s, v16.4s - rev32 v10.8h, v10.8h - eor v11.16b, v3.16b, v11.16b - add v17.4s, v17.4s, v13.4s - eor v8.16b, v1.16b, v8.16b - add v5.4s, v5.4s, v10.4s - rev32 v11.8h, v11.8h - eor v0.16b, v17.16b, v0.16b - ushr v15.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - eor v9.16b, v5.16b, v9.16b - add v4.4s, v4.4s, v11.4s - rev32 v0.8h, v0.8h - orr v8.16b, v8.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v31.16b, v4.16b, v31.16b - add v12.4s, v12.4s, v0.4s - add v14.4s, v14.4s, v21.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - eor v13.16b, v12.16b, v13.16b - add v14.4s, v14.4s, v8.4s - add v2.4s, v2.4s, v30.4s - orr v31.16b, v31.16b, v15.16b - ushr v15.4s, v13.4s, #12 - shl v13.4s, v13.4s, #20 - eor v16.16b, v14.16b, v16.16b - add v2.4s, v2.4s, v9.4s - orr v13.16b, v13.16b, v15.16b - ushr v15.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v10.16b, v2.16b, v10.16b - orr v16.16b, v16.16b, v15.16b - ushr v15.4s, v10.4s, #8 - shl v10.4s, v10.4s, #24 - add v3.4s, v3.4s, v18.4s - orr v10.16b, v10.16b, v15.16b - add v15.4s, v3.4s, v31.4s - eor v3.16b, v15.16b, v11.16b - ushr v11.4s, v3.4s, #8 - shl v3.4s, v3.4s, #24 - orr v11.16b, v3.16b, v11.16b - add v3.4s, v17.4s, v6.4s - add v17.4s, v3.4s, v13.4s - eor v0.16b, v17.16b, v0.16b - ushr v3.4s, v0.4s, #8 - shl v0.4s, v0.4s, #24 - add v1.4s, v16.4s, v1.4s - orr v0.16b, v0.16b, v3.16b - eor v3.16b, v1.16b, v8.16b - ushr v8.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - add v5.4s, v10.4s, v5.4s - orr v8.16b, v3.16b, v8.16b - eor v3.16b, v5.16b, v9.16b - add v4.4s, v11.4s, v4.4s - ushr v9.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - eor v31.16b, v4.16b, v31.16b - mov v7.16b, v23.16b - mov v23.16b, v28.16b - mov v28.16b, v6.16b - orr v3.16b, v3.16b, v9.16b - ushr v9.4s, v31.4s, #7 - shl v31.4s, v31.4s, #25 - ldur q6, [x29, #-64] - orr v31.16b, v31.16b, v9.16b - add v9.4s, v0.4s, v12.4s - eor v12.16b, v9.16b, v13.16b - ushr v13.4s, v12.4s, #7 - shl v12.4s, v12.4s, #25 - orr v12.16b, v12.16b, v13.16b - add v13.4s, v14.4s, v6.4s - add v13.4s, v13.4s, v3.4s - eor v0.16b, v13.16b, v0.16b - add v2.4s, v2.4s, v24.4s - rev32 v14.8h, v0.8h - add v0.4s, v2.4s, v31.4s - add v6.4s, v4.4s, v14.4s - eor v2.16b, v0.16b, v16.16b - eor v3.16b, v6.16b, v3.16b - rev32 v16.8h, v2.8h - ushr v4.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - add v2.4s, v9.4s, v16.4s - orr v4.16b, v3.16b, v4.16b - eor v3.16b, v2.16b, v31.16b - ushr v31.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - orr v3.16b, v3.16b, v31.16b - add v31.4s, v15.4s, v22.4s - add v31.4s, v31.4s, v12.4s - add v17.4s, v17.4s, v7.4s - eor v9.16b, v31.16b, v10.16b - add v17.4s, v17.4s, v8.4s - rev32 v9.8h, v9.8h - eor v11.16b, v17.16b, v11.16b - add v1.4s, v1.4s, v9.4s - rev32 v11.8h, v11.8h - eor v10.16b, v1.16b, v12.16b - add v5.4s, v5.4s, v11.4s - ushr v12.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v8.16b, v5.16b, v8.16b - orr v10.16b, v10.16b, v12.16b - ushr v12.4s, v8.4s, #12 - shl v8.4s, v8.4s, #20 - orr v8.16b, v8.16b, v12.16b - add v12.4s, v13.4s, v27.4s - add v12.4s, v12.4s, v4.4s - eor v13.16b, v12.16b, v14.16b - ldur q14, [x29, #-96] - mov v25.16b, v29.16b - add v29.4s, v12.4s, v20.4s - add v20.4s, v31.4s, v26.4s - add v0.4s, v0.4s, v14.4s - add v0.4s, v0.4s, v3.4s - eor v16.16b, v0.16b, v16.16b - add v0.4s, v0.4s, v30.4s - ldur q30, [x29, #-112] - add v20.4s, v20.4s, v10.4s - eor v31.16b, v20.16b, v9.16b - add v20.4s, v20.4s, v28.4s - add v17.4s, v17.4s, v30.4s - add v17.4s, v17.4s, v8.4s - eor v9.16b, v17.16b, v11.16b - ushr v28.4s, v13.4s, #8 - shl v11.4s, v13.4s, #24 - orr v28.16b, v11.16b, v28.16b - ushr v11.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - orr v16.16b, v16.16b, v11.16b - ushr v11.4s, v31.4s, #8 - shl v31.4s, v31.4s, #24 - add v6.4s, v28.4s, v6.4s - orr v31.16b, v31.16b, v11.16b - ushr v11.4s, v9.4s, #8 - shl v9.4s, v9.4s, #24 - add v2.4s, v16.4s, v2.4s - eor v4.16b, v6.16b, v4.16b - orr v9.16b, v9.16b, v11.16b - add v1.4s, v31.4s, v1.4s - eor v3.16b, v2.16b, v3.16b - ushr v11.4s, v4.4s, #7 - shl v4.4s, v4.4s, #25 - add v5.4s, v9.4s, v5.4s - eor v10.16b, v1.16b, v10.16b - orr v4.16b, v4.16b, v11.16b - ushr v11.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - eor v8.16b, v5.16b, v8.16b - orr v3.16b, v3.16b, v11.16b - ushr v11.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - orr v10.16b, v10.16b, v11.16b - ushr v11.4s, v8.4s, #7 - shl v8.4s, v8.4s, #25 - orr v8.16b, v8.16b, v11.16b - add v29.4s, v29.4s, v8.4s - eor v16.16b, v29.16b, v16.16b - add v0.4s, v0.4s, v4.4s - mov v12.16b, v26.16b - add v17.4s, v17.4s, v19.4s - add v26.4s, v29.4s, v23.4s - eor v29.16b, v0.16b, v31.16b - add v20.4s, v20.4s, v3.4s - rev32 v16.8h, v16.8h - stur q18, [x29, #-176] - mov v18.16b, v27.16b - add v0.4s, v0.4s, v24.4s - eor v27.16b, v20.16b, v9.16b - add v17.4s, v17.4s, v10.4s - rev32 v24.8h, v29.8h - add v1.4s, v1.4s, v16.4s + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + ushr v31.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v17.4s, v17.4s, v2.4s + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + orr v26.16b, v26.16b, v31.16b + eor v6.16b, v17.16b, v6.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + add v23.4s, v26.4s, v23.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + eor v0.16b, v23.16b, v0.16b + ldp q28, q12, [x29, #-256] + orr v6.16b, v6.16b, v11.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v4.4s, v6.4s, v4.4s + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + add v18.4s, v18.4s, v12.4s + mov v15.16b, v29.16b + ldur q29, [x29, #-208] + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + str q15, [sp, #160] + add v20.4s, v20.4s, v29.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v27.4s + eor v6.16b, v6.16b, v18.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v28.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + rev32 v6.8h, v6.8h + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + add v3.4s, v3.4s, v6.4s + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + eor v0.16b, v3.16b, v0.16b + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + ushr v11.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v16.4s, v16.4s, v26.4s + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v18.4s, v18.4s, v24.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v20.4s, v20.4s, v22.4s + add v18.4s, v18.4s, v0.4s + mov v9.16b, v30.16b + mov v30.16b, v21.16b + ldur q21, [x29, #-224] + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + str q30, [sp, #144] + add v17.4s, v17.4s, v21.4s + ldur q21, [x29, #-192] + eor v6.16b, v18.16b, v6.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v30.4s + eor v7.16b, v20.16b, v7.16b + add v17.4s, v17.4s, v1.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v19.4s, v19.4s, v2.4s + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + eor v5.16b, v17.16b, v5.16b + orr v6.16b, v6.16b, v11.16b + eor v26.16b, v19.16b, v26.16b + orr v7.16b, v7.16b, v31.16b + ushr v31.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + add v3.4s, v6.4s, v3.4s + ushr v11.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v4.4s, v7.4s, v4.4s + orr v5.16b, v5.16b, v31.16b + eor v0.16b, v3.16b, v0.16b + orr v26.16b, v26.16b, v11.16b + eor v25.16b, v4.16b, v25.16b + add v23.4s, v5.4s, v23.4s + ushr v11.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v16.4s, v26.4s, v16.4s + ushr v31.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v20.4s, v20.4s, v21.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v10.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v18.4s, v18.4s, v14.4s + eor v26.16b, v20.16b, v26.16b + add v19.4s, v19.4s, v25.4s + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v9.4s + ldr q9, [sp, #208] + add v18.4s, v18.4s, v1.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + eor v7.16b, v18.16b, v7.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + eor v6.16b, v17.16b, v6.16b + rev32 v7.8h, v7.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + rev32 v6.8h, v6.8h + add v16.4s, v16.4s, v7.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + add v4.4s, v4.4s, v6.4s + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + add v20.4s, v20.4s, v8.4s + ushr v11.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v15.4s + add v20.4s, v20.4s, v0.4s + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v18.4s, v18.4s, v9.4s + add v19.4s, v19.4s, v25.4s + eor v26.16b, v20.16b, v26.16b + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v13.4s + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + ushr v31.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v17.4s, v17.4s, v2.4s + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + orr v26.16b, v26.16b, v31.16b + eor v6.16b, v17.16b, v6.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + add v23.4s, v26.4s, v23.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + eor v0.16b, v23.16b, v0.16b + orr v6.16b, v6.16b, v11.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v4.4s, v6.4s, v4.4s + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + add v18.4s, v18.4s, v24.4s + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v12.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v30.4s + eor v6.16b, v6.16b, v18.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v27.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + rev32 v6.8h, v6.8h + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + add v3.4s, v3.4s, v6.4s + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + eor v0.16b, v3.16b, v0.16b + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + ushr v11.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v16.4s, v16.4s, v26.4s + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v18.4s, v18.4s, v14.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v20.4s, v20.4s, v28.4s + add v18.4s, v18.4s, v0.4s + mov v10.16b, v13.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v1.16b, v31.16b add v20.4s, v20.4s, v25.4s - eor v25.16b, v17.16b, v28.16b - rev32 v27.8h, v27.8h - add v5.4s, v5.4s, v24.4s - eor v28.16b, v1.16b, v8.16b - rev32 v25.8h, v25.8h - add v6.4s, v6.4s, v27.4s - eor v4.16b, v5.16b, v4.16b - ushr v31.4s, v28.4s, #12 - shl v28.4s, v28.4s, #20 - add v2.4s, v2.4s, v25.4s - eor v3.16b, v6.16b, v3.16b - orr v28.16b, v28.16b, v31.16b - ushr v31.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - eor v29.16b, v2.16b, v10.16b - orr v4.16b, v4.16b, v31.16b - ushr v31.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - add v26.4s, v26.4s, v28.4s - orr v3.16b, v3.16b, v31.16b - ushr v31.4s, v29.4s, #12 - shl v29.4s, v29.4s, #20 - eor v16.16b, v26.16b, v16.16b - add v0.4s, v0.4s, v4.4s - add v17.4s, v17.4s, v12.4s - orr v29.16b, v29.16b, v31.16b - eor v24.16b, v0.16b, v24.16b - add v0.4s, v0.4s, v22.4s - add v20.4s, v20.4s, v3.4s - ushr v22.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - add v23.4s, v26.4s, v21.4s - eor v21.16b, v20.16b, v27.16b add v17.4s, v17.4s, v29.4s - orr v16.16b, v16.16b, v22.16b - ushr v22.4s, v24.4s, #8 - shl v24.4s, v24.4s, #24 - eor v25.16b, v17.16b, v25.16b - orr v22.16b, v24.16b, v22.16b + eor v6.16b, v18.16b, v6.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v10.4s + eor v7.16b, v20.16b, v7.16b + add v17.4s, v17.4s, v1.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v19.4s, v19.4s, v2.4s + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + eor v5.16b, v17.16b, v5.16b + orr v6.16b, v6.16b, v11.16b + eor v26.16b, v19.16b, v26.16b + orr v7.16b, v7.16b, v31.16b + ushr v31.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + add v3.4s, v6.4s, v3.4s + ushr v11.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v4.4s, v7.4s, v4.4s + orr v5.16b, v5.16b, v31.16b + eor v0.16b, v3.16b, v0.16b + mov v22.16b, v8.16b + ldp q8, q28, [sp, #240] + orr v26.16b, v26.16b, v11.16b + eor v25.16b, v4.16b, v25.16b + add v23.4s, v5.4s, v23.4s + ushr v11.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v16.4s, v26.4s, v16.4s + ushr v31.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v20.4s, v20.4s, v28.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v15.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v18.4s, v18.4s, v8.4s + eor v26.16b, v20.16b, v26.16b + add v19.4s, v19.4s, v25.4s + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v22.4s + ldur q22, [x29, #-256] + add v18.4s, v18.4s, v1.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + eor v7.16b, v18.16b, v7.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + eor v6.16b, v17.16b, v6.16b + rev32 v7.8h, v7.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + rev32 v6.8h, v6.8h + add v16.4s, v16.4s, v7.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + add v4.4s, v4.4s, v6.4s + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + add v20.4s, v20.4s, v9.4s + mov v13.16b, v12.16b + mov v12.16b, v27.16b + mov v27.16b, v9.16b + ldur q9, [x29, #-192] + mov v21.16b, v15.16b + ldr q15, [sp, #224] + ushr v11.4s, v1.4s, #12 + ldur q21, [x29, #-224] + shl v1.4s, v1.4s, #20 + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v9.4s + add v20.4s, v20.4s, v0.4s + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v18.4s, v18.4s, v21.4s + add v19.4s, v19.4s, v25.4s + eor v26.16b, v20.16b, v26.16b + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v15.4s + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + ushr v31.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v17.4s, v17.4s, v2.4s + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + orr v26.16b, v26.16b, v31.16b + eor v6.16b, v17.16b, v6.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + add v23.4s, v26.4s, v23.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + eor v0.16b, v23.16b, v0.16b + orr v6.16b, v6.16b, v11.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v4.4s, v6.4s, v4.4s + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + add v18.4s, v18.4s, v14.4s + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v24.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v10.4s + eor v6.16b, v6.16b, v18.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v30.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + rev32 v6.8h, v6.8h + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + add v3.4s, v3.4s, v6.4s + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + eor v0.16b, v3.16b, v0.16b + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + ushr v11.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v16.4s, v16.4s, v26.4s + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v18.4s, v18.4s, v8.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v20.4s, v20.4s, v12.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v13.4s + ldr q13, [sp, #160] + eor v6.16b, v18.16b, v6.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v15.4s + eor v7.16b, v20.16b, v7.16b + add v17.4s, v17.4s, v1.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v19.4s, v19.4s, v2.4s + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + eor v5.16b, v17.16b, v5.16b + orr v6.16b, v6.16b, v11.16b + eor v26.16b, v19.16b, v26.16b + orr v7.16b, v7.16b, v31.16b + ushr v31.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + add v3.4s, v6.4s, v3.4s + ushr v11.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v4.4s, v7.4s, v4.4s + orr v5.16b, v5.16b, v31.16b + eor v0.16b, v3.16b, v0.16b + orr v26.16b, v26.16b, v11.16b + eor v25.16b, v4.16b, v25.16b + add v23.4s, v5.4s, v23.4s + ushr v11.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v16.4s, v26.4s, v16.4s + ushr v31.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v20.4s, v20.4s, v22.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v9.4s + mov v29.16b, v14.16b + ldr q14, [sp, #128] + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v18.4s, v18.4s, v14.4s + eor v26.16b, v20.16b, v26.16b + add v19.4s, v19.4s, v25.4s + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v27.4s + add v18.4s, v18.4s, v1.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + eor v7.16b, v18.16b, v7.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + eor v6.16b, v17.16b, v6.16b + rev32 v7.8h, v7.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + rev32 v6.8h, v6.8h + add v16.4s, v16.4s, v7.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + add v4.4s, v4.4s, v6.4s + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + add v20.4s, v20.4s, v21.4s + ushr v11.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v28.4s + add v20.4s, v20.4s, v0.4s + mov v12.16b, v27.16b + ldur q27, [x29, #-208] + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v18.4s, v18.4s, v27.4s + add v19.4s, v19.4s, v25.4s + eor v26.16b, v20.16b, v26.16b + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v13.4s + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + ushr v31.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v17.4s, v17.4s, v2.4s + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + orr v26.16b, v26.16b, v31.16b + eor v6.16b, v17.16b, v6.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + add v23.4s, v26.4s, v23.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + eor v0.16b, v23.16b, v0.16b + orr v6.16b, v6.16b, v11.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v4.4s, v6.4s, v4.4s + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + add v18.4s, v18.4s, v8.4s + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v29.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v15.4s + eor v6.16b, v6.16b, v18.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v10.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + rev32 v6.8h, v6.8h + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + add v3.4s, v3.4s, v6.4s + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + eor v0.16b, v3.16b, v0.16b + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + ushr v11.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v16.4s, v16.4s, v26.4s + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v18.4s, v18.4s, v14.4s + mov v30.16b, v29.16b + mov v29.16b, v15.16b + ldr q15, [sp, #144] + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v20.4s, v20.4s, v15.4s + add v18.4s, v18.4s, v0.4s + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v24.4s + eor v6.16b, v18.16b, v6.16b + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v13.4s + eor v7.16b, v20.16b, v7.16b + add v17.4s, v17.4s, v1.4s + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v19.4s, v19.4s, v2.4s + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + eor v5.16b, v17.16b, v5.16b + orr v6.16b, v6.16b, v11.16b + eor v26.16b, v19.16b, v26.16b + orr v7.16b, v7.16b, v31.16b + ushr v31.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + add v3.4s, v6.4s, v3.4s + ushr v11.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v4.4s, v7.4s, v4.4s + orr v5.16b, v5.16b, v31.16b + eor v0.16b, v3.16b, v0.16b + orr v26.16b, v26.16b, v11.16b + eor v25.16b, v4.16b, v25.16b + add v23.4s, v5.4s, v23.4s + ushr v11.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + mov v9.16b, v28.16b + mov v28.16b, v10.16b + ldr q10, [sp, #176] + add v16.4s, v26.4s, v16.4s + ushr v31.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v23.16b, v1.16b + orr v0.16b, v0.16b, v11.16b + add v20.4s, v20.4s, v10.4s + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v9.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v18.4s, v18.4s, v12.4s + eor v26.16b, v20.16b, v26.16b + add v19.4s, v19.4s, v25.4s + orr v2.16b, v2.16b, v11.16b + add v17.4s, v17.4s, v21.4s + add v18.4s, v18.4s, v1.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + eor v7.16b, v18.16b, v7.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + eor v6.16b, v17.16b, v6.16b + rev32 v7.8h, v7.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + rev32 v6.8h, v6.8h + add v16.4s, v16.4s, v7.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + add v4.4s, v4.4s, v6.4s + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + ushr v11.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + eor v2.16b, v4.16b, v2.16b + add v20.4s, v20.4s, v27.4s + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v22.4s + mov v9.16b, v22.16b + ldur q22, [x29, #-240] + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v20.4s, v20.4s, v0.4s + add v18.4s, v18.4s, v22.4s + add v19.4s, v19.4s, v25.4s + mov v24.16b, v21.16b + ldur q21, [x29, #-192] + orr v2.16b, v2.16b, v11.16b + eor v26.16b, v20.16b, v26.16b + add v17.4s, v17.4s, v21.4s + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + ushr v31.4s, v26.4s, #8 + add v17.4s, v17.4s, v2.4s + shl v26.4s, v26.4s, #24 + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + orr v26.16b, v26.16b, v31.16b + eor v6.16b, v17.16b, v6.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v23.4s, v26.4s, v23.4s + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + orr v6.16b, v6.16b, v11.16b + eor v0.16b, v23.16b, v0.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + add v4.4s, v6.4s, v4.4s + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + orr v0.16b, v0.16b, v31.16b + eor v2.16b, v4.16b, v2.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v8.4s + add v18.4s, v18.4s, v14.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v13.4s + add v18.4s, v18.4s, v0.4s + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v29.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + eor v6.16b, v6.16b, v18.16b + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + rev32 v6.8h, v6.8h + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + add v3.4s, v3.4s, v6.4s + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + eor v0.16b, v3.16b, v0.16b + add v16.4s, v16.4s, v26.4s + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + ushr v11.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v1.16b, v23.16b, v1.16b + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + orr v0.16b, v0.16b, v11.16b + ushr v31.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v20.4s, v20.4s, v28.4s + add v18.4s, v18.4s, v12.4s + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + orr v1.16b, v1.16b, v31.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v30.4s + add v18.4s, v18.4s, v0.4s + orr v2.16b, v2.16b, v11.16b + add v19.4s, v19.4s, v21.4s + eor v7.16b, v20.16b, v7.16b + add v17.4s, v17.4s, v1.4s + eor v6.16b, v18.16b, v6.16b + add v19.4s, v19.4s, v2.4s + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + eor v5.16b, v17.16b, v5.16b + orr v7.16b, v7.16b, v31.16b + eor v26.16b, v19.16b, v26.16b + orr v6.16b, v6.16b, v11.16b + ushr v31.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + ushr v11.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + add v4.4s, v7.4s, v4.4s + orr v5.16b, v5.16b, v31.16b + add v3.4s, v6.4s, v3.4s + orr v26.16b, v26.16b, v11.16b + eor v25.16b, v4.16b, v25.16b + add v23.4s, v5.4s, v23.4s + eor v0.16b, v3.16b, v0.16b + add v16.4s, v26.4s, v16.4s + ushr v31.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + ushr v11.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + eor v1.16b, v23.16b, v1.16b + orr v25.16b, v25.16b, v31.16b + eor v2.16b, v16.16b, v2.16b + orr v0.16b, v0.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v20.4s, v20.4s, v15.4s + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v1.16b, v1.16b, v31.16b + add v18.4s, v18.4s, v24.4s + add v20.4s, v20.4s, v0.4s + add v19.4s, v19.4s, v9.4s + mov v8.16b, v13.16b + ldur q13, [x29, #-208] + orr v2.16b, v2.16b, v11.16b + add v18.4s, v18.4s, v1.4s + add v17.4s, v17.4s, v13.4s + eor v26.16b, v20.16b, v26.16b + add v19.4s, v19.4s, v25.4s + eor v7.16b, v18.16b, v7.16b + add v17.4s, v17.4s, v2.4s + rev32 v26.8h, v26.8h + eor v5.16b, v19.16b, v5.16b + rev32 v7.8h, v7.8h + eor v6.16b, v17.16b, v6.16b + add v23.4s, v23.4s, v26.4s + rev32 v5.8h, v5.8h + add v16.4s, v16.4s, v7.4s + rev32 v6.8h, v6.8h + eor v0.16b, v23.16b, v0.16b + add v3.4s, v3.4s, v5.4s + eor v1.16b, v16.16b, v1.16b + add v4.4s, v4.4s, v6.4s + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v25.16b, v3.16b, v25.16b + ushr v11.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + orr v0.16b, v0.16b, v31.16b + eor v2.16b, v4.16b, v2.16b + ushr v31.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + orr v1.16b, v1.16b, v11.16b + ushr v11.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v20.4s, v20.4s, v22.4s + orr v25.16b, v25.16b, v31.16b + add v19.4s, v19.4s, v10.4s + mov v27.16b, v12.16b + mov v12.16b, v30.16b + mov v29.16b, v21.16b + mov v21.16b, v24.16b + ldr q24, [sp, #192] + mov v30.16b, v22.16b + ldr q22, [sp, #256] + orr v2.16b, v2.16b, v11.16b + add v20.4s, v20.4s, v0.4s + add v18.4s, v18.4s, v24.4s + add v19.4s, v19.4s, v25.4s + add v17.4s, v17.4s, v22.4s + eor v26.16b, v20.16b, v26.16b + add v18.4s, v18.4s, v1.4s + eor v5.16b, v19.16b, v5.16b + add v17.4s, v17.4s, v2.4s + ushr v31.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + ushr v11.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + eor v7.16b, v18.16b, v7.16b + eor v6.16b, v17.16b, v6.16b + orr v26.16b, v26.16b, v31.16b + orr v5.16b, v5.16b, v11.16b + ushr v31.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + ushr v11.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + add v23.4s, v26.4s, v23.4s + orr v7.16b, v7.16b, v31.16b + add v3.4s, v5.4s, v3.4s + orr v6.16b, v6.16b, v11.16b + eor v0.16b, v23.16b, v0.16b + add v16.4s, v7.4s, v16.4s + eor v25.16b, v3.16b, v25.16b + add v4.4s, v6.4s, v4.4s + ushr v31.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v11.4s, v25.4s, #7 + shl v25.4s, v25.4s, #25 + eor v1.16b, v16.16b, v1.16b + eor v2.16b, v4.16b, v2.16b + orr v0.16b, v0.16b, v31.16b + orr v25.16b, v25.16b, v11.16b + ushr v31.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ushr v11.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + add v20.4s, v20.4s, v14.4s + add v18.4s, v18.4s, v27.4s + ldr q27, [sp, #224] + orr v1.16b, v1.16b, v31.16b + orr v2.16b, v2.16b, v11.16b + add v20.4s, v20.4s, v25.4s + add v17.4s, v17.4s, v29.4s + add v18.4s, v18.4s, v0.4s + add v19.4s, v19.4s, v8.4s + eor v7.16b, v7.16b, v20.16b + add v17.4s, v17.4s, v1.4s + eor v6.16b, v6.16b, v18.16b + add v19.4s, v19.4s, v2.4s + rev32 v7.8h, v7.8h + eor v5.16b, v17.16b, v5.16b + rev32 v6.8h, v6.8h + eor v26.16b, v19.16b, v26.16b + add v4.4s, v4.4s, v7.4s + rev32 v5.8h, v5.8h + add v3.4s, v3.4s, v6.4s + rev32 v26.8h, v26.8h + eor v25.16b, v4.16b, v25.16b + add v23.4s, v23.4s, v5.4s + eor v0.16b, v3.16b, v0.16b + add v16.4s, v16.4s, v26.4s + ushr v29.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + ushr v31.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v1.16b, v23.16b, v1.16b + eor v2.16b, v16.16b, v2.16b + orr v25.16b, v25.16b, v29.16b + orr v0.16b, v0.16b, v31.16b + ushr v29.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + ushr v31.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + add v18.4s, v18.4s, v21.4s + ldr q21, [sp, #240] + add v20.4s, v20.4s, v27.4s + prfm pldl1keep, [x17, #256] + orr v1.16b, v1.16b, v29.16b + prfm pldl1keep, [x21, #256] + orr v2.16b, v2.16b, v31.16b + prfm pldl1keep, [x16, #256] + add v18.4s, v18.4s, v0.4s + prfm pldl1keep, [x6, #256] + add v17.4s, v17.4s, v21.4s + add v19.4s, v19.4s, v22.4s + add v20.4s, v20.4s, v25.4s + eor v6.16b, v18.16b, v6.16b + add v17.4s, v17.4s, v1.4s + add v19.4s, v19.4s, v2.4s + eor v7.16b, v20.16b, v7.16b + ushr v22.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + eor v5.16b, v17.16b, v5.16b + eor v26.16b, v19.16b, v26.16b + ushr v21.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + orr v6.16b, v6.16b, v22.16b + ushr v22.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 + ushr v29.4s, v26.4s, #8 + shl v26.4s, v26.4s, #24 + orr v7.16b, v7.16b, v21.16b + orr v5.16b, v5.16b, v22.16b + add v3.4s, v6.4s, v3.4s + orr v21.16b, v26.16b, v29.16b + add v4.4s, v7.4s, v4.4s + add v22.4s, v5.4s, v23.4s + eor v0.16b, v3.16b, v0.16b + add v16.4s, v21.4s, v16.4s + eor v23.16b, v4.16b, v25.16b + eor v1.16b, v22.16b, v1.16b + ushr v25.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + eor v2.16b, v16.16b, v2.16b + ushr v26.4s, v23.4s, #7 + shl v23.4s, v23.4s, #25 + orr v0.16b, v0.16b, v25.16b + ushr v25.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ushr v29.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + add v20.4s, v20.4s, v28.4s + orr v23.16b, v23.16b, v26.16b + orr v1.16b, v1.16b, v25.16b + orr v2.16b, v2.16b, v29.16b + add v20.4s, v20.4s, v0.4s + add v18.4s, v18.4s, v13.4s + add v17.4s, v17.4s, v30.4s + add v19.4s, v19.4s, v10.4s + eor v21.16b, v20.16b, v21.16b + add v18.4s, v18.4s, v1.4s + add v17.4s, v17.4s, v2.4s + add v19.4s, v19.4s, v23.4s + rev32 v21.8h, v21.8h + eor v7.16b, v18.16b, v7.16b + eor v6.16b, v17.16b, v6.16b + eor v5.16b, v19.16b, v5.16b + add v22.4s, v22.4s, v21.4s + rev32 v7.8h, v7.8h + rev32 v6.8h, v6.8h + rev32 v5.8h, v5.8h + eor v0.16b, v22.16b, v0.16b + add v16.4s, v16.4s, v7.4s + add v4.4s, v4.4s, v6.4s + add v3.4s, v3.4s, v5.4s + ushr v25.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v1.16b, v16.16b, v1.16b + eor v2.16b, v4.16b, v2.16b + eor v23.16b, v3.16b, v23.16b + orr v0.16b, v0.16b, v25.16b + ushr v25.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + ushr v26.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + ushr v27.4s, v23.4s, #12 + shl v23.4s, v23.4s, #20 + orr v1.16b, v1.16b, v25.16b + add v20.4s, v20.4s, v24.4s + orr v2.16b, v2.16b, v26.16b + orr v23.16b, v23.16b, v27.16b + add v18.4s, v18.4s, v12.4s + add v17.4s, v17.4s, v9.4s + add v19.4s, v19.4s, v15.4s + add v20.4s, v20.4s, v0.4s + add v18.4s, v18.4s, v1.4s + add v17.4s, v17.4s, v2.4s + add v19.4s, v19.4s, v23.4s + eor v21.16b, v20.16b, v21.16b + eor v7.16b, v18.16b, v7.16b + eor v6.16b, v17.16b, v6.16b + eor v5.16b, v19.16b, v5.16b ushr v24.4s, v21.4s, #8 shl v21.4s, v21.4s, #24 + ushr v25.4s, v7.4s, #8 + shl v7.4s, v7.4s, #24 + ushr v26.4s, v6.4s, #8 + shl v6.4s, v6.4s, #24 + ushr v27.4s, v5.4s, #8 + shl v5.4s, v5.4s, #24 orr v21.16b, v21.16b, v24.16b - ushr v24.4s, v25.4s, #8 - shl v25.4s, v25.4s, #24 - add v1.4s, v16.4s, v1.4s - orr v24.16b, v25.16b, v24.16b - add v5.4s, v22.4s, v5.4s - eor v25.16b, v1.16b, v28.16b - add v6.4s, v21.4s, v6.4s - eor v4.16b, v5.16b, v4.16b - ushr v27.4s, v25.4s, #7 - shl v25.4s, v25.4s, #25 - add v2.4s, v24.4s, v2.4s - eor v3.16b, v6.16b, v3.16b - orr v25.16b, v25.16b, v27.16b - ushr v27.4s, v4.4s, #7 - shl v4.4s, v4.4s, #25 - ldur q19, [x29, #-176] - eor v26.16b, v2.16b, v29.16b - orr v4.16b, v4.16b, v27.16b - ushr v27.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - orr v3.16b, v3.16b, v27.16b - ushr v27.4s, v26.4s, #7 - shl v26.4s, v26.4s, #25 - add v20.4s, v20.4s, v18.4s - add v17.4s, v17.4s, v30.4s - orr v26.16b, v26.16b, v27.16b - add v0.4s, v0.4s, v3.4s - eor v16.16b, v0.16b, v16.16b - add v0.4s, v0.4s, v19.4s - add v19.4s, v20.4s, v26.4s - add v17.4s, v17.4s, v25.4s - eor v20.16b, v19.16b, v22.16b - add v7.4s, v19.4s, v7.4s - eor v19.16b, v17.16b, v21.16b - ldur q21, [x29, #-64] - add v23.4s, v23.4s, v4.4s - eor v24.16b, v23.16b, v24.16b - rev32 v16.8h, v16.8h - add v17.4s, v17.4s, v21.4s - rev32 v21.8h, v24.8h - add v6.4s, v6.4s, v21.4s - rev32 v20.8h, v20.8h - add v2.4s, v2.4s, v16.4s - eor v4.16b, v6.16b, v4.16b - rev32 v19.8h, v19.8h - add v1.4s, v1.4s, v20.4s - eor v3.16b, v2.16b, v3.16b - ushr v24.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - add v5.4s, v5.4s, v19.4s - eor v22.16b, v1.16b, v26.16b - orr v4.16b, v4.16b, v24.16b - ushr v24.4s, v3.4s, #12 - shl v3.4s, v3.4s, #20 - add v18.4s, v23.4s, v14.4s - eor v23.16b, v5.16b, v25.16b - orr v3.16b, v3.16b, v24.16b - ushr v24.4s, v22.4s, #12 - shl v22.4s, v22.4s, #20 - orr v22.16b, v22.16b, v24.16b - ushr v24.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - orr v23.16b, v23.16b, v24.16b - add v18.4s, v18.4s, v4.4s - add v0.4s, v0.4s, v3.4s - add v24.4s, v17.4s, v23.4s - eor v17.16b, v18.16b, v21.16b - add v7.4s, v7.4s, v22.4s - eor v16.16b, v0.16b, v16.16b - ushr v21.4s, v17.4s, #8 - shl v17.4s, v17.4s, #24 - eor v20.16b, v7.16b, v20.16b - orr v21.16b, v17.16b, v21.16b - ushr v17.4s, v16.4s, #8 - shl v16.4s, v16.4s, #24 - eor v19.16b, v24.16b, v19.16b - orr v16.16b, v16.16b, v17.16b - ushr v17.4s, v20.4s, #8 - shl v20.4s, v20.4s, #24 - orr v25.16b, v20.16b, v17.16b - ushr v17.4s, v19.4s, #8 - shl v19.4s, v19.4s, #24 - orr v19.16b, v19.16b, v17.16b - add v1.4s, v25.4s, v1.4s - eor v22.16b, v1.16b, v22.16b - eor v20.16b, v1.16b, v18.16b - add v1.4s, v19.4s, v5.4s - eor v26.16b, v1.16b, v0.16b - add v0.4s, v21.4s, v6.4s - eor v5.16b, v1.16b, v23.16b - eor v1.16b, v0.16b, v4.16b - eor v17.16b, v0.16b, v7.16b - add v0.4s, v16.4s, v2.4s - eor v2.16b, v0.16b, v3.16b - eor v6.16b, v0.16b, v24.16b - ushr v0.4s, v1.4s, #7 + orr v7.16b, v7.16b, v25.16b + orr v6.16b, v6.16b, v26.16b + orr v5.16b, v5.16b, v27.16b + add v22.4s, v21.4s, v22.4s + add v16.4s, v7.4s, v16.4s + add v4.4s, v6.4s, v4.4s + add v3.4s, v5.4s, v3.4s + eor v0.16b, v22.16b, v0.16b + eor v1.16b, v16.16b, v1.16b + eor v2.16b, v4.16b, v2.16b + eor v23.16b, v3.16b, v23.16b + ushr v24.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v25.4s, v1.4s, #7 shl v1.4s, v1.4s, #25 - orr v0.16b, v1.16b, v0.16b - ushr v1.4s, v2.4s, #7 + ushr v26.4s, v2.4s, #7 shl v2.4s, v2.4s, #25 - orr v1.16b, v2.16b, v1.16b - ushr v2.4s, v22.4s, #7 - shl v3.4s, v22.4s, #25 - orr v2.16b, v3.16b, v2.16b - ushr v3.4s, v5.4s, #7 - shl v4.4s, v5.4s, #25 - orr v3.16b, v4.16b, v3.16b - eor v8.16b, v16.16b, v3.16b - eor v9.16b, v25.16b, v0.16b - eor v31.16b, v1.16b, v19.16b - cmp x17, x22 - eor v15.16b, v2.16b, v21.16b - mov w18, w19 - b.ne .LBB2_4 -.LBB2_7: - zip1 v0.4s, v20.4s, v26.4s - zip2 v1.4s, v20.4s, v26.4s - zip1 v2.4s, v17.4s, v6.4s - zip2 v3.4s, v17.4s, v6.4s - zip1 v4.4s, v8.4s, v9.4s - zip2 v5.4s, v8.4s, v9.4s - zip1 v6.4s, v31.4s, v15.4s - zip2 v7.4s, v31.4s, v15.4s - add x13, x20, #4 - tst w5, #0x1 - sub x28, x28, #4 - zip1 v16.2d, v0.2d, v2.2d - zip2 v0.2d, v0.2d, v2.2d - zip1 v2.2d, v1.2d, v3.2d - zip2 v1.2d, v1.2d, v3.2d - zip1 v3.2d, v4.2d, v6.2d - zip2 v4.2d, v4.2d, v6.2d - zip1 v6.2d, v5.2d, v7.2d - zip2 v5.2d, v5.2d, v7.2d - add x24, x24, #32 - csel x20, x13, x20, ne - cmp x28, #3 - stp q16, q3, [x26] - stp q0, q4, [x26, #32] - stp q2, q6, [x26, #64] - stp q1, q5, [x26, #96] - add x26, x26, #128 - b.hi .LBB2_2 -.LBB2_8: - cbz x28, .LBB2_16 + ushr v27.4s, v23.4s, #7 + shl v23.4s, v23.4s, #25 + orr v0.16b, v0.16b, v24.16b + orr v1.16b, v1.16b, v25.16b + orr v2.16b, v2.16b, v26.16b + orr v23.16b, v23.16b, v27.16b + movi v24.4s, #64 + eor v12.16b, v4.16b, v20.16b + eor v31.16b, v18.16b, v3.16b + eor v29.16b, v17.16b, v22.16b + eor v30.16b, v16.16b, v19.16b + eor v28.16b, v7.16b, v23.16b + eor v23.16b, v6.16b, v0.16b + eor v13.16b, v1.16b, v5.16b + eor v25.16b, v2.16b, v21.16b + cbnz x15, .LBB3_5 + b .LBB3_2 +.LBB3_6: + cbz x24, .LBB3_14 orr w8, w7, w19 - and x21, x5, #0x1 - stur w8, [x29, #-64] -.LBB2_10: + and x22, x5, #0x1 + stur w8, [x29, #-192] +.LBB3_8: ldr x8, [sp, #40] - ldr x25, [x24] - ldur w4, [x29, #-64] - ldp q1, q0, [x8] - mov x8, x22 - stp q1, q0, [x29, #-48] -.LBB2_11: - subs x23, x8, #1 - b.eq .LBB2_13 - cbnz x8, .LBB2_14 - b .LBB2_15 -.LBB2_13: - orr w4, w4, w27 -.LBB2_14: - sub x0, x29, #48 - mov w2, #64 - mov x1, x25 - mov x3, x20 - bl zfs_blake3_compress_in_place_sse2 + mov x28, x0 + ldr x25, [x0] + mov x23, x2 + ldur w5, [x29, #-192] + ldp q0, q1, [x8] + mov x8, x2 + b .LBB3_11 +.LBB3_9: + orr w5, w5, w27 +.LBB3_10: + sub x0, x29, #144 + sub x1, x29, #176 + mov x2, x25 + mov w3, #64 + mov x4, x20 + bl compress_pre + ldp q0, q1, [x29, #-144] add x25, x25, #64 - mov x8, x23 - mov w4, w19 - b .LBB2_11 -.LBB2_15: - ldp q0, q1, [x29, #-48] - add x20, x20, x21 - add x24, x24, #8 - subs x28, x28, #1 - stp q0, q1, [x26], #32 - b.ne .LBB2_10 -.LBB2_16: - add sp, sp, #384 + mov x8, x21 + mov w5, w19 + ldp q2, q3, [x29, #-112] + eor v0.16b, v2.16b, v0.16b + eor v1.16b, v3.16b, v1.16b +.LBB3_11: + subs x21, x8, #1 + stp q0, q1, [x29, #-176] + b.eq .LBB3_9 + cbnz x8, .LBB3_10 + ldp q1, q0, [x29, #-176] + mov x0, x28 + add x20, x20, x22 + add x0, x28, #8 + subs x24, x24, #1 + mov x2, x23 + stp q1, q0, [x26], #32 + b.ne .LBB3_8 +.LBB3_14: + add sp, sp, #464 ldp x20, x19, [sp, #144] ldp x22, x21, [sp, #128] ldp x24, x23, [sp, #112] @@ -2442,9 +2052,10 @@ zfs_blake3_hash_many_sse2: ldp d11, d10, [sp, #32] ldp d13, d12, [sp, #16] ldp d15, d14, [sp], #160 + hint #29 ret -.Lfunc_end2: - .size zfs_blake3_hash_many_sse2, .Lfunc_end2-zfs_blake3_hash_many_sse2 +.Lfunc_end3: + .size zfs_blake3_hash_many_sse2, .Lfunc_end3-zfs_blake3_hash_many_sse2 .cfi_endproc .section ".note.GNU-stack","",@progbits -#endif +#endif \ No newline at end of file diff --git a/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S b/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S index a05baec96ce5..c4c2dfc5bcde 100644 --- a/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S +++ b/module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S @@ -22,518 +22,61 @@ /* * Based on BLAKE3 v1.3.1, https://github.com/BLAKE3-team/BLAKE3 * Copyright (c) 2019-2022 Samuel Neves - * Copyright (c) 2022 Tino Reichardt + * Copyright (c) 2022-2023 Tino Reichardt * * This is converted assembly: SSE4.1 -> ARMv8-A * Used tools: SIMDe https://github.com/simd-everywhere/simde + * + * Should work on FreeBSD, Linux and macOS + * see: https://github.com/mcmilk/BLAKE3-tests/blob/master/contrib/simde.sh */ #if defined(__aarch64__) .text - .section .rodata.cst16,"aM",@progbits,16 - .p2align 4 -.LCPI0_0: - .byte 2 - .byte 3 - .byte 0 - .byte 1 - .byte 6 - .byte 7 - .byte 4 - .byte 5 - .byte 10 - .byte 11 - .byte 8 - .byte 9 - .byte 14 - .byte 15 - .byte 12 - .byte 13 -.LCPI0_1: - .word 1779033703 - .word 3144134277 - .word 1013904242 - .word 2773480762 -.LCPI0_2: - .byte 1 - .byte 2 - .byte 3 - .byte 0 - .byte 5 - .byte 6 - .byte 7 - .byte 4 - .byte 9 - .byte 10 - .byte 11 - .byte 8 - .byte 13 - .byte 14 - .byte 15 - .byte 12 -.LCPI0_3: - .byte 0 - .byte 1 - .byte 2 - .byte 3 - .byte 20 - .byte 21 - .byte 22 - .byte 23 - .byte 8 - .byte 9 - .byte 10 - .byte 11 - .byte 28 - .byte 29 - .byte 30 - .byte 31 -.LCPI0_4: - .byte 0 - .byte 1 - .byte 2 - .byte 3 - .byte 4 - .byte 5 - .byte 6 - .byte 7 - .byte 8 - .byte 9 - .byte 10 - .byte 11 - .byte 28 - .byte 29 - .byte 30 - .byte 31 + .section .note.gnu.property,"a",@note + .p2align 3 + .word 4 + .word 16 + .word 5 + .asciz "GNU" + .word 3221225472 + .word 4 + .word 3 + .word 0 +.Lsec_end0: .text .globl zfs_blake3_compress_in_place_sse41 .p2align 2 .type zfs_blake3_compress_in_place_sse41,@function zfs_blake3_compress_in_place_sse41: .cfi_startproc - ldp q7, q6, [x0] - ldp q17, q18, [x1] - add x12, x1, #32 - ld2 { v4.4s, v5.4s }, [x12] - lsr x10, x3, #32 - fmov s16, w3 - adrp x13, .LCPI0_0 - adrp x11, .LCPI0_1 - and w8, w2, #0xff - mov v16.s[1], w10 - ldr q0, [x13, :lo12:.LCPI0_0] - ldr q20, [x11, :lo12:.LCPI0_1] - adrp x11, .LCPI0_4 - and w9, w4, #0xff - ldr q2, [x11, :lo12:.LCPI0_4] - mov v16.s[2], w8 - uzp1 v21.4s, v17.4s, v18.4s - add v7.4s, v6.4s, v7.4s - adrp x12, .LCPI0_3 - mov v16.s[3], w9 - uzp2 v18.4s, v17.4s, v18.4s - add v7.4s, v7.4s, v21.4s - ext v17.16b, v5.16b, v5.16b, #12 - ldr q3, [x12, :lo12:.LCPI0_3] - ext v24.16b, v4.16b, v4.16b, #12 - eor v16.16b, v7.16b, v16.16b - mov v27.16b, v17.16b - uzp1 v19.4s, v21.4s, v21.4s - ext v25.16b, v21.16b, v21.16b, #12 - zip2 v28.4s, v18.4s, v17.4s - tbl v29.16b, { v16.16b }, v0.16b - mov v27.s[1], v24.s[2] - zip1 v23.2d, v17.2d, v18.2d - ext v19.16b, v19.16b, v21.16b, #8 - add v22.4s, v29.4s, v20.4s - ext v26.16b, v21.16b, v25.16b, #12 - tbl v20.16b, { v23.16b, v24.16b }, v2.16b - zip1 v21.4s, v28.4s, v24.4s - zip1 v23.4s, v24.4s, v28.4s - uzp2 v19.4s, v19.4s, v18.4s - eor v24.16b, v22.16b, v6.16b - ext v25.16b, v20.16b, v20.16b, #12 - ext v6.16b, v23.16b, v21.16b, #8 - add v7.4s, v7.4s, v18.4s - ext v18.16b, v19.16b, v19.16b, #4 - tbl v16.16b, { v26.16b, v27.16b }, v3.16b - uzp1 v21.4s, v20.4s, v25.4s - mov v26.16b, v6.16b - ext v23.16b, v18.16b, v18.16b, #12 - mov v26.s[1], v21.s[2] - adrp x10, .LCPI0_2 - ext v25.16b, v18.16b, v23.16b, #12 - uzp1 v23.4s, v18.4s, v18.4s - ldr q1, [x10, :lo12:.LCPI0_2] - ext v18.16b, v23.16b, v18.16b, #8 - ushr v23.4s, v24.4s, #12 - shl v24.4s, v24.4s, #20 - orr v23.16b, v24.16b, v23.16b - add v7.4s, v7.4s, v23.4s - eor v27.16b, v29.16b, v7.16b - add v4.4s, v7.4s, v4.4s - tbl v7.16b, { v25.16b, v26.16b }, v3.16b - tbl v26.16b, { v27.16b }, v1.16b - add v22.4s, v22.4s, v26.4s - uzp2 v18.4s, v18.4s, v16.4s - eor v23.16b, v23.16b, v22.16b - ext v5.16b, v18.16b, v18.16b, #4 - ushr v27.4s, v23.4s, #7 - shl v23.4s, v23.4s, #25 - uzp1 v25.4s, v5.4s, v5.4s - orr v23.16b, v23.16b, v27.16b - ext v28.16b, v4.16b, v4.16b, #12 - ext v4.16b, v25.16b, v5.16b, #8 - ext v25.16b, v26.16b, v26.16b, #8 - add v26.4s, v28.4s, v23.4s - eor v25.16b, v26.16b, v25.16b - ext v22.16b, v22.16b, v22.16b, #4 - tbl v25.16b, { v25.16b }, v0.16b - add v22.4s, v22.4s, v25.4s - eor v23.16b, v23.16b, v22.16b - add v17.4s, v26.4s, v17.4s - ushr v26.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - orr v23.16b, v23.16b, v26.16b - add v17.4s, v17.4s, v23.4s - eor v25.16b, v25.16b, v17.16b - add v17.4s, v17.4s, v19.4s - tbl v19.16b, { v25.16b }, v1.16b - add v22.4s, v22.4s, v19.4s - eor v23.16b, v23.16b, v22.16b - ushr v25.4s, v23.4s, #7 - shl v23.4s, v23.4s, #25 - ext v17.16b, v17.16b, v17.16b, #4 - orr v23.16b, v23.16b, v25.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v17.4s, v17.4s, v23.4s - eor v19.16b, v17.16b, v19.16b - ext v22.16b, v22.16b, v22.16b, #12 - tbl v19.16b, { v19.16b }, v0.16b - add v22.4s, v22.4s, v19.4s - eor v23.16b, v23.16b, v22.16b - ushr v25.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - add v17.4s, v17.4s, v16.4s - orr v23.16b, v23.16b, v25.16b - add v17.4s, v17.4s, v23.4s - ext v25.16b, v17.16b, v17.16b, #12 - eor v17.16b, v19.16b, v17.16b - tbl v17.16b, { v17.16b }, v1.16b - add v19.4s, v22.4s, v17.4s - eor v22.16b, v23.16b, v19.16b - add v25.4s, v25.4s, v21.4s - zip1 v20.2d, v6.2d, v16.2d - ushr v23.4s, v22.4s, #7 - shl v22.4s, v22.4s, #25 - zip2 v24.4s, v16.4s, v6.4s - tbl v26.16b, { v20.16b, v21.16b }, v2.16b - orr v22.16b, v22.16b, v23.16b - zip1 v16.4s, v24.4s, v21.4s - zip1 v20.4s, v21.4s, v24.4s - ext v21.16b, v26.16b, v26.16b, #12 - ext v17.16b, v17.16b, v17.16b, #8 - add v25.4s, v25.4s, v22.4s - ext v16.16b, v20.16b, v16.16b, #8 - uzp1 v21.4s, v26.4s, v21.4s - eor v26.16b, v25.16b, v17.16b - ext v19.16b, v19.16b, v19.16b, #4 - tbl v26.16b, { v26.16b }, v0.16b - mov v29.16b, v16.16b - add v19.4s, v19.4s, v26.4s - ext v27.16b, v5.16b, v5.16b, #12 - mov v29.s[1], v21.s[2] - eor v22.16b, v22.16b, v19.16b - ext v28.16b, v5.16b, v27.16b, #12 - ushr v27.4s, v22.4s, #12 - shl v22.4s, v22.4s, #20 - add v6.4s, v25.4s, v6.4s - orr v22.16b, v22.16b, v27.16b - add v6.4s, v6.4s, v22.4s - eor v26.16b, v26.16b, v6.16b - add v6.4s, v6.4s, v18.4s - tbl v18.16b, { v26.16b }, v1.16b - add v19.4s, v19.4s, v18.4s - eor v22.16b, v22.16b, v19.16b - ushr v26.4s, v22.4s, #7 - shl v22.4s, v22.4s, #25 - ext v6.16b, v6.16b, v6.16b, #4 - orr v22.16b, v22.16b, v26.16b - ext v18.16b, v18.16b, v18.16b, #8 - add v6.4s, v6.4s, v22.4s - eor v18.16b, v6.16b, v18.16b - ext v19.16b, v19.16b, v19.16b, #12 - tbl v18.16b, { v18.16b }, v0.16b - add v19.4s, v19.4s, v18.4s - eor v22.16b, v22.16b, v19.16b - ushr v26.4s, v22.4s, #12 - shl v22.4s, v22.4s, #20 - add v6.4s, v6.4s, v7.4s - orr v22.16b, v22.16b, v26.16b - add v6.4s, v6.4s, v22.4s - ext v26.16b, v6.16b, v6.16b, #12 - eor v6.16b, v18.16b, v6.16b - uzp2 v4.4s, v4.4s, v7.4s - zip2 v25.4s, v7.4s, v16.4s - add v26.4s, v26.4s, v21.4s - zip1 v20.2d, v16.2d, v7.2d - tbl v6.16b, { v6.16b }, v1.16b - ext v24.16b, v4.16b, v4.16b, #4 - tbl v27.16b, { v20.16b, v21.16b }, v2.16b - zip1 v7.4s, v25.4s, v21.4s - zip1 v20.4s, v21.4s, v25.4s - add v18.4s, v19.4s, v6.4s - uzp1 v5.4s, v24.4s, v24.4s - ext v21.16b, v27.16b, v27.16b, #12 - ext v7.16b, v20.16b, v7.16b, #8 - eor v19.16b, v22.16b, v18.16b - ext v5.16b, v5.16b, v24.16b, #8 - tbl v17.16b, { v28.16b, v29.16b }, v3.16b - uzp1 v21.4s, v27.4s, v21.4s - mov v28.16b, v7.16b - ushr v22.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v23.16b, v24.16b, v24.16b, #12 - uzp2 v5.4s, v5.4s, v17.4s - mov v28.s[1], v21.s[2] - orr v19.16b, v19.16b, v22.16b - ext v27.16b, v24.16b, v23.16b, #12 - ext v23.16b, v5.16b, v5.16b, #4 - ext v6.16b, v6.16b, v6.16b, #8 - ext v25.16b, v18.16b, v18.16b, #4 - add v18.4s, v26.4s, v19.4s - uzp1 v24.4s, v23.4s, v23.4s - eor v6.16b, v18.16b, v6.16b - ext v24.16b, v24.16b, v23.16b, #8 - add v16.4s, v18.4s, v16.4s - tbl v18.16b, { v27.16b, v28.16b }, v3.16b - tbl v27.16b, { v6.16b }, v0.16b - uzp2 v6.4s, v24.4s, v18.4s - add v24.4s, v25.4s, v27.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v19.16b, v19.16b, v25.16b - add v16.4s, v16.4s, v19.4s - eor v25.16b, v27.16b, v16.16b - add v4.4s, v16.4s, v4.4s - tbl v16.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v16.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v4.16b, v4.16b, v4.16b, #4 - orr v19.16b, v19.16b, v25.16b - ext v16.16b, v16.16b, v16.16b, #8 - add v4.4s, v4.4s, v19.4s - eor v16.16b, v4.16b, v16.16b - ext v24.16b, v24.16b, v24.16b, #12 - tbl v25.16b, { v16.16b }, v0.16b - add v24.4s, v24.4s, v25.4s - eor v16.16b, v19.16b, v24.16b - ushr v19.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - add v4.4s, v4.4s, v17.4s - orr v19.16b, v16.16b, v19.16b - add v27.4s, v4.4s, v19.4s - eor v25.16b, v25.16b, v27.16b - tbl v25.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v25.4s - zip2 v26.4s, v17.4s, v7.4s - ext v4.16b, v27.16b, v27.16b, #12 - eor v19.16b, v19.16b, v24.16b - add v28.4s, v4.4s, v21.4s - zip1 v20.2d, v7.2d, v17.2d - zip1 v4.4s, v26.4s, v21.4s - zip1 v17.4s, v21.4s, v26.4s - ushr v26.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v19.16b, v19.16b, v26.16b - ext v25.16b, v25.16b, v25.16b, #8 - add v27.4s, v28.4s, v19.4s - eor v25.16b, v27.16b, v25.16b - ext v24.16b, v24.16b, v24.16b, #4 - tbl v25.16b, { v25.16b }, v0.16b - add v24.4s, v24.4s, v25.4s - eor v19.16b, v19.16b, v24.16b - add v7.4s, v27.4s, v7.4s - ushr v27.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v19.16b, v19.16b, v27.16b - add v7.4s, v7.4s, v19.4s - eor v25.16b, v25.16b, v7.16b - add v5.4s, v7.4s, v5.4s - tbl v7.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v7.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v5.16b, v5.16b, v5.16b, #4 - orr v19.16b, v19.16b, v25.16b - ext v7.16b, v7.16b, v7.16b, #8 - add v5.4s, v5.4s, v19.4s - eor v7.16b, v5.16b, v7.16b - ext v24.16b, v24.16b, v24.16b, #12 - tbl v7.16b, { v7.16b }, v0.16b - add v24.4s, v24.4s, v7.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - tbl v16.16b, { v20.16b, v21.16b }, v2.16b - add v5.4s, v5.4s, v18.4s - orr v19.16b, v19.16b, v25.16b - ext v20.16b, v16.16b, v16.16b, #12 - ext v4.16b, v17.16b, v4.16b, #8 - add v5.4s, v5.4s, v19.4s - uzp1 v21.4s, v16.4s, v20.4s - mov v17.16b, v4.16b - ext v25.16b, v5.16b, v5.16b, #12 - mov v17.s[1], v21.s[2] - add v25.4s, v25.4s, v21.4s - zip1 v20.2d, v4.2d, v18.2d - ext v22.16b, v23.16b, v23.16b, #12 - zip2 v26.4s, v18.4s, v4.4s - tbl v18.16b, { v20.16b, v21.16b }, v2.16b - eor v5.16b, v7.16b, v5.16b - ext v16.16b, v23.16b, v22.16b, #12 - ext v22.16b, v6.16b, v6.16b, #4 - zip1 v27.4s, v26.4s, v21.4s - zip1 v20.4s, v21.4s, v26.4s - ext v21.16b, v18.16b, v18.16b, #12 - tbl v5.16b, { v5.16b }, v1.16b - ext v20.16b, v20.16b, v27.16b, #8 - uzp1 v27.4s, v18.4s, v21.4s - uzp1 v18.4s, v22.4s, v22.4s - add v21.4s, v24.4s, v5.4s - ext v18.16b, v18.16b, v22.16b, #8 - eor v19.16b, v19.16b, v21.16b - tbl v7.16b, { v16.16b, v17.16b }, v3.16b - uzp2 v18.4s, v18.4s, v17.4s - zip2 v16.4s, v16.4s, v20.4s - ushr v17.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v17.16b, v19.16b, v17.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v19.4s, v25.4s, v17.4s - eor v5.16b, v19.16b, v5.16b - ext v21.16b, v21.16b, v21.16b, #4 - tbl v5.16b, { v5.16b }, v0.16b - add v4.4s, v19.4s, v4.4s - add v19.4s, v21.4s, v5.4s - eor v17.16b, v17.16b, v19.16b - ushr v21.4s, v17.4s, #12 - shl v17.4s, v17.4s, #20 - orr v17.16b, v17.16b, v21.16b - add v4.4s, v4.4s, v17.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - add v4.4s, v4.4s, v6.4s - add v6.4s, v19.4s, v5.4s - eor v17.16b, v17.16b, v6.16b - ushr v19.4s, v17.4s, #7 - shl v17.4s, v17.4s, #25 - ext v4.16b, v4.16b, v4.16b, #4 - orr v17.16b, v17.16b, v19.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v4.4s, v17.4s - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #12 - tbl v5.16b, { v5.16b }, v0.16b - add v6.4s, v6.4s, v5.4s - eor v17.16b, v17.16b, v6.16b - ushr v19.4s, v17.4s, #12 - shl v17.4s, v17.4s, #20 - add v4.4s, v4.4s, v7.4s - orr v17.16b, v17.16b, v19.16b - add v4.4s, v4.4s, v17.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - mov v29.16b, v20.16b - ext v4.16b, v4.16b, v4.16b, #12 - add v6.4s, v6.4s, v5.4s - mov v29.s[1], v27.s[2] - add v4.4s, v4.4s, v27.4s - zip1 v26.2d, v20.2d, v7.2d - zip1 v7.4s, v16.4s, v27.4s - zip1 v16.4s, v27.4s, v16.4s - eor v17.16b, v17.16b, v6.16b - ext v7.16b, v16.16b, v7.16b, #8 - ushr v16.4s, v17.4s, #7 - shl v17.4s, v17.4s, #25 - orr v16.16b, v17.16b, v16.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v4.4s, v16.4s - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #4 - tbl v5.16b, { v5.16b }, v0.16b - add v6.4s, v6.4s, v5.4s - eor v16.16b, v16.16b, v6.16b - ushr v17.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - add v4.4s, v4.4s, v20.4s - orr v16.16b, v16.16b, v17.16b - add v4.4s, v4.4s, v16.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - add v6.4s, v6.4s, v5.4s - eor v16.16b, v16.16b, v6.16b - add v4.4s, v4.4s, v18.4s - ushr v17.4s, v16.4s, #7 - shl v16.4s, v16.4s, #25 - ext v23.16b, v22.16b, v22.16b, #12 - ext v4.16b, v4.16b, v4.16b, #4 - orr v16.16b, v16.16b, v17.16b - ext v28.16b, v22.16b, v23.16b, #12 - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v16.4s, v4.4s - tbl v3.16b, { v28.16b, v29.16b }, v3.16b - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #12 - add v3.4s, v4.4s, v3.4s - tbl v4.16b, { v5.16b }, v0.16b - add v5.4s, v6.4s, v4.4s - eor v6.16b, v16.16b, v5.16b - ushr v16.4s, v6.4s, #12 - shl v6.4s, v6.4s, #20 - orr v6.16b, v6.16b, v16.16b - tbl v2.16b, { v26.16b, v27.16b }, v2.16b - add v3.4s, v3.4s, v6.4s - ext v19.16b, v2.16b, v2.16b, #12 - eor v4.16b, v4.16b, v3.16b - uzp1 v2.4s, v2.4s, v19.4s - ext v3.16b, v3.16b, v3.16b, #12 - tbl v4.16b, { v4.16b }, v1.16b - add v2.4s, v3.4s, v2.4s - add v3.4s, v5.4s, v4.4s - eor v5.16b, v6.16b, v3.16b - ushr v6.4s, v5.4s, #7 - shl v5.4s, v5.4s, #25 - orr v5.16b, v5.16b, v6.16b - ext v4.16b, v4.16b, v4.16b, #8 - add v2.4s, v2.4s, v5.4s - eor v4.16b, v2.16b, v4.16b - ext v3.16b, v3.16b, v3.16b, #4 - tbl v0.16b, { v4.16b }, v0.16b - add v3.4s, v3.4s, v0.4s - eor v4.16b, v5.16b, v3.16b - ushr v5.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - add v2.4s, v2.4s, v7.4s - orr v4.16b, v4.16b, v5.16b - add v2.4s, v2.4s, v4.4s - eor v0.16b, v0.16b, v2.16b - tbl v0.16b, { v0.16b }, v1.16b - add v1.4s, v3.4s, v0.4s - eor v3.16b, v4.16b, v1.16b - ext v2.16b, v2.16b, v2.16b, #4 - ext v1.16b, v1.16b, v1.16b, #12 - ushr v4.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - ext v0.16b, v0.16b, v0.16b, #8 - eor v1.16b, v2.16b, v1.16b - orr v2.16b, v3.16b, v4.16b + hint #25 + .cfi_negate_ra_state + sub sp, sp, #96 + stp x29, x30, [sp, #64] + add x29, sp, #64 + str x19, [sp, #80] + .cfi_def_cfa w29, 32 + .cfi_offset w19, -16 + .cfi_offset w30, -24 + .cfi_offset w29, -32 + mov x19, x0 + mov w5, w4 + mov x4, x3 + mov w3, w2 + mov x2, x1 + mov x0, sp + mov x1, x19 + bl compress_pre + ldp q0, q1, [sp] + ldp q2, q3, [sp, #32] eor v0.16b, v2.16b, v0.16b - stp q1, q0, [x0] + eor v1.16b, v3.16b, v1.16b + ldp x29, x30, [sp, #64] + stp q0, q1, [x19] + ldr x19, [sp, #80] + add sp, sp, #96 + hint #29 ret .Lfunc_end0: .size zfs_blake3_compress_in_place_sse41, .Lfunc_end0-zfs_blake3_compress_in_place_sse41 @@ -542,6 +85,9 @@ zfs_blake3_compress_in_place_sse41: .section .rodata.cst16,"aM",@progbits,16 .p2align 4 .LCPI1_0: + .xword -4942790177982912921 + .xword -6534734903820487822 +.LCPI1_1: .byte 2 .byte 3 .byte 0 @@ -558,11 +104,6 @@ zfs_blake3_compress_in_place_sse41: .byte 15 .byte 12 .byte 13 -.LCPI1_1: - .word 1779033703 - .word 3144134277 - .word 1013904242 - .word 2773480762 .LCPI1_2: .byte 1 .byte 2 @@ -580,488 +121,497 @@ zfs_blake3_compress_in_place_sse41: .byte 14 .byte 15 .byte 12 -.LCPI1_3: - .byte 0 - .byte 1 - .byte 2 - .byte 3 - .byte 20 - .byte 21 - .byte 22 - .byte 23 - .byte 8 - .byte 9 - .byte 10 - .byte 11 - .byte 28 - .byte 29 - .byte 30 - .byte 31 -.LCPI1_4: - .byte 0 - .byte 1 - .byte 2 - .byte 3 - .byte 4 - .byte 5 - .byte 6 - .byte 7 - .byte 8 - .byte 9 - .byte 10 - .byte 11 - .byte 28 - .byte 29 - .byte 30 - .byte 31 .text + .p2align 2 + .type compress_pre,@function +compress_pre: + .cfi_startproc + hint #34 + fmov s1, w3 + movi d0, #0x0000ff000000ff + ldr q2, [x1] + adrp x8, .LCPI1_0 + mov v1.s[1], w5 + str q2, [x0] + ldr q4, [x8, :lo12:.LCPI1_0] + ldr q5, [x1, #16] + adrp x8, .LCPI1_1 + and v0.8b, v1.8b, v0.8b + fmov d1, x4 + stp q5, q4, [x0, #16] + mov v1.d[1], v0.d[0] + str q1, [x0, #48] + ldp q6, q7, [x2] + uzp1 v3.4s, v6.4s, v7.4s + add v0.4s, v2.4s, v3.4s + uzp2 v2.4s, v6.4s, v7.4s + add v16.4s, v0.4s, v5.4s + ldr q0, [x8, :lo12:.LCPI1_1] + adrp x8, .LCPI1_2 + eor v1.16b, v16.16b, v1.16b + add v7.4s, v16.4s, v2.4s + tbl v1.16b, { v1.16b }, v0.16b + add v4.4s, v1.4s, v4.4s + eor v5.16b, v4.16b, v5.16b + ushr v6.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v6.16b + add v6.4s, v7.4s, v5.4s + eor v7.16b, v1.16b, v6.16b + ldr q1, [x8, :lo12:.LCPI1_2] + add x8, x2, #32 + tbl v7.16b, { v7.16b }, v1.16b + ld2 { v16.4s, v17.4s }, [x8] + add v4.4s, v4.4s, v7.4s + ext v7.16b, v7.16b, v7.16b, #8 + add v6.4s, v6.4s, v16.4s + eor v5.16b, v4.16b, v5.16b + ext v4.16b, v4.16b, v4.16b, #4 + ext v16.16b, v16.16b, v16.16b, #12 + ext v6.16b, v6.16b, v6.16b, #12 + ushr v18.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v18.16b + ext v18.16b, v17.16b, v17.16b, #12 + add v6.4s, v6.4s, v5.4s + mov v17.16b, v18.16b + eor v7.16b, v7.16b, v6.16b + add v6.4s, v6.4s, v18.4s + mov v17.s[1], v16.s[2] + tbl v7.16b, { v7.16b }, v0.16b + add v4.4s, v4.4s, v7.4s + eor v5.16b, v4.16b, v5.16b + ushr v19.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v19.16b + uzp1 v19.4s, v3.4s, v3.4s + add v6.4s, v6.4s, v5.4s + ext v19.16b, v19.16b, v3.16b, #8 + eor v7.16b, v7.16b, v6.16b + uzp2 v19.4s, v19.4s, v2.4s + tbl v7.16b, { v7.16b }, v1.16b + add v6.4s, v6.4s, v19.4s + add v4.4s, v4.4s, v7.4s + ext v6.16b, v6.16b, v6.16b, #4 + ext v7.16b, v7.16b, v7.16b, #8 + eor v5.16b, v4.16b, v5.16b + ext v4.16b, v4.16b, v4.16b, #12 + ushr v20.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v20.16b + ext v20.16b, v3.16b, v3.16b, #12 + add v6.4s, v6.4s, v5.4s + ext v3.16b, v3.16b, v20.16b, #12 + eor v7.16b, v7.16b, v6.16b + rev64 v3.4s, v3.4s + tbl v7.16b, { v7.16b }, v0.16b + trn2 v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v7.4s + add v6.4s, v6.4s, v3.4s + eor v5.16b, v4.16b, v5.16b + ushr v17.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v17.16b + zip1 v17.2d, v18.2d, v2.2d + zip2 v2.4s, v2.4s, v18.4s + add v6.4s, v6.4s, v5.4s + mov v17.s[3], v16.s[3] + zip1 v18.4s, v2.4s, v16.4s + zip1 v2.4s, v16.4s, v2.4s + eor v7.16b, v7.16b, v6.16b + ext v6.16b, v6.16b, v6.16b, #12 + ext v16.16b, v2.16b, v18.16b, #8 + tbl v7.16b, { v7.16b }, v1.16b + add v20.4s, v4.4s, v7.4s + ext v4.16b, v17.16b, v17.16b, #12 + ext v7.16b, v7.16b, v7.16b, #8 + eor v5.16b, v20.16b, v5.16b + uzp1 v4.4s, v17.4s, v4.4s + ushr v17.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v6.4s, v6.4s, v4.4s + orr v5.16b, v5.16b, v17.16b + ext v17.16b, v20.16b, v20.16b, #4 + add v6.4s, v6.4s, v5.4s + eor v7.16b, v7.16b, v6.16b + add v6.4s, v6.4s, v16.4s + tbl v7.16b, { v7.16b }, v0.16b + add v17.4s, v17.4s, v7.4s + eor v5.16b, v17.16b, v5.16b + ushr v2.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v2.16b, v5.16b, v2.16b + add v5.4s, v6.4s, v2.4s + ext v6.16b, v19.16b, v19.16b, #4 + eor v7.16b, v7.16b, v5.16b + uzp1 v18.4s, v6.4s, v6.4s + tbl v7.16b, { v7.16b }, v1.16b + ext v18.16b, v18.16b, v6.16b, #8 + add v17.4s, v17.4s, v7.4s + uzp2 v18.4s, v18.4s, v3.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v2.16b, v17.16b, v2.16b + add v5.4s, v5.4s, v18.4s + ext v17.16b, v17.16b, v17.16b, #12 + ushr v19.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ext v5.16b, v5.16b, v5.16b, #4 + orr v2.16b, v2.16b, v19.16b + ext v19.16b, v6.16b, v6.16b, #12 + add v5.4s, v5.4s, v2.4s + ext v6.16b, v6.16b, v19.16b, #12 + mov v19.16b, v16.16b + eor v7.16b, v7.16b, v5.16b + rev64 v6.4s, v6.4s + mov v19.s[1], v4.s[2] + tbl v7.16b, { v7.16b }, v0.16b + add v17.4s, v17.4s, v7.4s + eor v20.16b, v17.16b, v2.16b + trn2 v2.4s, v6.4s, v19.4s + ushr v6.4s, v20.4s, #12 + shl v19.4s, v20.4s, #20 + add v5.4s, v5.4s, v2.4s + orr v6.16b, v19.16b, v6.16b + add v19.4s, v5.4s, v6.4s + eor v5.16b, v7.16b, v19.16b + zip1 v7.2d, v16.2d, v3.2d + zip2 v3.4s, v3.4s, v16.4s + tbl v20.16b, { v5.16b }, v1.16b + mov v7.s[3], v4.s[3] + add v17.4s, v17.4s, v20.4s + ext v5.16b, v7.16b, v7.16b, #12 + eor v6.16b, v17.16b, v6.16b + uzp1 v5.4s, v7.4s, v5.4s + ext v7.16b, v19.16b, v19.16b, #12 + ext v17.16b, v17.16b, v17.16b, #4 + ushr v19.4s, v6.4s, #7 + shl v6.4s, v6.4s, #25 + add v7.4s, v7.4s, v5.4s + orr v6.16b, v6.16b, v19.16b + ext v19.16b, v20.16b, v20.16b, #8 + add v7.4s, v7.4s, v6.4s + eor v19.16b, v19.16b, v7.16b + tbl v19.16b, { v19.16b }, v0.16b + add v16.4s, v17.4s, v19.4s + zip1 v17.4s, v3.4s, v4.4s + zip1 v3.4s, v4.4s, v3.4s + eor v4.16b, v16.16b, v6.16b + ext v17.16b, v3.16b, v17.16b, #8 + ushr v3.4s, v4.4s, #12 + shl v4.4s, v4.4s, #20 + add v6.4s, v7.4s, v17.4s + orr v3.16b, v4.16b, v3.16b + add v4.4s, v6.4s, v3.4s + ext v6.16b, v18.16b, v18.16b, #4 + eor v7.16b, v19.16b, v4.16b + uzp1 v18.4s, v6.4s, v6.4s + tbl v7.16b, { v7.16b }, v1.16b + ext v18.16b, v18.16b, v6.16b, #8 + add v16.4s, v16.4s, v7.4s + uzp2 v18.4s, v18.4s, v2.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v3.16b, v16.16b, v3.16b + add v4.4s, v4.4s, v18.4s + ext v16.16b, v16.16b, v16.16b, #12 + ushr v19.4s, v3.4s, #7 + shl v3.4s, v3.4s, #25 + ext v4.16b, v4.16b, v4.16b, #4 + orr v3.16b, v3.16b, v19.16b + ext v19.16b, v6.16b, v6.16b, #12 + add v4.4s, v4.4s, v3.4s + ext v6.16b, v6.16b, v19.16b, #12 + mov v19.16b, v17.16b + eor v7.16b, v7.16b, v4.16b + rev64 v6.4s, v6.4s + mov v19.s[1], v5.s[2] + tbl v7.16b, { v7.16b }, v0.16b + add v16.4s, v16.4s, v7.4s + eor v20.16b, v16.16b, v3.16b + trn2 v3.4s, v6.4s, v19.4s + ushr v6.4s, v20.4s, #12 + shl v19.4s, v20.4s, #20 + add v4.4s, v4.4s, v3.4s + orr v6.16b, v19.16b, v6.16b + zip1 v19.2d, v17.2d, v2.2d + zip2 v2.4s, v2.4s, v17.4s + add v4.4s, v4.4s, v6.4s + mov v19.s[3], v5.s[3] + zip1 v17.4s, v2.4s, v5.4s + zip1 v2.4s, v5.4s, v2.4s + eor v7.16b, v7.16b, v4.16b + ext v20.16b, v19.16b, v19.16b, #12 + ext v4.16b, v4.16b, v4.16b, #12 + ext v2.16b, v2.16b, v17.16b, #8 + tbl v7.16b, { v7.16b }, v1.16b + add v16.4s, v16.4s, v7.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v21.16b, v16.16b, v6.16b + uzp1 v6.4s, v19.4s, v20.4s + ext v16.16b, v16.16b, v16.16b, #4 + ushr v19.4s, v21.4s, #7 + shl v20.4s, v21.4s, #25 + add v4.4s, v4.4s, v6.4s + orr v19.16b, v20.16b, v19.16b + add v4.4s, v4.4s, v19.4s + eor v7.16b, v7.16b, v4.16b + add v4.4s, v4.4s, v2.4s + tbl v7.16b, { v7.16b }, v0.16b + add v16.4s, v16.4s, v7.4s + eor v5.16b, v16.16b, v19.16b + ushr v17.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v17.16b + ext v17.16b, v18.16b, v18.16b, #4 + add v4.4s, v4.4s, v5.4s + uzp1 v18.4s, v17.4s, v17.4s + eor v7.16b, v7.16b, v4.16b + ext v18.16b, v18.16b, v17.16b, #8 + tbl v7.16b, { v7.16b }, v1.16b + uzp2 v18.4s, v18.4s, v3.4s + add v16.4s, v16.4s, v7.4s + add v4.4s, v4.4s, v18.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v5.16b, v16.16b, v5.16b + ext v4.16b, v4.16b, v4.16b, #4 + ext v16.16b, v16.16b, v16.16b, #12 + ushr v19.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v19.16b + add v19.4s, v4.4s, v5.4s + eor v4.16b, v7.16b, v19.16b + ext v7.16b, v17.16b, v17.16b, #12 + tbl v20.16b, { v4.16b }, v0.16b + ext v4.16b, v17.16b, v7.16b, #12 + mov v7.16b, v2.16b + add v16.4s, v16.4s, v20.4s + rev64 v4.4s, v4.4s + mov v7.s[1], v6.s[2] + eor v5.16b, v16.16b, v5.16b + trn2 v4.4s, v4.4s, v7.4s + ushr v7.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v19.4s, v4.4s + zip1 v19.2d, v2.2d, v3.2d + zip2 v2.4s, v3.4s, v2.4s + orr v5.16b, v5.16b, v7.16b + mov v19.s[3], v6.s[3] + add v7.4s, v17.4s, v5.4s + eor v17.16b, v20.16b, v7.16b + ext v20.16b, v19.16b, v19.16b, #12 + ext v7.16b, v7.16b, v7.16b, #12 + tbl v17.16b, { v17.16b }, v1.16b + add v16.4s, v16.4s, v17.4s + ext v17.16b, v17.16b, v17.16b, #8 + eor v21.16b, v16.16b, v5.16b + uzp1 v5.4s, v19.4s, v20.4s + ext v16.16b, v16.16b, v16.16b, #4 + ushr v19.4s, v21.4s, #7 + shl v20.4s, v21.4s, #25 + add v7.4s, v7.4s, v5.4s + orr v19.16b, v20.16b, v19.16b + add v7.4s, v7.4s, v19.4s + eor v17.16b, v17.16b, v7.16b + tbl v17.16b, { v17.16b }, v0.16b + add v3.4s, v16.4s, v17.4s + zip1 v16.4s, v2.4s, v6.4s + zip1 v2.4s, v6.4s, v2.4s + eor v6.16b, v3.16b, v19.16b + ext v16.16b, v2.16b, v16.16b, #8 + ushr v2.4s, v6.4s, #12 + shl v6.4s, v6.4s, #20 + add v7.4s, v7.4s, v16.4s + orr v2.16b, v6.16b, v2.16b + add v6.4s, v7.4s, v2.4s + ext v7.16b, v18.16b, v18.16b, #4 + eor v17.16b, v17.16b, v6.16b + uzp1 v18.4s, v7.4s, v7.4s + tbl v17.16b, { v17.16b }, v1.16b + ext v18.16b, v18.16b, v7.16b, #8 + add v3.4s, v3.4s, v17.4s + uzp2 v18.4s, v18.4s, v4.4s + eor v2.16b, v3.16b, v2.16b + add v6.4s, v6.4s, v18.4s + ext v3.16b, v3.16b, v3.16b, #12 + ext v18.16b, v18.16b, v18.16b, #4 + ushr v19.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ext v6.16b, v6.16b, v6.16b, #4 + orr v19.16b, v2.16b, v19.16b + ext v2.16b, v17.16b, v17.16b, #8 + ext v17.16b, v7.16b, v7.16b, #12 + add v6.4s, v6.4s, v19.4s + eor v2.16b, v2.16b, v6.16b + tbl v20.16b, { v2.16b }, v0.16b + ext v2.16b, v7.16b, v17.16b, #12 + mov v7.16b, v16.16b + add v17.4s, v3.4s, v20.4s + rev64 v3.4s, v2.4s + mov v7.s[1], v5.s[2] + eor v19.16b, v17.16b, v19.16b + trn2 v3.4s, v3.4s, v7.4s + ushr v21.4s, v19.4s, #12 + shl v19.4s, v19.4s, #20 + add v6.4s, v6.4s, v3.4s + orr v19.16b, v19.16b, v21.16b + add v21.4s, v6.4s, v19.4s + eor v6.16b, v20.16b, v21.16b + zip1 v20.2d, v16.2d, v4.2d + zip2 v4.4s, v4.4s, v16.4s + tbl v22.16b, { v6.16b }, v1.16b + mov v20.s[3], v5.s[3] + add v17.4s, v17.4s, v22.4s + ext v6.16b, v20.16b, v20.16b, #12 + eor v19.16b, v17.16b, v19.16b + uzp1 v6.4s, v20.4s, v6.4s + ext v20.16b, v21.16b, v21.16b, #12 + ext v17.16b, v17.16b, v17.16b, #4 + ushr v21.4s, v19.4s, #7 + shl v19.4s, v19.4s, #25 + add v20.4s, v20.4s, v6.4s + orr v19.16b, v19.16b, v21.16b + ext v21.16b, v22.16b, v22.16b, #8 + add v20.4s, v20.4s, v19.4s + eor v21.16b, v21.16b, v20.16b + tbl v21.16b, { v21.16b }, v0.16b + add v16.4s, v17.4s, v21.4s + zip1 v17.4s, v4.4s, v5.4s + zip1 v4.4s, v5.4s, v4.4s + eor v5.16b, v16.16b, v19.16b + ext v4.16b, v4.16b, v17.16b, #8 + ushr v17.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v19.4s, v20.4s, v4.4s + ext v20.16b, v18.16b, v18.16b, #8 + zip1 v3.2d, v4.2d, v3.2d + orr v5.16b, v5.16b, v17.16b + zip2 v2.4s, v2.4s, v4.4s + uzp2 v7.4s, v20.4s, v7.4s + mov v3.s[3], v6.s[3] + add v17.4s, v19.4s, v5.4s + ext v7.16b, v7.16b, v20.16b, #4 + eor v19.16b, v21.16b, v17.16b + ext v17.16b, v17.16b, v17.16b, #4 + tbl v19.16b, { v19.16b }, v1.16b + add v7.4s, v17.4s, v7.4s + add v16.4s, v16.4s, v19.4s + ext v17.16b, v19.16b, v19.16b, #8 + ext v19.16b, v18.16b, v18.16b, #12 + eor v5.16b, v16.16b, v5.16b + ext v16.16b, v16.16b, v16.16b, #12 + ext v18.16b, v18.16b, v19.16b, #12 + mov v19.16b, v4.16b + ushr v20.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + rev64 v18.4s, v18.4s + mov v19.s[1], v6.s[2] + orr v5.16b, v5.16b, v20.16b + trn2 v18.4s, v18.4s, v19.4s + add v7.4s, v5.4s, v7.4s + eor v17.16b, v17.16b, v7.16b + add v7.4s, v7.4s, v18.4s + ext v18.16b, v3.16b, v3.16b, #12 + tbl v17.16b, { v17.16b }, v0.16b + uzp1 v3.4s, v3.4s, v18.4s + add v16.4s, v16.4s, v17.4s + eor v5.16b, v16.16b, v5.16b + ushr v19.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v5.16b, v5.16b, v19.16b + add v7.4s, v7.4s, v5.4s + eor v17.16b, v17.16b, v7.16b + ext v7.16b, v7.16b, v7.16b, #12 + tbl v17.16b, { v17.16b }, v1.16b + add v3.4s, v7.4s, v3.4s + add v16.4s, v16.4s, v17.4s + ext v7.16b, v17.16b, v17.16b, #8 + eor v5.16b, v16.16b, v5.16b + ext v16.16b, v16.16b, v16.16b, #4 + ushr v18.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v5.16b, v5.16b, v18.16b + add v3.4s, v3.4s, v5.4s + eor v7.16b, v7.16b, v3.16b + tbl v0.16b, { v7.16b }, v0.16b + zip1 v7.4s, v2.4s, v6.4s + zip1 v2.4s, v6.4s, v2.4s + add v4.4s, v16.4s, v0.4s + ext v2.16b, v2.16b, v7.16b, #8 + eor v5.16b, v4.16b, v5.16b + add v2.4s, v3.4s, v2.4s + ushr v6.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + orr v3.16b, v5.16b, v6.16b + add v2.4s, v2.4s, v3.4s + eor v0.16b, v0.16b, v2.16b + ext v2.16b, v2.16b, v2.16b, #4 + tbl v0.16b, { v0.16b }, v1.16b + add v1.4s, v4.4s, v0.4s + ext v0.16b, v0.16b, v0.16b, #8 + eor v3.16b, v1.16b, v3.16b + ext v1.16b, v1.16b, v1.16b, #12 + ushr v4.4s, v3.4s, #7 + shl v3.4s, v3.4s, #25 + stp q1, q0, [x0, #32] + orr v3.16b, v3.16b, v4.16b + stp q2, q3, [x0] + ret +.Lfunc_end1: + .size compress_pre, .Lfunc_end1-compress_pre + .cfi_endproc + .globl zfs_blake3_compress_xof_sse41 .p2align 2 .type zfs_blake3_compress_xof_sse41,@function zfs_blake3_compress_xof_sse41: .cfi_startproc - ldp q7, q6, [x0] - ldp q17, q18, [x1] - add x12, x1, #32 - ld2 { v4.4s, v5.4s }, [x12] - lsr x10, x3, #32 - fmov s16, w3 - adrp x13, .LCPI1_0 - adrp x11, .LCPI1_1 - and w8, w2, #0xff - mov v16.s[1], w10 - ldr q0, [x13, :lo12:.LCPI1_0] - ldr q20, [x11, :lo12:.LCPI1_1] - adrp x11, .LCPI1_4 - and w9, w4, #0xff - ldr q2, [x11, :lo12:.LCPI1_4] - mov v16.s[2], w8 - uzp1 v21.4s, v17.4s, v18.4s - add v7.4s, v6.4s, v7.4s - adrp x12, .LCPI1_3 - mov v16.s[3], w9 - uzp2 v18.4s, v17.4s, v18.4s - add v7.4s, v7.4s, v21.4s - ext v17.16b, v5.16b, v5.16b, #12 - ldr q3, [x12, :lo12:.LCPI1_3] - ext v24.16b, v4.16b, v4.16b, #12 - eor v16.16b, v7.16b, v16.16b - mov v27.16b, v17.16b - uzp1 v19.4s, v21.4s, v21.4s - ext v25.16b, v21.16b, v21.16b, #12 - zip2 v28.4s, v18.4s, v17.4s - tbl v29.16b, { v16.16b }, v0.16b - mov v27.s[1], v24.s[2] - zip1 v23.2d, v17.2d, v18.2d - ext v19.16b, v19.16b, v21.16b, #8 - add v22.4s, v29.4s, v20.4s - ext v26.16b, v21.16b, v25.16b, #12 - tbl v20.16b, { v23.16b, v24.16b }, v2.16b - zip1 v21.4s, v28.4s, v24.4s - zip1 v23.4s, v24.4s, v28.4s - uzp2 v19.4s, v19.4s, v18.4s - eor v24.16b, v22.16b, v6.16b - ext v25.16b, v20.16b, v20.16b, #12 - ext v6.16b, v23.16b, v21.16b, #8 - add v7.4s, v7.4s, v18.4s - ext v18.16b, v19.16b, v19.16b, #4 - tbl v16.16b, { v26.16b, v27.16b }, v3.16b - uzp1 v21.4s, v20.4s, v25.4s - mov v26.16b, v6.16b - ext v23.16b, v18.16b, v18.16b, #12 - mov v26.s[1], v21.s[2] - adrp x10, .LCPI1_2 - ext v25.16b, v18.16b, v23.16b, #12 - uzp1 v23.4s, v18.4s, v18.4s - ldr q1, [x10, :lo12:.LCPI1_2] - ext v18.16b, v23.16b, v18.16b, #8 - ushr v23.4s, v24.4s, #12 - shl v24.4s, v24.4s, #20 - orr v23.16b, v24.16b, v23.16b - add v7.4s, v7.4s, v23.4s - eor v27.16b, v29.16b, v7.16b - add v4.4s, v7.4s, v4.4s - tbl v7.16b, { v25.16b, v26.16b }, v3.16b - tbl v26.16b, { v27.16b }, v1.16b - add v22.4s, v22.4s, v26.4s - uzp2 v18.4s, v18.4s, v16.4s - eor v23.16b, v23.16b, v22.16b - ext v5.16b, v18.16b, v18.16b, #4 - ushr v27.4s, v23.4s, #7 - shl v23.4s, v23.4s, #25 - uzp1 v25.4s, v5.4s, v5.4s - orr v23.16b, v23.16b, v27.16b - ext v28.16b, v4.16b, v4.16b, #12 - ext v4.16b, v25.16b, v5.16b, #8 - ext v25.16b, v26.16b, v26.16b, #8 - add v26.4s, v28.4s, v23.4s - eor v25.16b, v26.16b, v25.16b - ext v22.16b, v22.16b, v22.16b, #4 - tbl v25.16b, { v25.16b }, v0.16b - add v22.4s, v22.4s, v25.4s - eor v23.16b, v23.16b, v22.16b - add v17.4s, v26.4s, v17.4s - ushr v26.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - orr v23.16b, v23.16b, v26.16b - add v17.4s, v17.4s, v23.4s - eor v25.16b, v25.16b, v17.16b - add v17.4s, v17.4s, v19.4s - tbl v19.16b, { v25.16b }, v1.16b - add v22.4s, v22.4s, v19.4s - eor v23.16b, v23.16b, v22.16b - ushr v25.4s, v23.4s, #7 - shl v23.4s, v23.4s, #25 - ext v17.16b, v17.16b, v17.16b, #4 - orr v23.16b, v23.16b, v25.16b - ext v19.16b, v19.16b, v19.16b, #8 - add v17.4s, v17.4s, v23.4s - eor v19.16b, v17.16b, v19.16b - ext v22.16b, v22.16b, v22.16b, #12 - tbl v19.16b, { v19.16b }, v0.16b - add v22.4s, v22.4s, v19.4s - eor v23.16b, v23.16b, v22.16b - ushr v25.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - add v17.4s, v17.4s, v16.4s - orr v23.16b, v23.16b, v25.16b - add v17.4s, v17.4s, v23.4s - ext v25.16b, v17.16b, v17.16b, #12 - eor v17.16b, v19.16b, v17.16b - tbl v17.16b, { v17.16b }, v1.16b - add v19.4s, v22.4s, v17.4s - eor v22.16b, v23.16b, v19.16b - add v25.4s, v25.4s, v21.4s - zip1 v20.2d, v6.2d, v16.2d - ushr v23.4s, v22.4s, #7 - shl v22.4s, v22.4s, #25 - zip2 v24.4s, v16.4s, v6.4s - tbl v26.16b, { v20.16b, v21.16b }, v2.16b - orr v22.16b, v22.16b, v23.16b - zip1 v16.4s, v24.4s, v21.4s - zip1 v20.4s, v21.4s, v24.4s - ext v21.16b, v26.16b, v26.16b, #12 - ext v17.16b, v17.16b, v17.16b, #8 - add v25.4s, v25.4s, v22.4s - ext v16.16b, v20.16b, v16.16b, #8 - uzp1 v21.4s, v26.4s, v21.4s - eor v26.16b, v25.16b, v17.16b - ext v19.16b, v19.16b, v19.16b, #4 - tbl v26.16b, { v26.16b }, v0.16b - mov v29.16b, v16.16b - add v19.4s, v19.4s, v26.4s - ext v27.16b, v5.16b, v5.16b, #12 - mov v29.s[1], v21.s[2] - eor v22.16b, v22.16b, v19.16b - ext v28.16b, v5.16b, v27.16b, #12 - ushr v27.4s, v22.4s, #12 - shl v22.4s, v22.4s, #20 - add v6.4s, v25.4s, v6.4s - orr v22.16b, v22.16b, v27.16b - add v6.4s, v6.4s, v22.4s - eor v26.16b, v26.16b, v6.16b - add v6.4s, v6.4s, v18.4s - tbl v18.16b, { v26.16b }, v1.16b - add v19.4s, v19.4s, v18.4s - eor v22.16b, v22.16b, v19.16b - ushr v26.4s, v22.4s, #7 - shl v22.4s, v22.4s, #25 - ext v6.16b, v6.16b, v6.16b, #4 - orr v22.16b, v22.16b, v26.16b - ext v18.16b, v18.16b, v18.16b, #8 - add v6.4s, v6.4s, v22.4s - eor v18.16b, v6.16b, v18.16b - ext v19.16b, v19.16b, v19.16b, #12 - tbl v18.16b, { v18.16b }, v0.16b - add v19.4s, v19.4s, v18.4s - eor v22.16b, v22.16b, v19.16b - ushr v26.4s, v22.4s, #12 - shl v22.4s, v22.4s, #20 - add v6.4s, v6.4s, v7.4s - orr v22.16b, v22.16b, v26.16b - add v6.4s, v6.4s, v22.4s - ext v26.16b, v6.16b, v6.16b, #12 - eor v6.16b, v18.16b, v6.16b - uzp2 v4.4s, v4.4s, v7.4s - zip2 v25.4s, v7.4s, v16.4s - add v26.4s, v26.4s, v21.4s - zip1 v20.2d, v16.2d, v7.2d - tbl v6.16b, { v6.16b }, v1.16b - ext v24.16b, v4.16b, v4.16b, #4 - tbl v27.16b, { v20.16b, v21.16b }, v2.16b - zip1 v7.4s, v25.4s, v21.4s - zip1 v20.4s, v21.4s, v25.4s - add v18.4s, v19.4s, v6.4s - uzp1 v5.4s, v24.4s, v24.4s - ext v21.16b, v27.16b, v27.16b, #12 - ext v7.16b, v20.16b, v7.16b, #8 - eor v19.16b, v22.16b, v18.16b - ext v5.16b, v5.16b, v24.16b, #8 - tbl v17.16b, { v28.16b, v29.16b }, v3.16b - uzp1 v21.4s, v27.4s, v21.4s - mov v28.16b, v7.16b - ushr v22.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v23.16b, v24.16b, v24.16b, #12 - uzp2 v5.4s, v5.4s, v17.4s - mov v28.s[1], v21.s[2] - orr v19.16b, v19.16b, v22.16b - ext v27.16b, v24.16b, v23.16b, #12 - ext v23.16b, v5.16b, v5.16b, #4 - ext v6.16b, v6.16b, v6.16b, #8 - ext v25.16b, v18.16b, v18.16b, #4 - add v18.4s, v26.4s, v19.4s - uzp1 v24.4s, v23.4s, v23.4s - eor v6.16b, v18.16b, v6.16b - ext v24.16b, v24.16b, v23.16b, #8 - add v16.4s, v18.4s, v16.4s - tbl v18.16b, { v27.16b, v28.16b }, v3.16b - tbl v27.16b, { v6.16b }, v0.16b - uzp2 v6.4s, v24.4s, v18.4s - add v24.4s, v25.4s, v27.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v19.16b, v19.16b, v25.16b - add v16.4s, v16.4s, v19.4s - eor v25.16b, v27.16b, v16.16b - add v4.4s, v16.4s, v4.4s - tbl v16.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v16.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v4.16b, v4.16b, v4.16b, #4 - orr v19.16b, v19.16b, v25.16b - ext v16.16b, v16.16b, v16.16b, #8 - add v4.4s, v4.4s, v19.4s - eor v16.16b, v4.16b, v16.16b - ext v24.16b, v24.16b, v24.16b, #12 - tbl v25.16b, { v16.16b }, v0.16b - add v24.4s, v24.4s, v25.4s - eor v16.16b, v19.16b, v24.16b - ushr v19.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - add v4.4s, v4.4s, v17.4s - orr v19.16b, v16.16b, v19.16b - add v27.4s, v4.4s, v19.4s - eor v25.16b, v25.16b, v27.16b - tbl v25.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v25.4s - zip2 v26.4s, v17.4s, v7.4s - ext v4.16b, v27.16b, v27.16b, #12 - eor v19.16b, v19.16b, v24.16b - add v28.4s, v4.4s, v21.4s - zip1 v20.2d, v7.2d, v17.2d - zip1 v4.4s, v26.4s, v21.4s - zip1 v17.4s, v21.4s, v26.4s - ushr v26.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v19.16b, v19.16b, v26.16b - ext v25.16b, v25.16b, v25.16b, #8 - add v27.4s, v28.4s, v19.4s - eor v25.16b, v27.16b, v25.16b - ext v24.16b, v24.16b, v24.16b, #4 - tbl v25.16b, { v25.16b }, v0.16b - add v24.4s, v24.4s, v25.4s - eor v19.16b, v19.16b, v24.16b - add v7.4s, v27.4s, v7.4s - ushr v27.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v19.16b, v19.16b, v27.16b - add v7.4s, v7.4s, v19.4s - eor v25.16b, v25.16b, v7.16b - add v5.4s, v7.4s, v5.4s - tbl v7.16b, { v25.16b }, v1.16b - add v24.4s, v24.4s, v7.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - ext v5.16b, v5.16b, v5.16b, #4 - orr v19.16b, v19.16b, v25.16b - ext v7.16b, v7.16b, v7.16b, #8 - add v5.4s, v5.4s, v19.4s - eor v7.16b, v5.16b, v7.16b - ext v24.16b, v24.16b, v24.16b, #12 - tbl v7.16b, { v7.16b }, v0.16b - add v24.4s, v24.4s, v7.4s - eor v19.16b, v19.16b, v24.16b - ushr v25.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - tbl v16.16b, { v20.16b, v21.16b }, v2.16b - add v5.4s, v5.4s, v18.4s - orr v19.16b, v19.16b, v25.16b - ext v20.16b, v16.16b, v16.16b, #12 - ext v4.16b, v17.16b, v4.16b, #8 - add v5.4s, v5.4s, v19.4s - uzp1 v21.4s, v16.4s, v20.4s - mov v17.16b, v4.16b - ext v25.16b, v5.16b, v5.16b, #12 - mov v17.s[1], v21.s[2] - add v25.4s, v25.4s, v21.4s - zip1 v20.2d, v4.2d, v18.2d - ext v22.16b, v23.16b, v23.16b, #12 - zip2 v26.4s, v18.4s, v4.4s - tbl v18.16b, { v20.16b, v21.16b }, v2.16b - eor v5.16b, v7.16b, v5.16b - ext v16.16b, v23.16b, v22.16b, #12 - ext v22.16b, v6.16b, v6.16b, #4 - zip1 v27.4s, v26.4s, v21.4s - zip1 v20.4s, v21.4s, v26.4s - ext v21.16b, v18.16b, v18.16b, #12 - tbl v5.16b, { v5.16b }, v1.16b - ext v20.16b, v20.16b, v27.16b, #8 - uzp1 v27.4s, v18.4s, v21.4s - uzp1 v18.4s, v22.4s, v22.4s - add v21.4s, v24.4s, v5.4s - ext v18.16b, v18.16b, v22.16b, #8 - eor v19.16b, v19.16b, v21.16b - tbl v7.16b, { v16.16b, v17.16b }, v3.16b - uzp2 v18.4s, v18.4s, v17.4s - zip2 v16.4s, v16.4s, v20.4s - ushr v17.4s, v19.4s, #7 - shl v19.4s, v19.4s, #25 - orr v17.16b, v19.16b, v17.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v19.4s, v25.4s, v17.4s - eor v5.16b, v19.16b, v5.16b - ext v21.16b, v21.16b, v21.16b, #4 - tbl v5.16b, { v5.16b }, v0.16b - add v4.4s, v19.4s, v4.4s - add v19.4s, v21.4s, v5.4s - eor v17.16b, v17.16b, v19.16b - ushr v21.4s, v17.4s, #12 - shl v17.4s, v17.4s, #20 - orr v17.16b, v17.16b, v21.16b - add v4.4s, v4.4s, v17.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - add v4.4s, v4.4s, v6.4s - add v6.4s, v19.4s, v5.4s - eor v17.16b, v17.16b, v6.16b - ushr v19.4s, v17.4s, #7 - shl v17.4s, v17.4s, #25 - ext v4.16b, v4.16b, v4.16b, #4 - orr v17.16b, v17.16b, v19.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v4.4s, v17.4s - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #12 - tbl v5.16b, { v5.16b }, v0.16b - add v6.4s, v6.4s, v5.4s - eor v17.16b, v17.16b, v6.16b - ushr v19.4s, v17.4s, #12 - shl v17.4s, v17.4s, #20 - add v4.4s, v4.4s, v7.4s - orr v17.16b, v17.16b, v19.16b - add v4.4s, v4.4s, v17.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - mov v29.16b, v20.16b - ext v4.16b, v4.16b, v4.16b, #12 - add v6.4s, v6.4s, v5.4s - mov v29.s[1], v27.s[2] - add v4.4s, v4.4s, v27.4s - zip1 v26.2d, v20.2d, v7.2d - zip1 v7.4s, v16.4s, v27.4s - zip1 v16.4s, v27.4s, v16.4s - eor v17.16b, v17.16b, v6.16b - ext v7.16b, v16.16b, v7.16b, #8 - ushr v16.4s, v17.4s, #7 - shl v17.4s, v17.4s, #25 - orr v16.16b, v17.16b, v16.16b - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v4.4s, v16.4s - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #4 - tbl v5.16b, { v5.16b }, v0.16b - add v6.4s, v6.4s, v5.4s - eor v16.16b, v16.16b, v6.16b - ushr v17.4s, v16.4s, #12 - shl v16.4s, v16.4s, #20 - add v4.4s, v4.4s, v20.4s - orr v16.16b, v16.16b, v17.16b - add v4.4s, v4.4s, v16.4s - eor v5.16b, v5.16b, v4.16b - tbl v5.16b, { v5.16b }, v1.16b - add v6.4s, v6.4s, v5.4s - eor v16.16b, v16.16b, v6.16b - add v4.4s, v4.4s, v18.4s - ushr v17.4s, v16.4s, #7 - shl v16.4s, v16.4s, #25 - ext v23.16b, v22.16b, v22.16b, #12 - ext v4.16b, v4.16b, v4.16b, #4 - orr v16.16b, v16.16b, v17.16b - ext v28.16b, v22.16b, v23.16b, #12 - ext v5.16b, v5.16b, v5.16b, #8 - add v4.4s, v16.4s, v4.4s - tbl v3.16b, { v28.16b, v29.16b }, v3.16b - eor v5.16b, v4.16b, v5.16b - ext v6.16b, v6.16b, v6.16b, #12 - add v3.4s, v4.4s, v3.4s - tbl v4.16b, { v5.16b }, v0.16b - add v5.4s, v6.4s, v4.4s - eor v6.16b, v16.16b, v5.16b - ushr v16.4s, v6.4s, #12 - shl v6.4s, v6.4s, #20 - orr v6.16b, v6.16b, v16.16b - tbl v2.16b, { v26.16b, v27.16b }, v2.16b - add v3.4s, v3.4s, v6.4s - ext v19.16b, v2.16b, v2.16b, #12 - eor v4.16b, v4.16b, v3.16b - uzp1 v2.4s, v2.4s, v19.4s - ext v3.16b, v3.16b, v3.16b, #12 - tbl v4.16b, { v4.16b }, v1.16b - add v2.4s, v3.4s, v2.4s - add v3.4s, v5.4s, v4.4s - eor v5.16b, v6.16b, v3.16b - ushr v6.4s, v5.4s, #7 - shl v5.4s, v5.4s, #25 - orr v5.16b, v5.16b, v6.16b - ext v4.16b, v4.16b, v4.16b, #8 - add v2.4s, v2.4s, v5.4s - eor v4.16b, v2.16b, v4.16b - ext v3.16b, v3.16b, v3.16b, #4 - tbl v0.16b, { v4.16b }, v0.16b - add v3.4s, v3.4s, v0.4s - eor v4.16b, v5.16b, v3.16b - ushr v5.4s, v4.4s, #12 - shl v4.4s, v4.4s, #20 - add v2.4s, v2.4s, v7.4s - orr v4.16b, v4.16b, v5.16b - add v2.4s, v2.4s, v4.4s + hint #25 + .cfi_negate_ra_state + sub sp, sp, #96 + stp x29, x30, [sp, #64] + add x29, sp, #64 + stp x20, x19, [sp, #80] + .cfi_def_cfa w29, 32 + .cfi_offset w19, -8 + .cfi_offset w20, -16 + .cfi_offset w30, -24 + .cfi_offset w29, -32 + mov x20, x0 + mov x19, x5 + mov w5, w4 + mov x4, x3 + mov w3, w2 + mov x2, x1 + mov x0, sp + mov x1, x20 + bl compress_pre + ldp q0, q1, [sp] + ldp q2, q3, [sp, #32] + eor v0.16b, v2.16b, v0.16b + eor v1.16b, v3.16b, v1.16b + ldp x29, x30, [sp, #64] + stp q0, q1, [x19] + ldr q0, [x20] eor v0.16b, v0.16b, v2.16b - tbl v0.16b, { v0.16b }, v1.16b - add v1.4s, v3.4s, v0.4s - eor v3.16b, v4.16b, v1.16b - ushr v4.4s, v3.4s, #7 - shl v3.4s, v3.4s, #25 - ext v2.16b, v2.16b, v2.16b, #4 - ext v0.16b, v0.16b, v0.16b, #8 - ext v1.16b, v1.16b, v1.16b, #12 - orr v3.16b, v3.16b, v4.16b - eor v2.16b, v2.16b, v1.16b - eor v3.16b, v3.16b, v0.16b - stp q2, q3, [x5] - ldr q2, [x0] - eor v1.16b, v2.16b, v1.16b - str q1, [x5, #32] - ldr q1, [x0, #16] - eor v0.16b, v1.16b, v0.16b - str q0, [x5, #48] + str q0, [x19, #32] + ldr q0, [x20, #16] + eor v0.16b, v0.16b, v3.16b + str q0, [x19, #48] + ldp x20, x19, [sp, #80] + add sp, sp, #96 + hint #29 ret -.Lfunc_end1: - .size zfs_blake3_compress_xof_sse41, .Lfunc_end1-zfs_blake3_compress_xof_sse41 +.Lfunc_end2: + .size zfs_blake3_compress_xof_sse41, .Lfunc_end2-zfs_blake3_compress_xof_sse41 .cfi_endproc .section .rodata.cst16,"aM",@progbits,16 .p2align 4 -.LCPI2_0: +.LCPI3_0: .word 0 .word 1 .word 2 .word 3 -.LCPI2_1: +.LCPI3_1: .byte 2 .byte 3 .byte 0 @@ -1078,7 +628,7 @@ zfs_blake3_compress_xof_sse41: .byte 15 .byte 12 .byte 13 -.LCPI2_2: +.LCPI3_2: .byte 1 .byte 2 .byte 3 @@ -1095,25 +645,29 @@ zfs_blake3_compress_xof_sse41: .byte 14 .byte 15 .byte 12 +.LCPI3_3: + .word 1779033703 + .word 3144134277 + .word 1013904242 + .word 2773480762 .text .globl zfs_blake3_hash_many_sse41 .p2align 2 .type zfs_blake3_hash_many_sse41,@function zfs_blake3_hash_many_sse41: .cfi_startproc - stp d15, d14, [sp, #-160]! + hint #34 + stp d15, d14, [sp, #-144]! stp d13, d12, [sp, #16] stp d11, d10, [sp, #32] stp d9, d8, [sp, #48] - stp x29, x30, [sp, #64] - stp x28, x27, [sp, #80] - stp x26, x25, [sp, #96] - stp x24, x23, [sp, #112] - stp x22, x21, [sp, #128] - stp x20, x19, [sp, #144] - mov x29, sp - sub sp, sp, #448 - .cfi_def_cfa w29, 160 + stp x29, x27, [sp, #64] + stp x26, x25, [sp, #80] + stp x24, x23, [sp, #96] + stp x22, x21, [sp, #112] + stp x20, x19, [sp, #128] + sub sp, sp, #368 + .cfi_def_cfa_offset 512 .cfi_offset w19, -8 .cfi_offset w20, -16 .cfi_offset w21, -24 @@ -1123,1341 +677,1722 @@ zfs_blake3_hash_many_sse41: .cfi_offset w25, -56 .cfi_offset w26, -64 .cfi_offset w27, -72 - .cfi_offset w28, -80 - .cfi_offset w30, -88 - .cfi_offset w29, -96 - .cfi_offset b8, -104 - .cfi_offset b9, -112 - .cfi_offset b10, -120 - .cfi_offset b11, -128 - .cfi_offset b12, -136 - .cfi_offset b13, -144 - .cfi_offset b14, -152 - .cfi_offset b15, -160 - ldr x26, [x29, #168] - ldrb w27, [x29, #160] - mov w19, w6 - mov x20, x4 - mov x22, x2 - mov x28, x1 + .cfi_offset w29, -80 + .cfi_offset b8, -88 + .cfi_offset b9, -96 + .cfi_offset b10, -104 + .cfi_offset b11, -112 + .cfi_offset b12, -120 + .cfi_offset b13, -128 + .cfi_offset b14, -136 + .cfi_offset b15, -144 + ldr x8, [sp, #520] + adrp x11, .LCPI3_1 + ldrb w9, [sp, #512] + adrp x10, .LCPI3_2 cmp x1, #4 - mov x24, x0 - str x3, [sp, #40] - b.lo .LBB2_8 - adrp x11, .LCPI2_0 - ldr q0, [x11, :lo12:.LCPI2_0] + b.lo .LBB3_6 + adrp x12, .LCPI3_0 sbfx w13, w5, #0, #1 + mov w15, #58983 + mov w16, #44677 + movk w15, #27145, lsl #16 + movk w16, #47975, lsl #16 + ldr q0, [x12, :lo12:.LCPI3_0] dup v1.4s, w13 - mov w10, #58983 - mov w11, #44677 - mov w12, #62322 + movi v13.4s, #64 + mov w13, #62322 + mov w14, #62778 + orr w12, w7, w6 and v0.16b, v1.16b, v0.16b - mov w13, #62778 - orr w8, w7, w19 - adrp x9, .LCPI2_1 - movk w10, #27145, lsl #16 - movk w11, #47975, lsl #16 - movk w12, #15470, lsl #16 - movk w13, #42319, lsl #16 - str q0, [sp, #16] + ldr q1, [x11, :lo12:.LCPI3_1] + movk w13, #15470, lsl #16 + movk w14, #42319, lsl #16 + dup v14.4s, w15 + stp q0, q1, [sp, #16] orr v0.4s, #128, lsl #24 - adrp x14, .LCPI2_2 str q0, [sp] -.LBB2_2: - ldr x2, [sp, #40] - mov x15, x2 - ld1r { v7.4s }, [x15], #4 - add x16, x2, #8 - add x17, x2, #12 - add x18, x2, #16 - add x0, x2, #20 - add x3, x2, #24 - add x2, x2, #28 - ld1r { v6.4s }, [x16] - ld1r { v17.4s }, [x17] - ld1r { v10.4s }, [x18] - ld1r { v11.4s }, [x0] - ld1r { v19.4s }, [x3] - ld1r { v18.4s }, [x15] - ld1r { v16.4s }, [x2] - cbz x22, .LBB2_7 + dup v0.4s, w16 + stp q0, q14, [sp, #48] + b .LBB3_3 +.LBB3_2: + zip1 v0.4s, v29.4s, v8.4s + add x15, x4, #4 + zip1 v1.4s, v30.4s, v31.4s + tst w5, #0x1 + zip1 v2.4s, v24.4s, v18.4s + csel x4, x15, x4, ne + zip1 v3.4s, v25.4s, v26.4s + add x0, x0, #32 + zip2 v6.4s, v29.4s, v8.4s + sub x1, x1, #4 + zip1 v4.2d, v0.2d, v1.2d + cmp x1, #3 + zip2 v7.4s, v30.4s, v31.4s + zip1 v5.2d, v2.2d, v3.2d + zip2 v0.2d, v0.2d, v1.2d + zip2 v1.2d, v2.2d, v3.2d + zip2 v2.4s, v24.4s, v18.4s + zip2 v3.4s, v25.4s, v26.4s + stp q4, q5, [x8] + zip2 v4.2d, v6.2d, v7.2d + stp q0, q1, [x8, #32] + zip1 v0.2d, v6.2d, v7.2d + zip1 v1.2d, v2.2d, v3.2d + zip2 v2.2d, v2.2d, v3.2d + stp q0, q1, [x8, #64] + stp q4, q2, [x8, #96] + add x8, x8, #128 + b.ls .LBB3_6 +.LBB3_3: + mov x15, x3 + add x16, x3, #8 + add x17, x3, #12 + add x19, x3, #16 + add x20, x3, #20 + ld1r { v29.4s }, [x15], #4 + ld1r { v30.4s }, [x16] + add x16, x3, #24 + ld1r { v31.4s }, [x17] + add x17, x3, #28 + ld1r { v24.4s }, [x19] + ld1r { v18.4s }, [x20] + ld1r { v25.4s }, [x16] + ld1r { v8.4s }, [x15] + ld1r { v26.4s }, [x17] + cbz x2, .LBB3_2 ldr q1, [sp, #16] - dup v0.4s, w20 - ldp x15, x16, [x24] - ldp x17, x18, [x24, #16] + dup v0.4s, w4 + lsr x17, x4, #32 + mov x15, xzr + ldp x19, x20, [x0, #16] add v1.4s, v0.4s, v1.4s + mov x21, x2 movi v0.4s, #128, lsl #24 - str q1, [sp, #64] + mov w26, w12 + str q1, [sp, #96] eor v0.16b, v1.16b, v0.16b ldr q1, [sp] - lsr x2, x20, #32 - mov x0, xzr - mov w6, w8 cmgt v0.4s, v1.4s, v0.4s - dup v1.4s, w2 + dup v1.4s, w17 + ldp x16, x17, [x0] sub v0.4s, v1.4s, v0.4s - str q0, [sp, #48] -.LBB2_4: - mov w4, #16 - stp q16, q17, [sp, #192] - bfi x4, x0, #6, #58 - ldr q1, [x15, x4] - ldr q3, [x16, x4] - ldr q2, [x17, x4] - ldr q4, [x18, x4] - mov w4, #32 - bfi x4, x0, #6, #58 - ldr q5, [x15, x4] - ldr q20, [x16, x4] - ldr q21, [x17, x4] - ldr q22, [x18, x4] - mov w4, #48 - lsl x3, x0, #6 - bfi x4, x0, #6, #58 - add x0, x0, #1 - ldr q0, [x15, x3] - ldr q23, [x16, x3] - ldr q16, [x17, x3] - ldr q17, [x18, x3] - cmp x0, x22 - ldr q25, [x15, x4] - ldr q14, [x16, x4] - ldr q28, [x17, x4] - ldr q31, [x18, x4] - csel w4, w27, wzr, eq - orr w4, w4, w6 - mov x2, xzr - and w6, w4, #0xff - add x3, x3, #256 -.LBB2_5: - ldr x4, [x24, x2] - add x2, x2, #8 - cmp x2, #32 - add x4, x4, x3 - prfm pldl1keep, [x4] - b.ne .LBB2_5 - zip1 v29.4s, v0.4s, v23.4s - zip2 v23.4s, v0.4s, v23.4s - zip1 v0.4s, v16.4s, v17.4s - zip2 v24.4s, v16.4s, v17.4s - zip1 v9.4s, v1.4s, v3.4s - zip2 v26.4s, v1.4s, v3.4s - zip1 v27.4s, v2.4s, v4.4s - zip2 v17.4s, v2.4s, v4.4s - zip1 v12.4s, v21.4s, v22.4s - zip2 v13.4s, v21.4s, v22.4s - add v2.4s, v7.4s, v10.4s - add v1.4s, v18.4s, v11.4s - ext v7.16b, v0.16b, v29.16b, #8 - ext v22.16b, v24.16b, v23.16b, #8 - zip1 v30.4s, v5.4s, v20.4s - zip2 v20.4s, v5.4s, v20.4s - stp q1, q2, [sp, #112] - ext v2.16b, v29.16b, v7.16b, #8 - mov v29.d[1], v0.d[0] - ext v18.16b, v23.16b, v22.16b, #8 - mov v23.d[1], v24.d[0] - zip1 v21.4s, v25.4s, v14.4s - zip2 v4.4s, v25.4s, v14.4s - zip1 v14.4s, v28.4s, v31.4s - zip2 v15.4s, v28.4s, v31.4s - add v8.4s, v6.4s, v19.4s - ext v28.16b, v27.16b, v9.16b, #8 - ext v31.16b, v17.16b, v26.16b, #8 - stur q2, [x29, #-208] - mov v7.16b, v29.16b - ext v0.16b, v12.16b, v30.16b, #8 - stp q23, q29, [x29, #-80] - mov v2.16b, v19.16b - ext v19.16b, v13.16b, v20.16b, #8 - mov v29.16b, v9.16b - ext v25.16b, v9.16b, v28.16b, #8 - mov v29.d[1], v27.d[0] - ext v24.16b, v26.16b, v31.16b, #8 - mov v26.d[1], v17.d[0] - ext v17.16b, v15.16b, v4.16b, #8 - ext v27.16b, v30.16b, v0.16b, #8 - ext v0.16b, v20.16b, v19.16b, #8 - stp q0, q25, [sp, #80] - ext v0.16b, v4.16b, v17.16b, #8 - str q0, [sp, #224] - ldr q0, [sp, #128] - mov v6.16b, v23.16b - mov v22.16b, v4.16b - ldr q16, [x9, :lo12:.LCPI2_1] - add v17.4s, v0.4s, v7.4s - ldr q0, [sp, #112] - mov v30.d[1], v12.d[0] - add v7.4s, v8.4s, v29.4s - mov v20.d[1], v13.d[0] - add v4.4s, v0.4s, v6.4s - ldr q0, [sp, #64] - dup v3.4s, w12 - ext v28.16b, v14.16b, v21.16b, #8 - dup v1.4s, w10 - eor v19.16b, v17.16b, v0.16b - ldr q0, [sp, #48] - ext v23.16b, v21.16b, v28.16b, #8 - mov v21.d[1], v14.d[0] - tbl v14.16b, { v19.16b }, v16.16b - eor v12.16b, v4.16b, v0.16b - movi v0.4s, #64 - eor v13.16b, v7.16b, v0.16b - tbl v13.16b, { v13.16b }, v16.16b - add v6.4s, v13.4s, v3.4s - dup v5.4s, w11 - tbl v12.16b, { v12.16b }, v16.16b - add v1.4s, v14.4s, v1.4s - eor v9.16b, v6.16b, v2.16b - ldp q2, q0, [sp, #192] - add v5.4s, v12.4s, v5.4s - eor v19.16b, v1.16b, v10.16b - eor v10.16b, v5.16b, v11.16b - ushr v11.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v11.16b, v19.16b, v11.16b - ushr v19.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - mov v22.d[1], v15.d[0] - orr v10.16b, v10.16b, v19.16b - ushr v19.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - add v15.4s, v0.4s, v2.4s - orr v9.16b, v9.16b, v19.16b - dup v19.4s, w6 - add v15.4s, v15.4s, v26.4s - eor v19.16b, v15.16b, v19.16b - tbl v3.16b, { v19.16b }, v16.16b - dup v19.4s, w13 - add v8.4s, v3.4s, v19.4s - ldur q31, [x29, #-208] - eor v19.16b, v8.16b, v2.16b - ushr v0.4s, v19.4s, #12 - shl v19.4s, v19.4s, #20 - orr v2.16b, v19.16b, v0.16b - ldr q19, [x14, :lo12:.LCPI2_2] - add v17.4s, v17.4s, v31.4s - add v17.4s, v17.4s, v11.4s - eor v14.16b, v14.16b, v17.16b - tbl v14.16b, { v14.16b }, v19.16b - add v1.4s, v1.4s, v14.4s - eor v11.16b, v1.16b, v11.16b - add v4.4s, v4.4s, v18.4s - ushr v0.4s, v11.4s, #7 - shl v11.4s, v11.4s, #25 - add v4.4s, v4.4s, v10.4s - orr v0.16b, v11.16b, v0.16b - eor v11.16b, v12.16b, v4.16b - tbl v11.16b, { v11.16b }, v19.16b - add v5.4s, v5.4s, v11.4s - eor v10.16b, v5.16b, v10.16b - add v7.4s, v7.4s, v25.4s - ushr v12.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - add v7.4s, v7.4s, v9.4s - orr v10.16b, v10.16b, v12.16b - eor v12.16b, v13.16b, v7.16b - tbl v12.16b, { v12.16b }, v19.16b - add v6.4s, v6.4s, v12.4s - eor v9.16b, v6.16b, v9.16b - ushr v13.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - orr v9.16b, v9.16b, v13.16b - add v13.4s, v15.4s, v24.4s - add v13.4s, v13.4s, v2.4s - eor v3.16b, v3.16b, v13.16b - tbl v3.16b, { v3.16b }, v19.16b - add v8.4s, v8.4s, v3.4s - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v30.4s - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v20.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v21.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v22.4s - mov v28.16b, v26.16b - stur q26, [x29, #-112] - mov v26.16b, v18.16b - mov v18.16b, v24.16b - stur q24, [x29, #-160] - add v6.4s, v6.4s, v3.4s - mov v24.16b, v20.16b - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - ldr q20, [sp, #80] - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v13.16b - stp q30, q22, [x29, #-192] - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - mov v30.16b, v27.16b - add v17.4s, v17.4s, v27.4s - ldr q27, [sp, #224] - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 + str q0, [sp, #80] +.LBB3_5: + add x23, x16, x15 + add x24, x17, x15 + add x22, x19, x15 + add x25, x20, x15 + subs x21, x21, #1 + add x15, x15, #64 + ldp q1, q2, [x23] + csel w27, w9, wzr, eq + orr w26, w27, w26 + and w26, w26, #0xff + ldp q4, q5, [x24] + dup v0.4s, w26 + mov w26, w6 + zip1 v22.4s, v1.4s, v4.4s + zip2 v20.4s, v1.4s, v4.4s + ldp q6, q7, [x22] + zip1 v17.4s, v2.4s, v5.4s + zip2 v23.4s, v2.4s, v5.4s + ldp q16, q21, [x25] + zip1 v19.4s, v6.4s, v16.4s + zip2 v1.4s, v6.4s, v16.4s + ldp q27, q28, [x23, #32] + zip1 v4.4s, v7.4s, v21.4s + zip2 v5.4s, v7.4s, v21.4s + zip2 v15.2d, v17.2d, v4.2d + ldp q9, q10, [x24, #32] + mov v17.d[1], v4.d[0] + add v4.4s, v30.4s, v25.4s + zip2 v11.2d, v23.2d, v5.2d + zip2 v3.4s, v27.4s, v9.4s + zip1 v7.4s, v27.4s, v9.4s + ldp q12, q6, [x22, #32] + mov v23.d[1], v5.d[0] + stp q11, q3, [sp, #256] + add v5.4s, v31.4s, v26.4s + add v4.4s, v4.4s, v17.4s + str q23, [sp, #352] + ldp q16, q2, [x25, #32] + add v5.4s, v5.4s, v23.4s + zip1 v3.4s, v12.4s, v16.4s eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v20.4s - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v23.4s - orr v0.16b, v0.16b, v15.16b - tbl v3.16b, { v3.16b }, v19.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v27.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v13.16b - stur q21, [x29, #-144] - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - ldur q21, [x29, #-80] - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - eor v0.16b, v5.16b, v0.16b - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v15.16b - add v17.4s, v17.4s, v21.4s - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v26.4s - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v18.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v29.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b + zip1 v9.4s, v6.4s, v2.4s + zip2 v2.4s, v6.4s, v2.4s + stp q7, q3, [sp, #208] + zip2 v3.4s, v12.4s, v16.4s + zip1 v12.4s, v28.4s, v10.4s + zip2 v10.4s, v28.4s, v10.4s + stp q17, q2, [sp, #160] + zip2 v28.2d, v22.2d, v19.2d + mov v22.d[1], v19.d[0] + str q3, [sp, #240] + add v2.4s, v8.4s, v18.4s + eor v16.16b, v4.16b, v13.16b + dup v17.4s, w13 + mov v3.16b, v22.16b + stp q22, q28, [sp, #320] + zip2 v22.2d, v20.2d, v1.2d + mov v20.d[1], v1.d[0] + add v1.4s, v29.4s, v24.4s + add v4.4s, v4.4s, v15.4s add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - eor v3.16b, v3.16b, v13.16b - ldur q22, [x29, #-64] - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v16.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - add v17.4s, v17.4s, v28.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v24.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v22.4s - orr v2.16b, v2.16b, v15.16b - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v23.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - eor v3.16b, v3.16b, v13.16b - ldur q22, [x29, #-144] - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v19.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v31.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v22.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v30.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v27.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - ldr q27, [sp, #96] - mov v21.16b, v26.16b - stur q26, [x29, #-96] - mov v28.16b, v31.16b - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v13.16b - ldp q31, q26, [x29, #-192] - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - add v17.4s, v17.4s, v20.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v27.4s - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v26.4s - orr v0.16b, v0.16b, v15.16b - tbl v3.16b, { v3.16b }, v19.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v31.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v13.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - eor v0.16b, v5.16b, v0.16b - mov v18.16b, v24.16b - mov v24.16b, v20.16b - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - ldur q20, [x29, #-160] - orr v0.16b, v0.16b, v15.16b - add v17.4s, v17.4s, v21.4s - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v18.4s - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v23.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v20.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - eor v3.16b, v3.16b, v13.16b - ldur q25, [x29, #-80] - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v16.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - add v17.4s, v17.4s, v29.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v22.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v25.4s - orr v2.16b, v2.16b, v15.16b - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v26.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - ldur q25, [x29, #-112] - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - eor v3.16b, v3.16b, v13.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v19.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v25.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v30.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v24.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v31.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - ldur q25, [x29, #-64] - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v13.16b - ldr q31, [sp, #224] - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - add v17.4s, v17.4s, v27.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v25.4s - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v31.4s - orr v0.16b, v0.16b, v15.16b - tbl v3.16b, { v3.16b }, v19.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v28.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v13.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - eor v0.16b, v5.16b, v0.16b - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v15.16b - add v17.4s, v17.4s, v18.4s - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v22.4s - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v26.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v23.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - mov v21.16b, v29.16b - stur q29, [x29, #-128] - mov v29.16b, v30.16b - mov v30.16b, v27.16b - mov v27.16b, v18.16b - str q18, [sp, #176] - eor v0.16b, v0.16b, v1.16b - mov v18.16b, v22.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - eor v3.16b, v3.16b, v13.16b - ldur q22, [x29, #-96] - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v16.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - add v17.4s, v17.4s, v20.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v29.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v22.4s - orr v2.16b, v2.16b, v15.16b - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v31.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - eor v3.16b, v3.16b, v13.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v19.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v21.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v24.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v30.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v28.4s - add v6.4s, v6.4s, v3.4s - mov v22.16b, v24.16b - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - ldur q24, [x29, #-80] - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - mov v21.16b, v30.16b - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v13.16b - ldur q30, [x29, #-192] - mov v20.16b, v29.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - ldur q29, [x29, #-112] - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - add v17.4s, v17.4s, v25.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v24.4s - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v30.4s - orr v0.16b, v0.16b, v15.16b - tbl v3.16b, { v3.16b }, v19.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v29.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v13.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - eor v0.16b, v5.16b, v0.16b - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v15.16b - add v17.4s, v17.4s, v18.4s - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v20.4s - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v31.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v26.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - eor v3.16b, v3.16b, v13.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v16.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - add v17.4s, v17.4s, v23.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v22.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v27.4s - orr v2.16b, v2.16b, v15.16b - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v30.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - ldur q27, [x29, #-160] - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - eor v3.16b, v3.16b, v13.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v19.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v27.4s - mov v28.16b, v25.16b - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v21.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v28.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v29.4s - mov v25.16b, v31.16b - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - ldur q31, [x29, #-96] - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v13.16b - ldur q28, [x29, #-208] - mov v18.16b, v20.16b - str q20, [sp, #144] - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - ldur q20, [x29, #-128] - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - add v17.4s, v17.4s, v24.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v31.4s - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v28.4s - orr v0.16b, v0.16b, v15.16b - tbl v3.16b, { v3.16b }, v19.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v13.4s, v13.4s, v20.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v0.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v13.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v1.16b, v2.16b - add v5.4s, v5.4s, v12.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - eor v0.16b, v5.16b, v0.16b - orr v2.16b, v2.16b, v15.16b - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v15.16b - add v17.4s, v17.4s, v18.4s - add v17.4s, v17.4s, v0.4s - add v4.4s, v4.4s, v22.4s - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v30.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v25.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v16.16b - eor v3.16b, v3.16b, v13.16b - add v17.4s, v17.4s, v26.4s - mov v26.16b, v21.16b - add v4.4s, v4.4s, v21.4s - ldur q21, [x29, #-144] - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v16.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v8.16b, v2.16b - add v17.4s, v17.4s, v0.4s - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - eor v14.16b, v14.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v7.4s, v7.4s, v21.4s - orr v2.16b, v2.16b, v15.16b - tbl v14.16b, { v14.16b }, v19.16b - eor v11.16b, v11.16b, v4.16b - add v7.4s, v7.4s, v9.4s - add v13.4s, v13.4s, v28.4s - add v1.4s, v1.4s, v14.4s - tbl v11.16b, { v11.16b }, v19.16b - eor v12.16b, v12.16b, v7.16b - add v13.4s, v13.4s, v2.4s - str q23, [sp, #160] - eor v0.16b, v0.16b, v1.16b - add v5.4s, v5.4s, v11.4s - tbl v12.16b, { v12.16b }, v19.16b - eor v3.16b, v3.16b, v13.16b - add v17.4s, v17.4s, v23.4s - ldur q23, [x29, #-64] - ushr v15.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v10.16b, v5.16b, v10.16b - add v6.4s, v6.4s, v12.4s - tbl v3.16b, { v3.16b }, v19.16b - orr v0.16b, v0.16b, v15.16b - ushr v15.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - eor v9.16b, v6.16b, v9.16b - add v8.4s, v8.4s, v3.4s - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v2.16b, v8.16b, v2.16b - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v23.4s - orr v2.16b, v2.16b, v15.16b - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v24.4s - tbl v3.16b, { v3.16b }, v16.16b - eor v14.16b, v14.16b, v4.16b - add v7.4s, v7.4s, v2.4s - add v6.4s, v6.4s, v3.4s - tbl v14.16b, { v14.16b }, v16.16b - eor v11.16b, v11.16b, v7.16b - add v13.4s, v13.4s, v20.4s - eor v10.16b, v6.16b, v10.16b - add v8.4s, v8.4s, v14.4s - tbl v11.16b, { v11.16b }, v16.16b - add v13.4s, v13.4s, v0.4s - ldr q20, [sp, #176] - ushr v15.4s, v10.4s, #12 - shl v10.4s, v10.4s, #20 - eor v9.16b, v8.16b, v9.16b - add v1.4s, v1.4s, v11.4s - eor v12.16b, v12.16b, v13.16b - orr v10.16b, v10.16b, v15.16b - ushr v15.4s, v9.4s, #12 - shl v9.4s, v9.4s, #20 - eor v2.16b, v1.16b, v2.16b - tbl v12.16b, { v12.16b }, v16.16b - orr v9.16b, v9.16b, v15.16b - ushr v15.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - add v5.4s, v5.4s, v12.4s - add v17.4s, v17.4s, v31.4s - orr v2.16b, v2.16b, v15.16b - eor v0.16b, v5.16b, v0.16b - add v17.4s, v17.4s, v10.4s - add v4.4s, v4.4s, v20.4s - add v7.4s, v7.4s, v29.4s - ushr v15.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v3.16b, v3.16b, v17.16b - add v4.4s, v4.4s, v9.4s - add v7.4s, v7.4s, v2.4s - orr v0.16b, v0.16b, v15.16b - mov v15.16b, v31.16b - add v17.4s, v17.4s, v22.4s - eor v31.16b, v14.16b, v4.16b - eor v22.16b, v11.16b, v7.16b - add v11.4s, v13.4s, v27.4s - tbl v3.16b, { v3.16b }, v19.16b - add v11.4s, v11.4s, v0.4s - tbl v31.16b, { v31.16b }, v19.16b - add v6.4s, v6.4s, v3.4s - eor v12.16b, v12.16b, v11.16b - tbl v22.16b, { v22.16b }, v19.16b - add v8.4s, v8.4s, v31.4s - eor v10.16b, v6.16b, v10.16b - add v30.4s, v11.4s, v30.4s - tbl v11.16b, { v12.16b }, v19.16b - add v1.4s, v1.4s, v22.4s - eor v9.16b, v8.16b, v9.16b - ushr v12.4s, v10.4s, #7 - shl v10.4s, v10.4s, #25 - add v5.4s, v5.4s, v11.4s - eor v2.16b, v1.16b, v2.16b - orr v10.16b, v10.16b, v12.16b - ushr v12.4s, v9.4s, #7 - shl v9.4s, v9.4s, #25 - eor v0.16b, v5.16b, v0.16b - orr v9.16b, v9.16b, v12.16b - ushr v12.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - orr v2.16b, v2.16b, v12.16b - ushr v12.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v12.16b - add v4.4s, v4.4s, v26.4s - add v17.4s, v17.4s, v0.4s - add v7.4s, v7.4s, v28.4s - mov v18.16b, v27.16b - eor v31.16b, v31.16b, v17.16b - add v4.4s, v4.4s, v10.4s - add v27.4s, v30.4s, v2.4s - eor v22.16b, v22.16b, v4.16b - add v7.4s, v7.4s, v9.4s - eor v3.16b, v3.16b, v27.16b - add v26.4s, v27.4s, v29.4s - tbl v27.16b, { v31.16b }, v16.16b - eor v28.16b, v11.16b, v7.16b - tbl v22.16b, { v22.16b }, v16.16b - add v1.4s, v1.4s, v27.4s - add v4.4s, v4.4s, v23.4s - ldr q23, [sp, #144] - tbl v28.16b, { v28.16b }, v16.16b - tbl v3.16b, { v3.16b }, v16.16b - add v5.4s, v5.4s, v22.4s - eor v0.16b, v0.16b, v1.16b - add v6.4s, v6.4s, v28.4s - add v29.4s, v8.4s, v3.4s - eor v30.16b, v5.16b, v10.16b - ushr v8.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - eor v31.16b, v6.16b, v9.16b - orr v0.16b, v0.16b, v8.16b - ushr v8.4s, v30.4s, #12 - shl v30.4s, v30.4s, #20 - eor v2.16b, v29.16b, v2.16b - orr v30.16b, v30.16b, v8.16b - ushr v8.4s, v31.4s, #12 - shl v31.4s, v31.4s, #20 - add v17.4s, v17.4s, v25.4s - add v7.4s, v7.4s, v23.4s - orr v31.16b, v31.16b, v8.16b - ushr v8.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - ldur q23, [x29, #-176] - orr v2.16b, v2.16b, v8.16b - add v17.4s, v17.4s, v0.4s - eor v27.16b, v27.16b, v17.16b - add v4.4s, v4.4s, v30.4s - add v25.4s, v26.4s, v2.4s - eor v22.16b, v22.16b, v4.16b - add v4.4s, v4.4s, v24.4s - add v7.4s, v7.4s, v31.4s - eor v3.16b, v3.16b, v25.16b - add v24.4s, v25.4s, v18.4s - tbl v25.16b, { v27.16b }, v19.16b - add v17.4s, v17.4s, v23.4s - eor v23.16b, v28.16b, v7.16b - tbl v22.16b, { v22.16b }, v19.16b - add v1.4s, v1.4s, v25.4s - tbl v23.16b, { v23.16b }, v19.16b - tbl v3.16b, { v3.16b }, v19.16b - add v5.4s, v5.4s, v22.4s - eor v0.16b, v0.16b, v1.16b - add v6.4s, v6.4s, v23.4s - add v26.4s, v29.4s, v3.4s - eor v27.16b, v5.16b, v30.16b - ushr v29.4s, v0.4s, #7 - shl v0.4s, v0.4s, #25 - eor v28.16b, v6.16b, v31.16b - orr v0.16b, v0.16b, v29.16b - ushr v29.4s, v27.4s, #7 - shl v27.4s, v27.4s, #25 - eor v2.16b, v26.16b, v2.16b - orr v27.16b, v27.16b, v29.16b - ushr v29.4s, v28.4s, #7 - shl v28.4s, v28.4s, #25 - ldur q18, [x29, #-128] - orr v28.16b, v28.16b, v29.16b - ushr v29.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - add v7.4s, v7.4s, v15.4s - orr v2.16b, v2.16b, v29.16b - add v17.4s, v17.4s, v27.4s - add v4.4s, v4.4s, v28.4s - add v7.4s, v7.4s, v2.4s - eor v3.16b, v3.16b, v17.16b - add v17.4s, v17.4s, v20.4s - eor v20.16b, v25.16b, v4.16b - add v4.4s, v4.4s, v21.4s - eor v21.16b, v22.16b, v7.16b - add v7.4s, v7.4s, v18.4s - add v18.4s, v24.4s, v0.4s - eor v22.16b, v23.16b, v18.16b - ldr q23, [sp, #160] - tbl v3.16b, { v3.16b }, v16.16b - tbl v20.16b, { v20.16b }, v16.16b - add v6.4s, v6.4s, v3.4s - add v18.4s, v18.4s, v23.4s - tbl v21.16b, { v21.16b }, v16.16b - tbl v16.16b, { v22.16b }, v16.16b - add v22.4s, v26.4s, v20.4s - eor v23.16b, v6.16b, v27.16b - add v1.4s, v1.4s, v21.4s - eor v24.16b, v22.16b, v28.16b - ushr v25.4s, v23.4s, #12 - shl v23.4s, v23.4s, #20 - add v5.4s, v5.4s, v16.4s - eor v2.16b, v1.16b, v2.16b - orr v23.16b, v23.16b, v25.16b - ushr v25.4s, v24.4s, #12 + add v2.4s, v2.4s, v20.4s + stp q15, q20, [sp, #288] + add v1.4s, v1.4s, v3.4s + ldr q3, [sp, #96] + dup v20.4s, w14 + mov v23.16b, v22.16b + mov v15.16b, v10.16b + eor v6.16b, v1.16b, v3.16b + ldr q3, [sp, #80] + add v1.4s, v1.4s, v28.4s + ldr q28, [sp, #272] + str q23, [sp, #128] + eor v7.16b, v2.16b, v3.16b + ldp q27, q3, [sp, #32] + add v2.4s, v2.4s, v22.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v7.16b, { v7.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + tbl v0.16b, { v0.16b }, v27.16b + add v19.4s, v6.4s, v14.4s + add v21.4s, v7.4s, v3.4s + add v30.4s, v16.4s, v17.4s + add v31.4s, v0.4s, v20.4s + eor v24.16b, v19.16b, v24.16b + eor v17.16b, v21.16b, v18.16b + ushr v18.4s, v24.4s, #12 + shl v20.4s, v24.4s, #20 + eor v24.16b, v30.16b, v25.16b + eor v25.16b, v31.16b, v26.16b + ushr v26.4s, v17.4s, #12 + shl v17.4s, v17.4s, #20 + ushr v29.4s, v24.4s, #12 shl v24.4s, v24.4s, #20 - eor v0.16b, v5.16b, v0.16b - orr v24.16b, v24.16b, v25.16b - ushr v25.4s, v2.4s, #12 - shl v2.4s, v2.4s, #20 - orr v2.16b, v2.16b, v25.16b - ushr v25.4s, v0.4s, #12 - shl v0.4s, v0.4s, #20 - orr v0.16b, v0.16b, v25.16b - add v25.4s, v7.4s, v2.4s - add v26.4s, v18.4s, v0.4s - eor v18.16b, v21.16b, v25.16b - add v17.4s, v17.4s, v23.4s - add v4.4s, v4.4s, v24.4s - eor v16.16b, v16.16b, v26.16b - tbl v21.16b, { v18.16b }, v19.16b - eor v3.16b, v3.16b, v17.16b - eor v7.16b, v20.16b, v4.16b - tbl v16.16b, { v16.16b }, v19.16b - add v1.4s, v1.4s, v21.4s - tbl v3.16b, { v3.16b }, v19.16b - tbl v20.16b, { v7.16b }, v19.16b - eor v2.16b, v1.16b, v2.16b - eor v7.16b, v1.16b, v17.16b - add v1.4s, v5.4s, v16.4s - eor v0.16b, v1.16b, v0.16b - eor v18.16b, v1.16b, v4.16b - add v1.4s, v6.4s, v3.4s - eor v4.16b, v1.16b, v23.16b - eor v6.16b, v25.16b, v1.16b - add v1.4s, v22.4s, v20.4s - eor v5.16b, v1.16b, v24.16b - eor v17.16b, v26.16b, v1.16b - ushr v1.4s, v4.4s, #7 - shl v4.4s, v4.4s, #25 - orr v1.16b, v4.16b, v1.16b - ushr v4.4s, v5.4s, #7 - shl v5.4s, v5.4s, #25 - orr v4.16b, v5.16b, v4.16b - ushr v5.4s, v2.4s, #7 - shl v2.4s, v2.4s, #25 - orr v2.16b, v2.16b, v5.16b + ushr v8.4s, v25.4s, #12 + shl v25.4s, v25.4s, #20 + orr v3.16b, v20.16b, v18.16b + ldr q18, [x10, :lo12:.LCPI3_2] + orr v13.16b, v17.16b, v26.16b + orr v24.16b, v24.16b, v29.16b + orr v14.16b, v25.16b, v8.16b + add v8.4s, v1.4s, v3.4s + add v29.4s, v2.4s, v13.4s + add v17.4s, v4.4s, v24.4s + add v20.4s, v5.4s, v14.4s + eor v1.16b, v6.16b, v8.16b + eor v2.16b, v7.16b, v29.16b + eor v4.16b, v16.16b, v17.16b + eor v0.16b, v0.16b, v20.16b + tbl v25.16b, { v1.16b }, v18.16b + tbl v16.16b, { v2.16b }, v18.16b + tbl v6.16b, { v4.16b }, v18.16b + tbl v4.16b, { v0.16b }, v18.16b + add v19.4s, v19.4s, v25.4s + add v21.4s, v21.4s, v16.4s + add v26.4s, v30.4s, v6.4s + add v7.4s, v31.4s, v4.4s + eor v0.16b, v19.16b, v3.16b + eor v1.16b, v21.16b, v13.16b + eor v2.16b, v26.16b, v24.16b + eor v3.16b, v7.16b, v14.16b ushr v5.4s, v0.4s, #7 shl v0.4s, v0.4s, #25 - orr v0.16b, v0.16b, v5.16b - eor v10.16b, v0.16b, v20.16b - eor v11.16b, v1.16b, v21.16b - eor v19.16b, v4.16b, v16.16b - cmp x0, x22 - eor v16.16b, v2.16b, v3.16b - mov w6, w19 - b.ne .LBB2_4 -.LBB2_7: - zip1 v0.4s, v7.4s, v18.4s - zip2 v1.4s, v7.4s, v18.4s - zip1 v2.4s, v6.4s, v17.4s - zip2 v3.4s, v6.4s, v17.4s - zip1 v4.4s, v10.4s, v11.4s - zip2 v5.4s, v10.4s, v11.4s - zip1 v6.4s, v19.4s, v16.4s - zip2 v7.4s, v19.4s, v16.4s - add x15, x20, #4 - tst w5, #0x1 - sub x28, x28, #4 - zip1 v16.2d, v0.2d, v2.2d - zip2 v0.2d, v0.2d, v2.2d - zip1 v2.2d, v1.2d, v3.2d - zip2 v1.2d, v1.2d, v3.2d - zip1 v3.2d, v4.2d, v6.2d - zip2 v4.2d, v4.2d, v6.2d - zip1 v6.2d, v5.2d, v7.2d - zip2 v5.2d, v5.2d, v7.2d - add x24, x24, #32 - csel x20, x15, x20, ne - cmp x28, #3 - stp q16, q3, [x26] - stp q0, q4, [x26, #32] - stp q2, q6, [x26, #64] - stp q1, q5, [x26, #96] - add x26, x26, #128 - b.hi .LBB2_2 -.LBB2_8: - cbz x28, .LBB2_16 - orr w8, w7, w19 - and x21, x5, #0x1 - stur w8, [x29, #-64] -.LBB2_10: - ldr x8, [sp, #40] - ldr x25, [x24] - ldur w4, [x29, #-64] - ldp q1, q0, [x8] - mov x8, x22 - stp q1, q0, [x29, #-48] -.LBB2_11: - subs x23, x8, #1 - b.eq .LBB2_13 - cbnz x8, .LBB2_14 - b .LBB2_15 -.LBB2_13: - orr w4, w4, w27 -.LBB2_14: - sub x0, x29, #48 - mov w2, #64 - mov x1, x25 - mov x3, x20 - bl zfs_blake3_compress_in_place_sse41 - add x25, x25, #64 - mov x8, x23 - mov w4, w19 - b .LBB2_11 -.LBB2_15: - ldp q0, q1, [x29, #-48] - add x20, x20, x21 - add x24, x24, #8 - subs x28, x28, #1 - stp q0, q1, [x26], #32 - b.ne .LBB2_10 -.LBB2_16: - add sp, sp, #448 - ldp x20, x19, [sp, #144] - ldp x22, x21, [sp, #128] - ldp x24, x23, [sp, #112] - ldp x26, x25, [sp, #96] - ldp x28, x27, [sp, #80] - ldp x29, x30, [sp, #64] + ushr v24.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ushr v30.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + orr v5.16b, v0.16b, v5.16b + orr v0.16b, v1.16b, v24.16b + ushr v31.4s, v3.4s, #7 + orr v2.16b, v2.16b, v30.16b + ldp q24, q30, [sp, #208] + shl v3.4s, v3.4s, #25 + zip2 v14.2d, v12.2d, v9.2d + mov v22.16b, v24.16b + orr v1.16b, v3.16b, v31.16b + zip2 v3.2d, v24.2d, v30.2d + mov v24.16b, v28.16b + mov v22.d[1], v30.d[0] + ldr q30, [sp, #240] + mov v31.16b, v12.16b + stp q22, q14, [sp, #224] + mov v24.d[1], v30.d[0] + add v12.4s, v8.4s, v22.4s + mov v31.d[1], v9.d[0] + add v22.4s, v29.4s, v24.4s + ldr q29, [sp, #176] + zip2 v28.2d, v28.2d, v30.2d + mov v9.16b, v24.16b + mov v15.d[1], v29.d[0] + zip2 v8.2d, v10.2d, v29.2d + add v10.4s, v12.4s, v0.4s + add v22.4s, v22.4s, v2.4s + str q9, [sp, #144] + add v20.4s, v20.4s, v15.4s + add v17.4s, v17.4s, v31.4s + stp q3, q8, [sp, #192] + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v10.4s, v10.4s, v3.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v22.4s, v22.4s, v28.4s + ushr v12.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v13.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v8.4s + orr v1.16b, v1.16b, v12.16b + add v17.4s, v17.4s, v14.4s + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v18.16b + tbl v25.16b, { v25.16b }, v18.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v22.4s, v22.4s, v23.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v17.4s, v17.4s, v11.4s + mov v30.16b, v28.16b + mov v28.16b, v23.16b + ldr q23, [sp, #304] + ushr v12.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v22.4s, v22.4s, v0.4s + mov v29.16b, v31.16b + ldr q31, [sp, #160] + orr v5.16b, v5.16b, v13.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v23.4s + orr v1.16b, v1.16b, v12.16b + str q29, [sp, #272] + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v31.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v27.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v27.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v27.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v27.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v5.16b, v19.16b + add v22.4s, v22.4s, v24.4s + ldr q24, [sp, #320] + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v17.4s, v24.4s + ldr q24, [sp, #352] + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v24.4s + ldr q24, [sp, #336] + orr v1.16b, v1.16b, v13.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v14.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v18.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v18.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v18.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v18.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v19.16b, v5.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + add v10.4s, v10.4s, v24.4s + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v22.4s, v22.4s, v29.4s + ushr v13.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v8.4s + ldr q8, [sp, #288] + orr v1.16b, v1.16b, v13.16b + add v17.4s, v17.4s, v3.4s + ldr q3, [sp, #352] + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v10.4s, v10.4s, v30.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v22.4s, v22.4s, v8.4s + mov v24.16b, v30.16b + mov v30.16b, v15.16b + add v17.4s, v17.4s, v15.4s + ldr q15, [sp, #224] + ushr v12.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + str q30, [sp, #176] + orr v5.16b, v5.16b, v13.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v15.4s + orr v1.16b, v1.16b, v12.16b + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v18.16b + tbl v25.16b, { v25.16b }, v18.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v22.4s, v22.4s, v9.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v17.4s, v17.4s, v14.4s + ushr v12.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v13.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v28.4s + orr v1.16b, v1.16b, v12.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v11.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v27.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v27.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v27.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v27.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v5.16b, v19.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + add v22.4s, v22.4s, v29.4s + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v17.4s, v23.4s + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v31.4s + orr v1.16b, v1.16b, v13.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v30.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v18.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v18.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v18.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v18.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v19.16b, v5.16b + add v10.4s, v10.4s, v3.4s + ldr q3, [sp, #192] + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v22.4s, v22.4s, v3.4s + ushr v13.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v15.4s + ldr q15, [sp, #128] + orr v1.16b, v1.16b, v13.16b + add v17.4s, v17.4s, v24.4s + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v21.16b, v5.16b + ldp q23, q11, [sp, #320] + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v10.4s, v10.4s, v8.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v22.4s, v22.4s, v23.4s + ushr v12.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + mov v28.16b, v31.16b + mov v31.16b, v8.16b + ldr q8, [sp, #208] + orr v5.16b, v5.16b, v13.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v11.4s + orr v1.16b, v1.16b, v12.16b + add v17.4s, v17.4s, v8.4s + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v18.16b + tbl v25.16b, { v25.16b }, v18.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v22.4s, v22.4s, v29.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v17.4s, v17.4s, v30.4s + ushr v12.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v13.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v9.4s + orr v1.16b, v1.16b, v12.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v14.4s + ldr q14, [sp, #256] + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v27.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v27.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v27.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v27.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v5.16b, v19.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + add v22.4s, v22.4s, v3.4s + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v17.4s, v15.4s + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v14.4s + orr v1.16b, v1.16b, v13.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v8.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v18.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v18.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v18.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v18.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v19.16b, v5.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + add v10.4s, v10.4s, v28.4s + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v22.4s, v22.4s, v24.4s + ushr v13.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v11.4s + ldr q11, [sp, #304] + orr v1.16b, v1.16b, v13.16b + add v17.4s, v17.4s, v31.4s + ldr q31, [sp, #224] + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v10.4s, v10.4s, v23.4s + ldr q23, [sp, #240] + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v22.4s, v22.4s, v11.4s + mov v30.16b, v8.16b + mov v8.16b, v24.16b + ldr q24, [sp, #352] + ushr v12.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v13.16b + str q8, [sp, #112] + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v24.4s + orr v1.16b, v1.16b, v12.16b + add v17.4s, v17.4s, v31.4s + eor v4.16b, v4.16b, v10.16b + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v18.16b + tbl v25.16b, { v25.16b }, v18.16b + eor v6.16b, v6.16b, v20.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + mov v29.16b, v3.16b + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v21.16b, v5.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v22.4s, v22.4s, v29.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v17.4s, v17.4s, v30.4s + ldr q30, [sp, #272] + ushr v12.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v22.4s, v22.4s, v0.4s + mov v3.16b, v28.16b + ldr q28, [sp, #176] + orr v5.16b, v5.16b, v13.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v30.4s + orr v1.16b, v1.16b, v12.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v28.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v27.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v27.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v27.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v27.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v5.16b, v19.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + add v22.4s, v22.4s, v8.4s + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v17.4s, v9.4s + ldr q9, [sp, #320] + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v22.4s, v22.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v17.4s, v17.4s, v2.4s + add v10.4s, v10.4s, v23.4s + orr v1.16b, v1.16b, v13.16b + eor v16.16b, v16.16b, v22.16b + add v20.4s, v20.4s, v31.4s + eor v6.16b, v6.16b, v17.16b + add v10.4s, v10.4s, v5.4s + tbl v16.16b, { v16.16b }, v18.16b + add v20.4s, v20.4s, v1.4s + tbl v6.16b, { v6.16b }, v18.16b + eor v25.16b, v25.16b, v10.16b + add v21.4s, v21.4s, v16.4s + eor v4.16b, v4.16b, v20.16b + add v26.4s, v26.4s, v6.4s + tbl v25.16b, { v25.16b }, v18.16b + eor v0.16b, v21.16b, v0.16b + tbl v4.16b, { v4.16b }, v18.16b + eor v2.16b, v26.16b, v2.16b + add v19.4s, v19.4s, v25.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + add v7.4s, v7.4s, v4.4s + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v5.16b, v19.16b, v5.16b + add v10.4s, v10.4s, v14.4s + ldr q14, [sp, #288] + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v7.16b, v1.16b + orr v2.16b, v2.16b, v13.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v22.4s, v22.4s, v14.4s + ushr v13.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v12.16b + add v22.4s, v22.4s, v2.4s + add v20.4s, v20.4s, v24.4s + orr v1.16b, v1.16b, v13.16b + eor v4.16b, v4.16b, v10.16b + add v17.4s, v17.4s, v9.4s + eor v25.16b, v25.16b, v22.16b + add v20.4s, v20.4s, v5.4s + tbl v4.16b, { v4.16b }, v27.16b + add v17.4s, v17.4s, v1.4s + tbl v25.16b, { v25.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + add v26.4s, v26.4s, v4.4s + eor v16.16b, v16.16b, v17.16b + add v7.4s, v7.4s, v25.4s + tbl v6.16b, { v6.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v2.16b, v7.16b, v2.16b + add v21.4s, v21.4s, v6.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + add v19.4s, v19.4s, v16.4s + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + eor v5.16b, v21.16b, v5.16b + orr v0.16b, v0.16b, v12.16b + eor v1.16b, v19.16b, v1.16b + add v10.4s, v10.4s, v11.4s + orr v2.16b, v2.16b, v13.16b + ushr v13.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + ushr v12.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + add v22.4s, v22.4s, v15.4s + orr v5.16b, v5.16b, v13.16b + add v20.4s, v20.4s, v3.4s + mov v24.16b, v3.16b + ldr q3, [sp, #336] + orr v1.16b, v1.16b, v12.16b + eor v4.16b, v4.16b, v10.16b + add v22.4s, v22.4s, v2.4s + add v17.4s, v17.4s, v3.4s + add v20.4s, v20.4s, v5.4s + tbl v4.16b, { v4.16b }, v18.16b + eor v25.16b, v25.16b, v22.16b + add v17.4s, v17.4s, v1.4s + eor v6.16b, v6.16b, v20.16b + add v26.4s, v26.4s, v4.4s + tbl v25.16b, { v25.16b }, v18.16b + eor v16.16b, v16.16b, v17.16b + tbl v6.16b, { v6.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + add v7.4s, v7.4s, v25.4s + tbl v16.16b, { v16.16b }, v18.16b + add v21.4s, v21.4s, v6.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + eor v2.16b, v7.16b, v2.16b + add v19.4s, v19.4s, v16.4s + eor v5.16b, v21.16b, v5.16b + orr v0.16b, v0.16b, v12.16b + ushr v12.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + eor v1.16b, v19.16b, v1.16b + ushr v13.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v22.4s, v22.4s, v8.4s + orr v2.16b, v2.16b, v12.16b + ushr v12.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + orr v5.16b, v5.16b, v13.16b + add v22.4s, v22.4s, v0.4s + add v10.4s, v10.4s, v29.4s + ldr q29, [sp, #208] + add v17.4s, v17.4s, v31.4s + orr v1.16b, v1.16b, v12.16b + add v20.4s, v20.4s, v29.4s + eor v16.16b, v16.16b, v22.16b + add v10.4s, v10.4s, v5.4s + add v17.4s, v17.4s, v2.4s + add v20.4s, v20.4s, v1.4s + tbl v16.16b, { v16.16b }, v27.16b + eor v25.16b, v25.16b, v10.16b + eor v6.16b, v6.16b, v17.16b + eor v4.16b, v4.16b, v20.16b + add v21.4s, v21.4s, v16.4s + tbl v25.16b, { v25.16b }, v27.16b + tbl v6.16b, { v6.16b }, v27.16b + tbl v4.16b, { v4.16b }, v27.16b + eor v0.16b, v21.16b, v0.16b + add v19.4s, v19.4s, v25.4s + add v26.4s, v26.4s, v6.4s + add v7.4s, v7.4s, v4.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v5.16b, v5.16b, v19.16b + eor v2.16b, v26.16b, v2.16b + eor v1.16b, v7.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + ushr v12.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v22.4s, v22.4s, v14.4s + mov v8.16b, v31.16b + ushr v13.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + mov v31.16b, v14.16b + ushr v14.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + orr v5.16b, v5.16b, v12.16b + add v22.4s, v22.4s, v0.4s + add v10.4s, v10.4s, v28.4s + ldr q28, [sp, #352] + orr v2.16b, v2.16b, v13.16b + orr v1.16b, v1.16b, v14.16b + add v17.4s, v17.4s, v30.4s + add v20.4s, v20.4s, v3.4s + eor v16.16b, v16.16b, v22.16b + add v10.4s, v10.4s, v5.4s + add v17.4s, v17.4s, v2.4s + add v20.4s, v20.4s, v1.4s + tbl v16.16b, { v16.16b }, v18.16b + eor v25.16b, v25.16b, v10.16b + eor v6.16b, v6.16b, v17.16b + eor v4.16b, v4.16b, v20.16b + add v21.4s, v21.4s, v16.4s + tbl v25.16b, { v25.16b }, v18.16b + tbl v6.16b, { v6.16b }, v18.16b + tbl v4.16b, { v4.16b }, v18.16b + eor v0.16b, v21.16b, v0.16b + add v19.4s, v19.4s, v25.4s + add v26.4s, v26.4s, v6.4s + add v7.4s, v7.4s, v4.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + eor v5.16b, v19.16b, v5.16b + eor v2.16b, v26.16b, v2.16b + eor v1.16b, v7.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + add v10.4s, v10.4s, v23.4s + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ushr v14.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + orr v5.16b, v5.16b, v12.16b + add v10.4s, v10.4s, v0.4s + add v20.4s, v20.4s, v24.4s + ldr q24, [sp, #144] + orr v2.16b, v2.16b, v13.16b + orr v1.16b, v1.16b, v14.16b + add v22.4s, v22.4s, v9.4s + add v17.4s, v17.4s, v11.4s + eor v4.16b, v4.16b, v10.16b + add v20.4s, v20.4s, v5.4s + add v22.4s, v22.4s, v2.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v27.16b + eor v6.16b, v6.16b, v20.16b + eor v25.16b, v25.16b, v22.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + tbl v6.16b, { v6.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + eor v0.16b, v26.16b, v0.16b + add v21.4s, v21.4s, v6.4s + add v7.4s, v7.4s, v25.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + eor v5.16b, v21.16b, v5.16b + eor v2.16b, v7.16b, v2.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + add v10.4s, v10.4s, v15.4s + ushr v14.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + mov v30.16b, v3.16b + ldr q3, [sp, #256] + ushr v12.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + add v10.4s, v10.4s, v0.4s + orr v5.16b, v5.16b, v14.16b + add v20.4s, v20.4s, v3.4s + orr v2.16b, v2.16b, v12.16b + orr v1.16b, v1.16b, v13.16b + add v22.4s, v22.4s, v24.4s + add v17.4s, v17.4s, v28.4s + eor v4.16b, v4.16b, v10.16b + add v20.4s, v20.4s, v5.4s + add v22.4s, v22.4s, v2.4s + add v17.4s, v17.4s, v1.4s + tbl v4.16b, { v4.16b }, v18.16b + eor v6.16b, v6.16b, v20.16b + eor v25.16b, v25.16b, v22.16b + eor v16.16b, v16.16b, v17.16b + add v26.4s, v26.4s, v4.4s + tbl v6.16b, { v6.16b }, v18.16b + tbl v25.16b, { v25.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + eor v0.16b, v26.16b, v0.16b + add v21.4s, v21.4s, v6.4s + add v7.4s, v7.4s, v25.4s + add v19.4s, v19.4s, v16.4s + ushr v12.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + eor v5.16b, v21.16b, v5.16b + eor v2.16b, v7.16b, v2.16b + eor v1.16b, v19.16b, v1.16b + orr v0.16b, v0.16b, v12.16b + ushr v12.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + mov v23.16b, v9.16b + ldr q9, [sp, #112] + ushr v13.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ushr v14.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + orr v5.16b, v5.16b, v12.16b + add v9.4s, v10.4s, v9.4s + orr v2.16b, v2.16b, v13.16b + orr v1.16b, v1.16b, v14.16b + ldr q14, [sp, #64] + add v22.4s, v22.4s, v31.4s + add v17.4s, v17.4s, v30.4s + add v20.4s, v20.4s, v8.4s + add v9.4s, v9.4s, v5.4s + add v22.4s, v22.4s, v0.4s + add v17.4s, v17.4s, v2.4s + add v20.4s, v20.4s, v1.4s + eor v25.16b, v25.16b, v9.16b + eor v16.16b, v16.16b, v22.16b + eor v6.16b, v6.16b, v17.16b + eor v4.16b, v4.16b, v20.16b + tbl v25.16b, { v25.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + tbl v6.16b, { v6.16b }, v27.16b + tbl v4.16b, { v4.16b }, v27.16b + add v19.4s, v19.4s, v25.4s + add v21.4s, v21.4s, v16.4s + add v26.4s, v26.4s, v6.4s + add v7.4s, v7.4s, v4.4s + eor v5.16b, v5.16b, v19.16b + eor v0.16b, v21.16b, v0.16b + eor v2.16b, v26.16b, v2.16b + eor v1.16b, v7.16b, v1.16b + ushr v30.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + ushr v10.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v12.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + ushr v13.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + orr v5.16b, v5.16b, v30.16b + add v30.4s, v9.4s, v29.4s + add v22.4s, v22.4s, v23.4s + ldr q23, [sp, #192] + orr v0.16b, v0.16b, v10.16b + orr v2.16b, v2.16b, v12.16b + orr v1.16b, v1.16b, v13.16b + add v17.4s, v17.4s, v23.4s + add v20.4s, v20.4s, v28.4s + add v23.4s, v30.4s, v5.4s + add v22.4s, v22.4s, v0.4s + add v17.4s, v17.4s, v2.4s + add v20.4s, v20.4s, v1.4s + eor v25.16b, v25.16b, v23.16b + eor v16.16b, v16.16b, v22.16b + eor v6.16b, v6.16b, v17.16b + eor v4.16b, v4.16b, v20.16b + tbl v25.16b, { v25.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + tbl v6.16b, { v6.16b }, v18.16b + tbl v4.16b, { v4.16b }, v18.16b + add v19.4s, v19.4s, v25.4s + add v21.4s, v21.4s, v16.4s + add v26.4s, v26.4s, v6.4s + add v7.4s, v7.4s, v4.4s + eor v5.16b, v19.16b, v5.16b + eor v0.16b, v21.16b, v0.16b + eor v2.16b, v26.16b, v2.16b + eor v1.16b, v7.16b, v1.16b + ushr v28.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + ushr v30.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v31.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ushr v8.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + orr v5.16b, v5.16b, v28.16b + ldr q28, [sp, #176] + orr v0.16b, v0.16b, v30.16b + orr v2.16b, v2.16b, v31.16b + orr v1.16b, v1.16b, v8.16b + add v23.4s, v23.4s, v28.4s + add v22.4s, v22.4s, v11.4s + add v17.4s, v17.4s, v15.4s + add v20.4s, v20.4s, v3.4s + ldr q3, [sp, #272] + add v23.4s, v23.4s, v0.4s + add v22.4s, v22.4s, v2.4s + add v17.4s, v17.4s, v1.4s + add v20.4s, v20.4s, v5.4s + eor v4.16b, v4.16b, v23.16b + eor v25.16b, v25.16b, v22.16b + eor v16.16b, v16.16b, v17.16b + eor v6.16b, v6.16b, v20.16b + tbl v4.16b, { v4.16b }, v27.16b + tbl v25.16b, { v25.16b }, v27.16b + tbl v16.16b, { v16.16b }, v27.16b + tbl v6.16b, { v6.16b }, v27.16b + add v26.4s, v26.4s, v4.4s + add v7.4s, v7.4s, v25.4s + add v19.4s, v19.4s, v16.4s + add v21.4s, v21.4s, v6.4s + eor v0.16b, v26.16b, v0.16b + eor v2.16b, v7.16b, v2.16b + eor v1.16b, v19.16b, v1.16b + eor v5.16b, v21.16b, v5.16b + add v3.4s, v22.4s, v3.4s + ldr q22, [sp, #160] + ushr v28.4s, v0.4s, #12 + shl v0.4s, v0.4s, #20 + ushr v29.4s, v2.4s, #12 + shl v2.4s, v2.4s, #20 + ushr v30.4s, v1.4s, #12 + shl v1.4s, v1.4s, #20 + ushr v31.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + add v17.4s, v17.4s, v22.4s + ldr q22, [sp, #240] + orr v0.16b, v0.16b, v28.16b + prfm pldl1keep, [x23, #256] + orr v2.16b, v2.16b, v29.16b + prfm pldl1keep, [x24, #256] + orr v1.16b, v1.16b, v30.16b + prfm pldl1keep, [x22, #256] + orr v5.16b, v5.16b, v31.16b + prfm pldl1keep, [x25, #256] + add v23.4s, v23.4s, v24.4s + add v20.4s, v20.4s, v22.4s + add v3.4s, v3.4s, v2.4s + add v17.4s, v17.4s, v1.4s + add v22.4s, v23.4s, v0.4s + add v20.4s, v20.4s, v5.4s + eor v23.16b, v25.16b, v3.16b + eor v16.16b, v16.16b, v17.16b + eor v4.16b, v4.16b, v22.16b + eor v6.16b, v6.16b, v20.16b + tbl v23.16b, { v23.16b }, v18.16b + tbl v16.16b, { v16.16b }, v18.16b + tbl v4.16b, { v4.16b }, v18.16b + tbl v6.16b, { v6.16b }, v18.16b + add v7.4s, v7.4s, v23.4s + add v19.4s, v19.4s, v16.4s + add v18.4s, v26.4s, v4.4s + add v21.4s, v21.4s, v6.4s + eor v2.16b, v7.16b, v2.16b + eor v1.16b, v19.16b, v1.16b + eor v0.16b, v18.16b, v0.16b + eor v5.16b, v21.16b, v5.16b + ushr v25.4s, v2.4s, #7 + shl v2.4s, v2.4s, #25 + ushr v24.4s, v0.4s, #7 + shl v0.4s, v0.4s, #25 + ushr v26.4s, v1.4s, #7 + shl v1.4s, v1.4s, #25 + ushr v27.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + orr v0.16b, v0.16b, v24.16b + orr v2.16b, v2.16b, v25.16b + orr v1.16b, v1.16b, v26.16b + orr v5.16b, v5.16b, v27.16b + movi v13.4s, #64 + eor v29.16b, v19.16b, v22.16b + eor v8.16b, v21.16b, v3.16b + eor v30.16b, v17.16b, v18.16b + eor v31.16b, v20.16b, v7.16b + eor v24.16b, v5.16b, v23.16b + eor v18.16b, v0.16b, v16.16b + eor v25.16b, v2.16b, v6.16b + eor v26.16b, v1.16b, v4.16b + cbnz x21, .LBB3_5 + b .LBB3_2 +.LBB3_6: + cbz x1, .LBB3_14 + adrp x12, .LCPI3_3 + ldr q0, [x11, :lo12:.LCPI3_1] + orr w11, w7, w6 + ldr q2, [x10, :lo12:.LCPI3_2] + ldr q1, [x12, :lo12:.LCPI3_3] + and x12, x5, #0x1 +.LBB3_8: + movi v3.4s, #64 + lsr x13, x4, #32 + ldp q5, q4, [x3] + mov x15, x2 + mov w14, w11 + mov v3.s[0], w4 + ldr x10, [x0] + mov v3.s[1], w13 + b .LBB3_11 +.LBB3_9: + orr w14, w14, w9 +.LBB3_10: + ldp q6, q7, [x10] + mov v16.16b, v3.16b + and w14, w14, #0xff + add v5.4s, v5.4s, v4.4s + mov x15, x13 + mov v16.s[3], w14 + add x14, x10, #32 + uzp1 v17.4s, v6.4s, v7.4s + add x10, x10, #64 + add v5.4s, v5.4s, v17.4s + eor v16.16b, v5.16b, v16.16b + tbl v16.16b, { v16.16b }, v0.16b + add v18.4s, v16.4s, v1.4s + eor v19.16b, v18.16b, v4.16b + uzp2 v4.4s, v6.4s, v7.4s + ushr v6.4s, v19.4s, #12 + shl v7.4s, v19.4s, #20 + ld2 { v19.4s, v20.4s }, [x14] + add v5.4s, v5.4s, v4.4s + mov w14, w6 + orr v6.16b, v7.16b, v6.16b + add v5.4s, v5.4s, v6.4s + eor v7.16b, v16.16b, v5.16b + add v5.4s, v5.4s, v19.4s + tbl v7.16b, { v7.16b }, v2.16b + ext v5.16b, v5.16b, v5.16b, #12 + add v16.4s, v18.4s, v7.4s + ext v7.16b, v7.16b, v7.16b, #8 + eor v6.16b, v6.16b, v16.16b + ext v16.16b, v16.16b, v16.16b, #4 + ushr v18.4s, v6.4s, #7 + shl v6.4s, v6.4s, #25 + orr v6.16b, v6.16b, v18.16b + ext v18.16b, v20.16b, v20.16b, #12 + add v5.4s, v5.4s, v6.4s + eor v7.16b, v5.16b, v7.16b + add v5.4s, v5.4s, v18.4s + tbl v7.16b, { v7.16b }, v0.16b + add v16.4s, v16.4s, v7.4s + eor v6.16b, v6.16b, v16.16b + ushr v21.4s, v6.4s, #12 + shl v6.4s, v6.4s, #20 + orr v6.16b, v6.16b, v21.16b + uzp1 v21.4s, v17.4s, v17.4s + add v5.4s, v5.4s, v6.4s + ext v21.16b, v21.16b, v17.16b, #8 + eor v7.16b, v7.16b, v5.16b + uzp2 v21.4s, v21.4s, v4.4s + tbl v7.16b, { v7.16b }, v2.16b + add v5.4s, v5.4s, v21.4s + add v16.4s, v16.4s, v7.4s + ext v5.16b, v5.16b, v5.16b, #4 + ext v7.16b, v7.16b, v7.16b, #8 + eor v6.16b, v6.16b, v16.16b + ushr v22.4s, v6.4s, #7 + shl v6.4s, v6.4s, #25 + orr v6.16b, v6.16b, v22.16b + add v22.4s, v5.4s, v6.4s + eor v5.16b, v22.16b, v7.16b + ext v7.16b, v16.16b, v16.16b, #12 + tbl v16.16b, { v5.16b }, v0.16b + ext v5.16b, v17.16b, v17.16b, #12 + add v7.4s, v7.4s, v16.4s + ext v5.16b, v17.16b, v5.16b, #12 + ext v17.16b, v19.16b, v19.16b, #12 + mov v19.16b, v18.16b + eor v6.16b, v6.16b, v7.16b + rev64 v5.4s, v5.4s + mov v19.s[1], v17.s[2] + ushr v20.4s, v6.4s, #12 + shl v6.4s, v6.4s, #20 + trn2 v5.4s, v5.4s, v19.4s + orr v6.16b, v6.16b, v20.16b + zip1 v20.2d, v18.2d, v4.2d + zip2 v4.4s, v4.4s, v18.4s + add v19.4s, v6.4s, v5.4s + mov v20.s[3], v17.s[3] + add v19.4s, v19.4s, v22.4s + ext v22.16b, v20.16b, v20.16b, #12 + eor v16.16b, v16.16b, v19.16b + ext v19.16b, v19.16b, v19.16b, #12 + tbl v16.16b, { v16.16b }, v2.16b + add v7.4s, v7.4s, v16.4s + ext v16.16b, v16.16b, v16.16b, #8 + eor v6.16b, v6.16b, v7.16b + ext v7.16b, v7.16b, v7.16b, #4 + ushr v23.4s, v6.4s, #7 + shl v24.4s, v6.4s, #25 + uzp1 v6.4s, v20.4s, v22.4s + orr v20.16b, v24.16b, v23.16b + add v22.4s, v20.4s, v6.4s + add v19.4s, v22.4s, v19.4s + eor v16.16b, v19.16b, v16.16b + tbl v16.16b, { v16.16b }, v0.16b + add v7.4s, v7.4s, v16.4s + eor v18.16b, v20.16b, v7.16b + zip1 v20.4s, v4.4s, v17.4s + zip1 v4.4s, v17.4s, v4.4s + ushr v17.4s, v18.4s, #12 + shl v18.4s, v18.4s, #20 + ext v20.16b, v4.16b, v20.16b, #8 + orr v4.16b, v18.16b, v17.16b + ext v18.16b, v21.16b, v21.16b, #4 + add v17.4s, v4.4s, v20.4s + add v17.4s, v17.4s, v19.4s + uzp1 v19.4s, v18.4s, v18.4s + eor v16.16b, v16.16b, v17.16b + ext v19.16b, v19.16b, v18.16b, #8 + tbl v16.16b, { v16.16b }, v2.16b + uzp2 v19.4s, v19.4s, v5.4s + add v7.4s, v7.4s, v16.4s + add v17.4s, v17.4s, v19.4s + ext v16.16b, v16.16b, v16.16b, #8 + eor v4.16b, v4.16b, v7.16b + ext v17.16b, v17.16b, v17.16b, #4 + ext v7.16b, v7.16b, v7.16b, #12 + ushr v21.4s, v4.4s, #7 + shl v4.4s, v4.4s, #25 + orr v4.16b, v4.16b, v21.16b + ext v21.16b, v18.16b, v18.16b, #12 + add v17.4s, v17.4s, v4.4s + ext v18.16b, v18.16b, v21.16b, #12 + mov v21.16b, v20.16b + eor v16.16b, v17.16b, v16.16b + rev64 v18.4s, v18.4s + mov v21.s[1], v6.s[2] + tbl v16.16b, { v16.16b }, v0.16b + add v7.4s, v7.4s, v16.4s + eor v4.16b, v4.16b, v7.16b + ushr v22.4s, v4.4s, #12 + shl v23.4s, v4.4s, #20 + trn2 v4.4s, v18.4s, v21.4s + orr v18.16b, v23.16b, v22.16b + add v21.4s, v18.4s, v4.4s + add v17.4s, v21.4s, v17.4s + zip1 v21.2d, v20.2d, v5.2d + zip2 v5.4s, v5.4s, v20.4s + eor v16.16b, v16.16b, v17.16b + mov v21.s[3], v6.s[3] + ext v17.16b, v17.16b, v17.16b, #12 + zip1 v20.4s, v5.4s, v6.4s + tbl v16.16b, { v16.16b }, v2.16b + zip1 v5.4s, v6.4s, v5.4s + add v22.4s, v7.4s, v16.4s + ext v16.16b, v16.16b, v16.16b, #8 + ext v20.16b, v5.16b, v20.16b, #8 + eor v7.16b, v18.16b, v22.16b + ext v18.16b, v21.16b, v21.16b, #12 + ushr v23.4s, v7.4s, #7 + shl v24.4s, v7.4s, #25 + uzp1 v7.4s, v21.4s, v18.4s + orr v18.16b, v24.16b, v23.16b + add v21.4s, v18.4s, v7.4s + add v17.4s, v21.4s, v17.4s + ext v21.16b, v22.16b, v22.16b, #4 + eor v16.16b, v17.16b, v16.16b + tbl v16.16b, { v16.16b }, v0.16b + add v21.4s, v21.4s, v16.4s + eor v18.16b, v18.16b, v21.16b + ushr v6.4s, v18.4s, #12 + shl v18.4s, v18.4s, #20 + orr v5.16b, v18.16b, v6.16b + add v6.4s, v5.4s, v20.4s + add v6.4s, v6.4s, v17.4s + ext v17.16b, v19.16b, v19.16b, #4 + eor v16.16b, v16.16b, v6.16b + uzp1 v18.4s, v17.4s, v17.4s + tbl v16.16b, { v16.16b }, v2.16b + ext v18.16b, v18.16b, v17.16b, #8 + add v19.4s, v21.4s, v16.4s + uzp2 v18.4s, v18.4s, v4.4s + ext v16.16b, v16.16b, v16.16b, #8 + eor v5.16b, v5.16b, v19.16b + add v6.4s, v6.4s, v18.4s + ext v19.16b, v19.16b, v19.16b, #12 + ushr v21.4s, v5.4s, #7 + shl v5.4s, v5.4s, #25 + ext v6.16b, v6.16b, v6.16b, #4 + orr v5.16b, v5.16b, v21.16b + ext v21.16b, v17.16b, v17.16b, #12 + add v6.4s, v6.4s, v5.4s + ext v17.16b, v17.16b, v21.16b, #12 + mov v21.16b, v20.16b + eor v16.16b, v6.16b, v16.16b + rev64 v17.4s, v17.4s + mov v21.s[1], v7.s[2] + tbl v16.16b, { v16.16b }, v0.16b + add v19.4s, v19.4s, v16.4s + eor v5.16b, v5.16b, v19.16b + ushr v22.4s, v5.4s, #12 + shl v23.4s, v5.4s, #20 + trn2 v5.4s, v17.4s, v21.4s + orr v17.16b, v23.16b, v22.16b + add v21.4s, v17.4s, v5.4s + add v6.4s, v21.4s, v6.4s + eor v16.16b, v16.16b, v6.16b + ext v6.16b, v6.16b, v6.16b, #12 + tbl v21.16b, { v16.16b }, v2.16b + zip1 v16.2d, v20.2d, v4.2d + zip2 v4.4s, v4.4s, v20.4s + add v19.4s, v19.4s, v21.4s + mov v16.s[3], v7.s[3] + ext v21.16b, v21.16b, v21.16b, #8 + zip1 v20.4s, v4.4s, v7.4s + eor v17.16b, v17.16b, v19.16b + ext v22.16b, v16.16b, v16.16b, #12 + ext v19.16b, v19.16b, v19.16b, #4 + zip1 v4.4s, v7.4s, v4.4s + ushr v23.4s, v17.4s, #7 + shl v17.4s, v17.4s, #25 + uzp1 v16.4s, v16.4s, v22.4s + ext v4.16b, v4.16b, v20.16b, #8 + orr v17.16b, v17.16b, v23.16b + add v22.4s, v17.4s, v16.4s + add v6.4s, v22.4s, v6.4s + eor v21.16b, v6.16b, v21.16b + tbl v21.16b, { v21.16b }, v0.16b + add v19.4s, v19.4s, v21.4s + eor v17.16b, v17.16b, v19.16b + ushr v7.4s, v17.4s, #12 + shl v17.4s, v17.4s, #20 + orr v7.16b, v17.16b, v7.16b + add v17.4s, v7.4s, v4.4s + add v6.4s, v17.4s, v6.4s + ext v17.16b, v18.16b, v18.16b, #4 + eor v18.16b, v21.16b, v6.16b + uzp1 v20.4s, v17.4s, v17.4s + tbl v18.16b, { v18.16b }, v2.16b + ext v20.16b, v20.16b, v17.16b, #8 + add v19.4s, v19.4s, v18.4s + uzp2 v20.4s, v20.4s, v5.4s + ext v18.16b, v18.16b, v18.16b, #8 + eor v7.16b, v7.16b, v19.16b + add v6.4s, v6.4s, v20.4s + ushr v21.4s, v7.4s, #7 + shl v7.4s, v7.4s, #25 + ext v6.16b, v6.16b, v6.16b, #4 + orr v7.16b, v7.16b, v21.16b + add v21.4s, v6.4s, v7.4s + eor v6.16b, v21.16b, v18.16b + ext v18.16b, v19.16b, v19.16b, #12 + tbl v19.16b, { v6.16b }, v0.16b + ext v6.16b, v17.16b, v17.16b, #12 + add v18.4s, v18.4s, v19.4s + ext v6.16b, v17.16b, v6.16b, #12 + mov v17.16b, v4.16b + eor v7.16b, v7.16b, v18.16b + rev64 v6.4s, v6.4s + mov v17.s[1], v16.s[2] + ushr v22.4s, v7.4s, #12 + shl v7.4s, v7.4s, #20 + trn2 v6.4s, v6.4s, v17.4s + orr v7.16b, v7.16b, v22.16b + add v17.4s, v7.4s, v6.4s + add v17.4s, v17.4s, v21.4s + zip1 v21.2d, v4.2d, v5.2d + zip2 v4.4s, v5.4s, v4.4s + eor v19.16b, v19.16b, v17.16b + mov v21.s[3], v16.s[3] + ext v17.16b, v17.16b, v17.16b, #12 + tbl v19.16b, { v19.16b }, v2.16b + ext v22.16b, v21.16b, v21.16b, #12 + add v18.4s, v18.4s, v19.4s + ext v19.16b, v19.16b, v19.16b, #8 + eor v7.16b, v7.16b, v18.16b + ext v18.16b, v18.16b, v18.16b, #4 + ushr v23.4s, v7.4s, #7 + shl v24.4s, v7.4s, #25 + uzp1 v7.4s, v21.4s, v22.4s + orr v21.16b, v24.16b, v23.16b + add v22.4s, v21.4s, v7.4s + add v17.4s, v22.4s, v17.4s + eor v19.16b, v17.16b, v19.16b + tbl v19.16b, { v19.16b }, v0.16b + add v18.4s, v18.4s, v19.4s + eor v5.16b, v21.16b, v18.16b + zip1 v21.4s, v4.4s, v16.4s + zip1 v4.4s, v16.4s, v4.4s + ushr v16.4s, v5.4s, #12 + shl v5.4s, v5.4s, #20 + ext v21.16b, v4.16b, v21.16b, #8 + orr v4.16b, v5.16b, v16.16b + ext v16.16b, v20.16b, v20.16b, #4 + mov v23.16b, v21.16b + add v5.4s, v4.4s, v21.4s + mov v23.s[1], v7.s[2] + add v5.4s, v5.4s, v17.4s + eor v17.16b, v19.16b, v5.16b + uzp1 v19.4s, v16.4s, v16.4s + tbl v17.16b, { v17.16b }, v2.16b + ext v19.16b, v19.16b, v16.16b, #8 + add v18.4s, v18.4s, v17.4s + uzp2 v19.4s, v19.4s, v6.4s + eor v4.16b, v4.16b, v18.16b + add v5.4s, v5.4s, v19.4s + ext v19.16b, v19.16b, v19.16b, #4 + ushr v20.4s, v4.4s, #7 + shl v4.4s, v4.4s, #25 + ext v5.16b, v5.16b, v5.16b, #4 + orr v20.16b, v4.16b, v20.16b + ext v4.16b, v17.16b, v17.16b, #8 + add v17.4s, v5.4s, v20.4s + ext v5.16b, v18.16b, v18.16b, #12 + eor v4.16b, v17.16b, v4.16b + tbl v18.16b, { v4.16b }, v0.16b + ext v4.16b, v16.16b, v16.16b, #12 + add v22.4s, v5.4s, v18.4s + ext v4.16b, v16.16b, v4.16b, #12 + eor v5.16b, v20.16b, v22.16b + rev64 v16.4s, v4.4s + ushr v20.4s, v5.4s, #12 + shl v24.4s, v5.4s, #20 + trn2 v5.4s, v16.4s, v23.4s + orr v16.16b, v24.16b, v20.16b + add v20.4s, v16.4s, v5.4s + add v17.4s, v20.4s, v17.4s + zip1 v20.2d, v21.2d, v6.2d + zip2 v6.4s, v6.4s, v21.4s + eor v18.16b, v18.16b, v17.16b + mov v20.s[3], v7.s[3] + ext v17.16b, v17.16b, v17.16b, #12 + zip1 v21.4s, v6.4s, v7.4s + tbl v18.16b, { v18.16b }, v2.16b + ext v24.16b, v20.16b, v20.16b, #12 + zip1 v6.4s, v7.4s, v6.4s + add v22.4s, v22.4s, v18.4s + ext v18.16b, v18.16b, v18.16b, #8 + ext v6.16b, v6.16b, v21.16b, #8 + eor v16.16b, v16.16b, v22.16b + ext v22.16b, v22.16b, v22.16b, #4 + zip1 v5.2d, v6.2d, v5.2d + zip2 v4.4s, v4.4s, v6.4s + ushr v25.4s, v16.4s, #7 + shl v26.4s, v16.4s, #25 + uzp1 v16.4s, v20.4s, v24.4s + orr v20.16b, v26.16b, v25.16b + mov v5.s[3], v16.s[3] + add v24.4s, v20.4s, v16.4s + add v17.4s, v24.4s, v17.4s + eor v18.16b, v17.16b, v18.16b + tbl v18.16b, { v18.16b }, v0.16b + add v22.4s, v22.4s, v18.4s + eor v20.16b, v20.16b, v22.16b + ushr v7.4s, v20.4s, #12 + shl v20.4s, v20.4s, #20 + orr v7.16b, v20.16b, v7.16b + add v20.4s, v7.4s, v6.4s + add v17.4s, v20.4s, v17.4s + ext v20.16b, v19.16b, v19.16b, #8 + eor v18.16b, v18.16b, v17.16b + ext v17.16b, v17.16b, v17.16b, #4 + tbl v18.16b, { v18.16b }, v2.16b + add v21.4s, v22.4s, v18.4s + uzp2 v22.4s, v20.4s, v23.4s + ext v18.16b, v18.16b, v18.16b, #8 + eor v7.16b, v7.16b, v21.16b + ext v20.16b, v22.16b, v20.16b, #4 + ushr v22.4s, v7.4s, #7 + shl v7.4s, v7.4s, #25 + add v17.4s, v17.4s, v20.4s + ext v20.16b, v21.16b, v21.16b, #12 + ext v21.16b, v19.16b, v19.16b, #12 + orr v7.16b, v7.16b, v22.16b + ext v19.16b, v19.16b, v21.16b, #12 + add v17.4s, v17.4s, v7.4s + mov v21.16b, v6.16b + rev64 v19.4s, v19.4s + eor v18.16b, v17.16b, v18.16b + mov v21.s[1], v16.s[2] + tbl v18.16b, { v18.16b }, v0.16b + trn2 v19.4s, v19.4s, v21.4s + add v20.4s, v20.4s, v18.4s + eor v7.16b, v7.16b, v20.16b + ushr v22.4s, v7.4s, #12 + shl v7.4s, v7.4s, #20 + orr v7.16b, v7.16b, v22.16b + add v19.4s, v7.4s, v19.4s + add v17.4s, v19.4s, v17.4s + eor v18.16b, v18.16b, v17.16b + ext v17.16b, v17.16b, v17.16b, #12 + tbl v18.16b, { v18.16b }, v2.16b + add v19.4s, v20.4s, v18.4s + ext v20.16b, v5.16b, v5.16b, #12 + ext v18.16b, v18.16b, v18.16b, #8 + eor v7.16b, v7.16b, v19.16b + uzp1 v5.4s, v5.4s, v20.4s + ushr v21.4s, v7.4s, #7 + shl v7.4s, v7.4s, #25 + orr v7.16b, v7.16b, v21.16b + add v5.4s, v7.4s, v5.4s + add v5.4s, v5.4s, v17.4s + eor v17.16b, v5.16b, v18.16b + ext v18.16b, v19.16b, v19.16b, #4 + tbl v17.16b, { v17.16b }, v0.16b + add v18.4s, v18.4s, v17.4s + eor v6.16b, v7.16b, v18.16b + zip1 v7.4s, v4.4s, v16.4s + zip1 v4.4s, v16.4s, v4.4s + ushr v16.4s, v6.4s, #12 + shl v6.4s, v6.4s, #20 + ext v4.16b, v4.16b, v7.16b, #8 + orr v6.16b, v6.16b, v16.16b + add v4.4s, v6.4s, v4.4s + add v4.4s, v4.4s, v5.4s + eor v5.16b, v17.16b, v4.16b + ext v4.16b, v4.16b, v4.16b, #4 + tbl v5.16b, { v5.16b }, v2.16b + add v7.4s, v18.4s, v5.4s + eor v6.16b, v6.16b, v7.16b + ext v7.16b, v7.16b, v7.16b, #12 + ushr v16.4s, v6.4s, #7 + shl v6.4s, v6.4s, #25 + orr v6.16b, v6.16b, v16.16b + ext v16.16b, v5.16b, v5.16b, #8 + eor v5.16b, v4.16b, v7.16b + eor v4.16b, v6.16b, v16.16b +.LBB3_11: + subs x13, x15, #1 + b.eq .LBB3_9 + cbnz x15, .LBB3_10 + add x4, x4, x12 + add x0, x0, #8 + subs x1, x1, #1 + stp q5, q4, [x8], #32 + b.ne .LBB3_8 +.LBB3_14: + add sp, sp, #368 + ldp x20, x19, [sp, #128] + ldp x22, x21, [sp, #112] + ldp x24, x23, [sp, #96] + ldp x26, x25, [sp, #80] + ldp x29, x27, [sp, #64] ldp d9, d8, [sp, #48] ldp d11, d10, [sp, #32] ldp d13, d12, [sp, #16] - ldp d15, d14, [sp], #160 + ldp d15, d14, [sp], #144 ret -.Lfunc_end2: - .size zfs_blake3_hash_many_sse41, .Lfunc_end2-zfs_blake3_hash_many_sse41 +.Lfunc_end3: + .size zfs_blake3_hash_many_sse41, .Lfunc_end3-zfs_blake3_hash_many_sse41 .cfi_endproc .section ".note.GNU-stack","",@progbits -#endif +#endif \ No newline at end of file From ae0d0f0e047edc0da20f9dcf28d161e31a259751 Mon Sep 17 00:00:00 2001 From: Val Packett Date: Thu, 27 Apr 2023 13:49:03 -0300 Subject: [PATCH 53/59] PAM: support the authentication facility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the pam_sm_authenticate method, using the noop argument of lzc_load_key to do a passphrase check without actually loading the key. This allows using ZFS as the source of truth for user passwords, without storing any password hashes in /etc or using other PAM modules. Reviewed-by: Brian Behlendorf Reviewed-by: Felix Dörre Signed-off-by: Val Packett Closes #14789 --- contrib/pam_zfs_key/pam_zfs_key.c | 63 ++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/contrib/pam_zfs_key/pam_zfs_key.c b/contrib/pam_zfs_key/pam_zfs_key.c index 6ba5b5fba75f..27c7d63781c5 100644 --- a/contrib/pam_zfs_key/pam_zfs_key.c +++ b/contrib/pam_zfs_key/pam_zfs_key.c @@ -371,7 +371,7 @@ change_key(pam_handle_t *pamh, const char *ds_name, static int decrypt_mount(pam_handle_t *pamh, const char *ds_name, - const char *passphrase) + const char *passphrase, boolean_t noop) { zfs_handle_t *ds = zfs_open(g_zfs, ds_name, ZFS_TYPE_FILESYSTEM); if (ds == NULL) { @@ -383,7 +383,7 @@ decrypt_mount(pam_handle_t *pamh, const char *ds_name, zfs_close(ds); return (-1); } - int ret = lzc_load_key(ds_name, B_FALSE, (uint8_t *)key->value, + int ret = lzc_load_key(ds_name, noop, (uint8_t *)key->value, WRAPPING_KEY_LEN); pw_free(key); if (ret) { @@ -391,12 +391,16 @@ decrypt_mount(pam_handle_t *pamh, const char *ds_name, zfs_close(ds); return (-1); } + if (noop) { + goto out; + } ret = zfs_mount(ds, NULL, 0); if (ret) { pam_syslog(pamh, LOG_ERR, "mount failed: %d", ret); zfs_close(ds); return (-1); } +out: zfs_close(ds); return (0); } @@ -443,13 +447,13 @@ zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config, config->homes_prefix = strdup("rpool/home"); if (config->homes_prefix == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); - return (-1); + return (PAM_SERVICE_ERR); } config->runstatedir = strdup(RUNSTATEDIR "/pam_zfs_key"); if (config->runstatedir == NULL) { pam_syslog(pamh, LOG_ERR, "strdup failure"); free(config->homes_prefix); - return (-1); + return (PAM_SERVICE_ERR); } const char *name; if (pam_get_user(pamh, &name, NULL) != PAM_SUCCESS) { @@ -457,13 +461,13 @@ zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config, "couldn't get username from PAM stack"); free(config->runstatedir); free(config->homes_prefix); - return (-1); + return (PAM_SERVICE_ERR); } struct passwd *entry = getpwnam(name); if (!entry) { free(config->runstatedir); free(config->homes_prefix); - return (-1); + return (PAM_USER_UNKNOWN); } config->uid = entry->pw_uid; config->username = name; @@ -484,7 +488,7 @@ zfs_key_config_load(pam_handle_t *pamh, zfs_key_config_t *config, config->homedir = strdup(entry->pw_dir); } } - return (0); + return (PAM_SUCCESS); } static void @@ -644,12 +648,43 @@ PAM_EXTERN int pam_sm_authenticate(pam_handle_t *pamh, int flags, int argc, const char **argv) { - (void) flags, (void) argc, (void) argv; + (void) flags; - if (pw_fetch_lazy(pamh) == NULL) { - return (PAM_AUTH_ERR); + if (geteuid() != 0) { + pam_syslog(pamh, LOG_ERR, + "Cannot zfs_mount when not being root."); + return (PAM_SERVICE_ERR); + } + zfs_key_config_t config; + int config_err = zfs_key_config_load(pamh, &config, argc, argv); + if (config_err != PAM_SUCCESS) { + return (config_err); } + const pw_password_t *token = pw_fetch_lazy(pamh); + if (token == NULL) { + zfs_key_config_free(&config); + return (PAM_AUTH_ERR); + } + if (pam_zfs_init(pamh) != 0) { + zfs_key_config_free(&config); + return (PAM_SERVICE_ERR); + } + char *dataset = zfs_key_config_get_dataset(&config); + if (!dataset) { + pam_zfs_free(); + zfs_key_config_free(&config); + return (PAM_SERVICE_ERR); + } + if (decrypt_mount(pamh, dataset, token->value, B_TRUE) == -1) { + free(dataset); + pam_zfs_free(); + zfs_key_config_free(&config); + return (PAM_AUTH_ERR); + } + free(dataset); + pam_zfs_free(); + zfs_key_config_free(&config); return (PAM_SUCCESS); } @@ -673,7 +708,7 @@ pam_sm_chauthtok(pam_handle_t *pamh, int flags, return (PAM_PERM_DENIED); } zfs_key_config_t config; - if (zfs_key_config_load(pamh, &config, argc, argv) == -1) { + if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SERVICE_ERR); } if (config.uid < 1000) { @@ -754,7 +789,7 @@ pam_sm_open_session(pam_handle_t *pamh, int flags, return (PAM_SUCCESS); } zfs_key_config_t config; - if (zfs_key_config_load(pamh, &config, argc, argv) != 0) { + if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SESSION_ERR); } @@ -784,7 +819,7 @@ pam_sm_open_session(pam_handle_t *pamh, int flags, zfs_key_config_free(&config); return (PAM_SERVICE_ERR); } - if (decrypt_mount(pamh, dataset, token->value) == -1) { + if (decrypt_mount(pamh, dataset, token->value, B_FALSE) == -1) { free(dataset); pam_zfs_free(); zfs_key_config_free(&config); @@ -813,7 +848,7 @@ pam_sm_close_session(pam_handle_t *pamh, int flags, return (PAM_SUCCESS); } zfs_key_config_t config; - if (zfs_key_config_load(pamh, &config, argc, argv) != 0) { + if (zfs_key_config_load(pamh, &config, argc, argv) != PAM_SUCCESS) { return (PAM_SESSION_ERR); } if (config.uid < 1000) { From 2fd1c30423620a5b198ac1a5aa2cff8e1e57b7f3 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Thu, 27 Apr 2023 15:32:58 -0400 Subject: [PATCH 54/59] Mark TX_COMMIT transaction with TXG_NOTHROTTLE. TX_COMMIT has no on-disk representation and does not produce any more dirty data. It should not wait for anything, and even just skipping the checks if not waiting gives improvement noticeable in profiler. Reviewed-by: Brian Behlendorf Reviewed-by: Prakash Surya Signed-off-by: Alexander Motin Sponsored by: iXsystems, Inc. Closes #14798 --- module/zfs/zil.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/module/zfs/zil.c b/module/zfs/zil.c index d1631c2ac9db..ec9da706a806 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -3155,7 +3155,14 @@ static void zil_commit_itx_assign(zilog_t *zilog, zil_commit_waiter_t *zcw) { dmu_tx_t *tx = dmu_tx_create(zilog->zl_os); - VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); + + /* + * Since we are not going to create any new dirty data, and we + * can even help with clearing the existing dirty data, we + * should not be subject to the dirty data based delays. We + * use TXG_NOTHROTTLE to bypass the delay mechanism. + */ + VERIFY0(dmu_tx_assign(tx, TXG_WAIT | TXG_NOTHROTTLE)); itx_t *itx = zil_itx_create(TX_COMMIT, sizeof (lr_t)); itx->itx_sync = B_TRUE; From 5a83f761c7c7445dda39d3fd3c5aa2a7bcb353f1 Mon Sep 17 00:00:00 2001 From: Justin Hibbits Date: Thu, 27 Apr 2023 15:49:21 -0400 Subject: [PATCH 55/59] powerpc64: Support ELFv2 asm on Big Endian FreeBSD/powerpc64 is all ELFv2 since FreeBSD 13, even big endian. The existing sha256 and sha512 asm code assumes that BE is all ELFv1, and LE is ELFv2. Minor changes to add ELFv2 in the BE side gets this working correctly on FreeBSD with latest OpenZFS import. Reviewed-by: Tino Reichardt Reviewed-by: Brian Behlendorf Signed-off-by: Justin Hibbits Closes #14779 --- include/os/freebsd/spl/sys/simd_powerpc.h | 2 +- lib/libspl/include/sys/simd.h | 2 +- module/icp/asm-ppc64/sha2/sha256-p8.S | 15 +++++++++++++++ module/icp/asm-ppc64/sha2/sha256-ppc.S | 15 +++++++++++++++ module/icp/asm-ppc64/sha2/sha512-p8.S | 16 ++++++++++++++++ module/icp/asm-ppc64/sha2/sha512-ppc.S | 15 +++++++++++++++ 6 files changed, 63 insertions(+), 2 deletions(-) diff --git a/include/os/freebsd/spl/sys/simd_powerpc.h b/include/os/freebsd/spl/sys/simd_powerpc.h index edaab81d15fc..cf3c712c6af2 100644 --- a/include/os/freebsd/spl/sys/simd_powerpc.h +++ b/include/os/freebsd/spl/sys/simd_powerpc.h @@ -49,7 +49,7 @@ #include #include -#define kfpu_allowed() 1 +#define kfpu_allowed() 0 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) diff --git a/lib/libspl/include/sys/simd.h b/lib/libspl/include/sys/simd.h index a106967d0725..41f9df506468 100644 --- a/lib/libspl/include/sys/simd.h +++ b/lib/libspl/include/sys/simd.h @@ -551,7 +551,7 @@ zfs_sha512_available(void) #elif defined(__powerpc__) -#define kfpu_allowed() 1 +#define kfpu_allowed() 0 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) diff --git a/module/icp/asm-ppc64/sha2/sha256-p8.S b/module/icp/asm-ppc64/sha2/sha256-p8.S index 6bbfe23b6e15..dc3c4cea669c 100644 --- a/module/icp/asm-ppc64/sha2/sha256-p8.S +++ b/module/icp/asm-ppc64/sha2/sha256-p8.S @@ -21,6 +21,7 @@ #if (defined(__PPC64__) && defined(__BIG_ENDIAN__)) +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .text .globl zfs_sha256_power8 @@ -33,6 +34,16 @@ zfs_sha256_power8: .previous .align 6 .zfs_sha256_power8: +#else +.abiversion 2 +.text + +.globl zfs_sha256_power8 +.type zfs_sha256_power8,@function +.align 6 +zfs_sha256_power8: +.localentry zfs_sha256_power8,0 +#endif stdu 1,-384(1) mflr 8 li 10,207 @@ -677,8 +688,12 @@ zfs_sha256_power8: .long 0 .byte 0,12,4,1,0x80,6,3,0 .long 0 +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .size .zfs_sha256_power8,.-.zfs_sha256_power8 .size zfs_sha256_power8,.-.zfs_sha256_power8 +#else +.size zfs_sha256_power8,.-zfs_sha256_power8 +#endif .align 6 .LPICmeup: mflr 0 diff --git a/module/icp/asm-ppc64/sha2/sha256-ppc.S b/module/icp/asm-ppc64/sha2/sha256-ppc.S index 2219e313c9c6..d039bc36ee11 100644 --- a/module/icp/asm-ppc64/sha2/sha256-ppc.S +++ b/module/icp/asm-ppc64/sha2/sha256-ppc.S @@ -21,6 +21,7 @@ #if (defined(__PPC64__) && defined(__BIG_ENDIAN__)) +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .text .globl zfs_sha256_ppc @@ -33,6 +34,16 @@ zfs_sha256_ppc: .previous .align 6 .zfs_sha256_ppc: +#else +.abiversion 2 +.text + +.globl zfs_sha256_ppc +.type zfs_sha256_ppc,@function +.align 6 +zfs_sha256_ppc: +.localentry zfs_sha256_ppc,0 +#endif stdu 1,-320(1) mflr 0 sldi 5,5,6 @@ -1312,8 +1323,12 @@ zfs_sha256_ppc: blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .size .zfs_sha256_ppc,.-.zfs_sha256_ppc .size zfs_sha256_ppc,.-.zfs_sha256_ppc +#else +.size zfs_sha256_ppc,.-zfs_sha256_ppc +#endif .align 6 .LPICmeup: mflr 0 diff --git a/module/icp/asm-ppc64/sha2/sha512-p8.S b/module/icp/asm-ppc64/sha2/sha512-p8.S index 39a90ede3dc5..2409c53385d6 100644 --- a/module/icp/asm-ppc64/sha2/sha512-p8.S +++ b/module/icp/asm-ppc64/sha2/sha512-p8.S @@ -21,6 +21,7 @@ #if (defined(__PPC64__) && defined(__BIG_ENDIAN__)) +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .text .globl zfs_sha512_power8 @@ -33,6 +34,17 @@ zfs_sha512_power8: .previous .align 6 .zfs_sha512_power8: +#else +.abiversion 2 +.text + +.globl zfs_sha512_power8 +.type zfs_sha512_power8,@function +.align 6 +zfs_sha512_power8: +.localentry zfs_sha512_power8,0 +#endif + stdu 1,-384(1) mflr 8 li 10,207 @@ -679,8 +691,12 @@ zfs_sha512_power8: .long 0 .byte 0,12,4,1,0x80,6,3,0 .long 0 +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .size .zfs_sha512_power8,.-.zfs_sha512_power8 .size zfs_sha512_power8,.-.zfs_sha512_power8 +#else +.size zfs_sha512_power8,.-zfs_sha512_power8 +#endif .align 6 .LPICmeup: mflr 0 diff --git a/module/icp/asm-ppc64/sha2/sha512-ppc.S b/module/icp/asm-ppc64/sha2/sha512-ppc.S index 37070115c3ff..57213f68abc5 100644 --- a/module/icp/asm-ppc64/sha2/sha512-ppc.S +++ b/module/icp/asm-ppc64/sha2/sha512-ppc.S @@ -21,6 +21,7 @@ #if (defined(__PPC64__) && defined(__BIG_ENDIAN__)) +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .text .globl zfs_sha512_ppc @@ -33,6 +34,16 @@ zfs_sha512_ppc: .previous .align 6 .zfs_sha512_ppc: +#else +.abiversion 2 +.text + +.globl zfs_sha512_ppc +.type zfs_sha512_ppc,@function +.align 6 +zfs_sha512_ppc: +.localentry zfs_sha512_ppc,0 +#endif stdu 1,-384(1) mflr 0 sldi 5,5,7 @@ -1350,8 +1361,12 @@ zfs_sha512_ppc: blr .long 0 .byte 0,12,0x14,0,0,0,0,0 +#if (!defined(_CALL_ELF) || _CALL_ELF == 1) .size .zfs_sha512_ppc,.-.zfs_sha512_ppc .size zfs_sha512_ppc,.-.zfs_sha512_ppc +#else +.size zfs_sha512_ppc,.-zfs_sha512_ppc +#endif .align 6 .LPICmeup: mflr 0 From 0c93d86f01e509cdfab271fa285497cbda4e3a9f Mon Sep 17 00:00:00 2001 From: Serapheim Dimitropoulos Date: Mon, 1 May 2023 17:18:42 -0700 Subject: [PATCH 56/59] Correct ABD size for split block ZIOs Currently when layering the ABD buffer of each split block on top of an indirect vdev's ZIO ABD we don't specify the split block's ABD. This results in those ABDs being incorrectly sized by inheriting the size of their parent ABD which is larger than what each split block needs. The above behavior isn't causing any bugs currently but can lead to unexpected ABD sizes for people analyzing and/or working on the ZIO codepath. This patch fixes this behavior by properly setting the ABD size for split block ZIOs. Reviewed-by: Matthew Ahrens Reviewed-by: Igor Kozhukhov Reviewed-by: Brian Behlendorf Reviewed-by: Mark Maybee Reviewed-by: Brian Atkinson Signed-off-by: Serapheim Dimitropoulos Closes #14804 --- module/zfs/vdev_indirect.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/module/zfs/vdev_indirect.c b/module/zfs/vdev_indirect.c index 8c11a574ae86..a16ad2f4e7cf 100644 --- a/module/zfs/vdev_indirect.c +++ b/module/zfs/vdev_indirect.c @@ -1370,9 +1370,10 @@ vdev_indirect_io_start(zio_t *zio) is != NULL; is = list_next(&iv->iv_splits, is)) { zio_nowait(zio_vdev_child_io(zio, NULL, is->is_vdev, is->is_target_offset, - abd_get_offset(zio->io_abd, - is->is_split_offset), is->is_size, - zio->io_type, zio->io_priority, 0, + abd_get_offset_size(zio->io_abd, + is->is_split_offset, is->is_size), + is->is_size, zio->io_type, + zio->io_priority, 0, vdev_indirect_child_io_done, zio)); } From e2a92d726e1849f646d137510796b11c26d45cae Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Tue, 2 May 2023 02:21:27 +0200 Subject: [PATCH 57/59] blake3: fix up bogus checksums in face of cpu migration This is a temporary measure until a better fix is sorted out. Reviewed-by: Richard Yao Reviewed-by: Alexander Motin Reviewed-by: Rich Ercolani Signed-off-by: Mateusz Guzik Sponsored by: Rubicon Communications, LLC ("Netgate") Closes #14785 Closes #14808 --- module/zfs/blake3_zfs.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/module/zfs/blake3_zfs.c b/module/zfs/blake3_zfs.c index bcc595bca8f2..7783282b671a 100644 --- a/module/zfs/blake3_zfs.c +++ b/module/zfs/blake3_zfs.c @@ -50,7 +50,8 @@ abd_checksum_blake3_native(abd_t *abd, uint64_t size, const void *ctx_template, ASSERT(ctx_template != NULL); #if defined(_KERNEL) - BLAKE3_CTX *ctx = blake3_per_cpu_ctx[CPU_SEQID_UNSTABLE]; + kpreempt_disable(); + BLAKE3_CTX *ctx = blake3_per_cpu_ctx[CPU_SEQID]; #else BLAKE3_CTX *ctx = kmem_alloc(sizeof (*ctx), KM_SLEEP); #endif @@ -59,7 +60,9 @@ abd_checksum_blake3_native(abd_t *abd, uint64_t size, const void *ctx_template, (void) abd_iterate_func(abd, 0, size, blake3_incremental, ctx); Blake3_Final(ctx, (uint8_t *)zcp); -#if !defined(_KERNEL) +#if defined(_KERNEL) + kpreempt_enable(); +#else memset(ctx, 0, sizeof (*ctx)); kmem_free(ctx, sizeof (*ctx)); #endif From 012829df0c99d843c9c873b9be57796eaecb155b Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 2 May 2023 09:21:47 -0700 Subject: [PATCH 58/59] Wrap clang specific pragma Clang specific pragmas need to be wrapped to prevent a build warning when compiling with gcc. Reviewed-by: Tino Reichardt Signed-off-by: Brian Behlendorf Closes #14814 --- include/os/linux/zfs/sys/trace_zil.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/os/linux/zfs/sys/trace_zil.h b/include/os/linux/zfs/sys/trace_zil.h index fb03d3149f8f..7bddd9d1f469 100644 --- a/include/os/linux/zfs/sys/trace_zil.h +++ b/include/os/linux/zfs/sys/trace_zil.h @@ -153,8 +153,10 @@ * itx_t *, ...); */ +#if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wordered-compare-function-pointers" +#endif /* BEGIN CSTYLED */ DECLARE_EVENT_CLASS(zfs_zil_process_itx_class, TP_PROTO(zilog_t *zilog, itx_t *itx), @@ -172,7 +174,9 @@ DECLARE_EVENT_CLASS(zfs_zil_process_itx_class, ZILOG_TP_PRINTK_ARGS, ITX_TP_PRINTK_ARGS) ); /* END CSTYLED */ +#if defined(__clang__) #pragma clang diagnostic pop +#endif #define DEFINE_ZIL_PROCESS_ITX_EVENT(name) \ DEFINE_EVENT(zfs_zil_process_itx_class, name, \ From d96e29576c89e6e547cb82b477651d2b85ea0fed Mon Sep 17 00:00:00 2001 From: Pawel Jakub Dawidek Date: Wed, 3 May 2023 01:24:26 +0900 Subject: [PATCH 59/59] Use correct block pointer in block cloning case. Reviewed-by: Brian Behlendorf Reviewed-by: Brian Atkinson Signed-off-by: Pawel Jakub Dawidek Closes #14806 --- module/zfs/dbuf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index c7f76e8d96f8..8193fb244079 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -1620,8 +1620,7 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags, * If this is not true it indicates tampering and we report an error. */ if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bpp)) { - spa_log_error(db->db_objset->os_spa, &zb, - &db->db_blkptr->blk_birth); + spa_log_error(db->db_objset->os_spa, &zb, &bpp->blk_birth); zfs_panic_recover("unencrypted block in encrypted " "object set %llu", dmu_objset_id(db->db_objset)); err = SET_ERROR(EIO);